From dfb12f06283f22c157d99830de21700a7f86c139 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E4=BC=AF=E6=88=90?= <yiwei.liyw@alibaba-inc.com>
Date: Thu, 1 Mar 2018 11:46:56 +0800
Subject: [PATCH 0001/1098] Boost up performance for redis PUB-SUB patterns
 matching

If lots of clients PSUBSCRIBE to same patterns, multiple pattens matching will take place. This commit change it into just one single pattern matching by using a `dict *` to store the unique pattern and which clients subscribe to it.
---
 src/pubsub.c | 58 +++++++++++++++++++++++++++++++++++++++++-----------
 src/server.c |  1 +
 src/server.h |  1 +
 3 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/src/pubsub.c b/src/pubsub.c
index d1fffa20a..bbcfc1f43 100644
--- a/src/pubsub.c
+++ b/src/pubsub.c
@@ -125,6 +125,8 @@ int pubsubUnsubscribeChannel(client *c, robj *channel, int notify) {
 
 /* Subscribe a client to a pattern. Returns 1 if the operation succeeded, or 0 if the client was already subscribed to that pattern. */
 int pubsubSubscribePattern(client *c, robj *pattern) {
+    dictEntry *de;
+    list *clients;
     int retval = 0;
 
     if (listSearchKey(c->pubsub_patterns,pattern) == NULL) {
@@ -136,6 +138,16 @@ int pubsubSubscribePattern(client *c, robj *pattern) {
         pat->pattern = getDecodedObject(pattern);
         pat->client = c;
         listAddNodeTail(server.pubsub_patterns,pat);
+        /* Add the client to the pattern -> list of clients hash table */
+        de = dictFind(server.pubsub_patterns_dict,pattern);
+        if (de == NULL) {
+            clients = listCreate();
+            dictAdd(server.pubsub_patterns_dict,pattern,clients);
+            incrRefCount(pattern);
+        } else {
+            clients = dictGetVal(de);
+        }
+        listAddNodeTail(clients,c);
     }
     /* Notify the client */
     addReply(c,shared.mbulkhdr[3]);
@@ -148,6 +160,8 @@ int pubsubSubscribePattern(client *c, robj *pattern) {
 /* Unsubscribe a client from a channel. Returns 1 if the operation succeeded, or
  * 0 if the client was not subscribed to the specified channel. */
 int pubsubUnsubscribePattern(client *c, robj *pattern, int notify) {
+    dictEntry *de;
+    list *clients;
     listNode *ln;
     pubsubPattern pat;
     int retval = 0;
@@ -160,6 +174,18 @@ int pubsubUnsubscribePattern(client *c, robj *pattern, int notify) {
         pat.pattern = pattern;
         ln = listSearchKey(server.pubsub_patterns,&pat);
         listDelNode(server.pubsub_patterns,ln);
+        /* Remove the client from the pattern -> clients list hash table */
+        de = dictFind(server.pubsub_patterns_dict,pattern);
+        serverAssertWithInfo(c,NULL,de != NULL);
+        clients = dictGetVal(de);
+        ln = listSearchKey(clients,c);
+        serverAssertWithInfo(c,NULL,ln != NULL);
+        listDelNode(clients,ln);
+        if (listLength(clients) == 0) {
+            /* Free the list and associated hash entry at all if this was
+             * the latest client. */
+            dictDelete(server.pubsub_patterns_dict,pattern);
+        }
     }
     /* Notify the client */
     if (notify) {
@@ -225,6 +251,7 @@ int pubsubUnsubscribeAllPatterns(client *c, int notify) {
 int pubsubPublishMessage(robj *channel, robj *message) {
     int receivers = 0;
     dictEntry *de;
+    dictIterator *di;
     listNode *ln;
     listIter li;
 
@@ -247,25 +274,32 @@ int pubsubPublishMessage(robj *channel, robj *message) {
         }
     }
     /* Send to clients listening to matching channels */
-    if (listLength(server.pubsub_patterns)) {
-        listRewind(server.pubsub_patterns,&li);
+    di = dictGetIterator(server.pubsub_patterns_dict);
+    if (di) {
         channel = getDecodedObject(channel);
-        while ((ln = listNext(&li)) != NULL) {
-            pubsubPattern *pat = ln->value;
-
-            if (stringmatchlen((char*)pat->pattern->ptr,
-                                sdslen(pat->pattern->ptr),
+        while((de = dictNext(di)) != NULL) {
+            robj *pattern = dictGetKey(de);
+            list *clients = dictGetVal(de);
+            if (!stringmatchlen((char*)pattern->ptr,
+                                sdslen(pattern->ptr),
                                 (char*)channel->ptr,
                                 sdslen(channel->ptr),0)) {
-                addReply(pat->client,shared.mbulkhdr[4]);
-                addReply(pat->client,shared.pmessagebulk);
-                addReplyBulk(pat->client,pat->pattern);
-                addReplyBulk(pat->client,channel);
-                addReplyBulk(pat->client,message);
+                continue;
+            }
+            listRewind(clients,&li);
+            while ((ln = listNext(&li)) != NULL) {
+                client *c = listNodeValue(ln);
+
+                addReply(c,shared.mbulkhdr[4]);
+                addReply(c,shared.pmessagebulk);
+                addReplyBulk(c,pattern);
+                addReplyBulk(c,channel);
+                addReplyBulk(c,message);
                 receivers++;
             }
         }
         decrRefCount(channel);
+        dictReleaseIterator(di);
     }
     return receivers;
 }
diff --git a/src/server.c b/src/server.c
index 1a6f30381..a071fceed 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1900,6 +1900,7 @@ void initServer(void) {
     evictionPoolAlloc(); /* Initialize the LRU keys pool. */
     server.pubsub_channels = dictCreate(&keylistDictType,NULL);
     server.pubsub_patterns = listCreate();
+    server.pubsub_patterns_dict = dictCreate(&keylistDictType,NULL);
     listSetFreeMethod(server.pubsub_patterns,freePubsubPattern);
     listSetMatchMethod(server.pubsub_patterns,listMatchPubsubPattern);
     server.cronloops = 0;
diff --git a/src/server.h b/src/server.h
index 29919f5ee..a56663495 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1163,6 +1163,7 @@ struct redisServer {
     /* Pubsub */
     dict *pubsub_channels;  /* Map channels to list of subscribed clients */
     list *pubsub_patterns;  /* A list of pubsub_patterns */
+    dict *pubsub_patterns_dict;  /* A dict of pubsub_patterns */
     int notify_keyspace_events; /* Events to propagate via Pub/Sub. This is an
                                    xor of NOTIFY_... flags. */
     /* Cluster */

From 8b33975944397a0522f6d05d0776d9871a7c6be0 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Tue, 21 May 2019 11:37:13 +0800
Subject: [PATCH 0002/1098] Threaded IO: use main thread to handle write work

---
 src/networking.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 4bc22120a..4da762a15 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2525,7 +2525,7 @@ pthread_mutex_t io_threads_mutex[IO_THREADS_MAX_NUM];
 _Atomic unsigned long io_threads_pending[IO_THREADS_MAX_NUM];
 int io_threads_active;  /* Are the threads currently spinning waiting I/O? */
 int io_threads_op;      /* IO_THREADS_OP_WRITE or IO_THREADS_OP_READ. */
-list *io_threads_list[IO_THREADS_MAX_NUM];
+list *io_threads_list[IO_THREADS_MAX_NUM+1];
 
 void *IOThreadMain(void *myid) {
     /* The ID is the thread number (from 0 to server.iothreads_num-1), and is
@@ -2598,6 +2598,7 @@ void initThreadedIO(void) {
         }
         io_threads[i] = tid;
     }
+    io_threads_list[server.io_threads_num] = listCreate();
 }
 
 void startThreadedIO(void) {
@@ -2669,7 +2670,7 @@ int handleClientsWithPendingWritesUsingThreads(void) {
     while((ln = listNext(&li))) {
         client *c = listNodeValue(ln);
         c->flags &= ~CLIENT_PENDING_WRITE;
-        int target_id = item_id % server.io_threads_num;
+        int target_id = item_id % (server.io_threads_num+1);
         listAddNodeTail(io_threads_list[target_id],c);
         item_id++;
     }
@@ -2682,6 +2683,13 @@ int handleClientsWithPendingWritesUsingThreads(void) {
         io_threads_pending[j] = count;
     }
 
+    listRewind(io_threads_list[server.io_threads_num],&li);
+    while((ln = listNext(&li))) {
+        client *c = listNodeValue(ln);
+        writeToClient(c->fd,c,0);
+    }
+    listEmpty(io_threads_list[server.io_threads_num]);
+
     /* Wait for all threads to end their work. */
     while(1) {
         unsigned long pending = 0;

From 1398fac3f1bd190c572a695aa898d71d1573f7e5 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Tue, 21 May 2019 11:42:10 +0800
Subject: [PATCH 0003/1098] Threaded IO: use main thread to handle read work

---
 src/networking.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index 4da762a15..c2ae35181 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2755,7 +2755,7 @@ int handleClientsWithPendingReadsUsingThreads(void) {
     int item_id = 0;
     while((ln = listNext(&li))) {
         client *c = listNodeValue(ln);
-        int target_id = item_id % server.io_threads_num;
+        int target_id = item_id % (server.io_threads_num+1);
         listAddNodeTail(io_threads_list[target_id],c);
         item_id++;
     }
@@ -2768,6 +2768,13 @@ int handleClientsWithPendingReadsUsingThreads(void) {
         io_threads_pending[j] = count;
     }
 
+    listRewind(io_threads_list[server.io_threads_num],&li);
+    while((ln = listNext(&li))) {
+        client *c = listNodeValue(ln);
+        readQueryFromClient(NULL,c->fd,c,0);
+    }
+    listEmpty(io_threads_list[server.io_threads_num]);
+
     /* Wait for all threads to end their work. */
     while(1) {
         unsigned long pending = 0;

From 45af2b174270af23a63b9032b2cbbb342d4e3a5f Mon Sep 17 00:00:00 2001
From: Jamie Scott <5336227+IAmATeaPot418@users.noreply.github.com>
Date: Wed, 16 Oct 2019 13:31:19 -0700
Subject: [PATCH 0004/1098] Update to directive in redis.conf (missing s)

The directive tls-prefer-server-cipher is actually tls-prefer-server-ciphers in config.c. This results in a failed directive call shown below. This pull request adds the "s" in ciphers so that the directive is able to be properly called in config.c

ubuntu@ip-172-31-16-31:~/redis$ src/redis-server ./redis.conf

*** FATAL CONFIG FILE ERROR ***
Reading the configuration file, at line 200
>>> 'tls-prefer-server-cipher yes'
Bad directive or wrong number of arguments
---
 redis.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/redis.conf b/redis.conf
index 408426f15..6e02cfed7 100644
--- a/redis.conf
+++ b/redis.conf
@@ -197,7 +197,7 @@ tcp-keepalive 300
 # When choosing a cipher, use the server's preference instead of the client
 # preference. By default, the server follows the client's preference.
 #
-# tls-prefer-server-cipher yes
+# tls-prefer-server-ciphers yes
 
 ################################# GENERAL #####################################
 

From 8beec4f0e78cebce9cd14d72fa4a622538b7c045 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Tue, 5 Nov 2019 19:23:37 +0530
Subject: [PATCH 0005/1098] Make sure Redis does not reply with negative zero

---
 src/util.c               | 4 ++++
 tests/unit/type/incr.tcl | 6 ++++++
 2 files changed, 10 insertions(+)

diff --git a/src/util.c b/src/util.c
index 062a572d4..80b5a2453 100644
--- a/src/util.c
+++ b/src/util.c
@@ -560,6 +560,10 @@ int ld2string(char *buf, size_t len, long double value, ld2string_mode mode) {
                 }
                 if (*p == '.') l--;
             }
+            if (l == 2 && buf[0] == '-' && buf[1] == '0') {
+                buf[0] = '0';
+                l = 1;
+            }
             break;
         default: return 0; /* Invalid mode. */
         }
diff --git a/tests/unit/type/incr.tcl b/tests/unit/type/incr.tcl
index a58710d39..63bf2e116 100644
--- a/tests/unit/type/incr.tcl
+++ b/tests/unit/type/incr.tcl
@@ -151,4 +151,10 @@ start_server {tags {"incr"}} {
         catch {r incrbyfloat foo 1} err
         format $err
     } {ERR*valid*}
+
+    test {No negative zero} {
+        r incrbyfloat foo [expr double(1)/41]
+        r incrbyfloat foo [expr double(-1)/41]
+        r get foo
+    } {0}
 }

From 173cca54135230ba2a8d180574a56baee67304ea Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 10 Dec 2019 11:16:13 +0200
Subject: [PATCH 0006/1098] fix possible warning on incomplete struct init

---
 src/module.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/module.c b/src/module.c
index 111e1c202..096a0bf2c 100644
--- a/src/module.c
+++ b/src/module.c
@@ -5652,7 +5652,7 @@ sds modulesCollectInfo(sds info, const char *section, int for_crash_report, int
         struct RedisModule *module = dictGetVal(de);
         if (!module->info_cb)
             continue;
-        RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0};
+        RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0, 0};
         module->info_cb(&info_ctx, for_crash_report);
         /* Implicitly end dicts (no way to handle errors, and we must add the newline). */
         if (info_ctx.in_dict_field)

From 5e0fe4c77c556255b738cecb330dc1bae060a3d0 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Wed, 18 Dec 2019 12:27:03 +0530
Subject: [PATCH 0007/1098] streamReplyWithRangeFromConsumerPEL: Redundant
 streamDecodeID

---
 src/t_stream.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index a499f7381..c35c9d1be 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1043,9 +1043,7 @@ size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start
              * by the user by other means. In that case we signal it emitting
              * the ID but then a NULL entry for the fields. */
             addReplyArrayLen(c,2);
-            streamID id;
-            streamDecodeID(ri.key,&id);
-            addReplyStreamID(c,&id);
+            addReplyStreamID(c,&thisid);
             addReplyNullArray(c);
         } else {
             streamNACK *nack = ri.data;

From 3c0ed0309ac5bae52464ecb45e92056e212f2b7f Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Wed, 17 Jul 2019 11:00:51 +0800
Subject: [PATCH 0008/1098] lazyfree: add a new configuration
 lazyfree-lazy-user-del

Delete keys in async way when executing DEL command, if
lazyfree-lazy-user-del is yes.
---
 redis.conf   | 5 ++++-
 src/config.c | 1 +
 src/db.c     | 2 +-
 src/server.h | 1 +
 4 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/redis.conf b/redis.conf
index 870849a79..a0e1b5aa9 100644
--- a/redis.conf
+++ b/redis.conf
@@ -924,7 +924,9 @@ replica-priority 100
 #    or SORT with STORE option may delete existing keys. The SET command
 #    itself removes any old content of the specified key in order to replace
 #    it with the specified string.
-# 4) During replication, when a replica performs a full resynchronization with
+# 4) The DEL command itself, and normally it's not easy to replace DEL with
+#    UNLINK in user's codes.
+# 5) During replication, when a replica performs a full resynchronization with
 #    its master, the content of the whole database is removed in order to
 #    load the RDB file just transferred.
 #
@@ -936,6 +938,7 @@ replica-priority 100
 lazyfree-lazy-eviction no
 lazyfree-lazy-expire no
 lazyfree-lazy-server-del no
+lazyfree-lazy-user-del no
 replica-lazy-flush no
 
 ############################## APPEND ONLY MODE ###############################
diff --git a/src/config.c b/src/config.c
index 3d53e656b..0bb2fc237 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2147,6 +2147,7 @@ standardConfig configs[] = {
     createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, 0, NULL, NULL),
     createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, 0, NULL, NULL),
     createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, 0, NULL, NULL),
+    createBoolConfig("lazyfree-lazy-user-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_user_del , 0, NULL, NULL),
     createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, 0, NULL, NULL),
     createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, 0, NULL, NULL),
     createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, 0, NULL, NULL),
diff --git a/src/db.c b/src/db.c
index 483095a7c..88102e557 100644
--- a/src/db.c
+++ b/src/db.c
@@ -536,7 +536,7 @@ void delGenericCommand(client *c, int lazy) {
 }
 
 void delCommand(client *c) {
-    delGenericCommand(c,0);
+    delGenericCommand(c,server.lazyfree_lazy_user_del);
 }
 
 void unlinkCommand(client *c) {
diff --git a/src/server.h b/src/server.h
index 7a78c884f..8c25534fe 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1375,6 +1375,7 @@ struct redisServer {
     int lazyfree_lazy_eviction;
     int lazyfree_lazy_expire;
     int lazyfree_lazy_server_del;
+    int lazyfree_lazy_user_del;
     /* Latency monitor */
     long long latency_monitor_threshold;
     dict *latency_events;

From dfb598cf3304818e608ceb6b5d9529a293345c4a Mon Sep 17 00:00:00 2001
From: Johannes Truschnigg <johannes@truschnigg.info>
Date: Thu, 19 Dec 2019 21:47:24 +0100
Subject: [PATCH 0009/1098] Signal systemd readiness atfer Partial Resync

"Partial Resynchronization" is a special variant of replication success
that we have to tell systemd about if it is managing redis-server via a
Type=Notify service unit.
---
 src/replication.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/replication.c b/src/replication.c
index 68dc77a61..01619810c 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2172,6 +2172,10 @@ void syncWithMaster(connection *conn) {
 
     if (psync_result == PSYNC_CONTINUE) {
         serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: Master accepted a Partial Resynchronization.");
+        if (server.supervised_mode == SUPERVISED_SYSTEMD) {
+            redisCommunicateSystemd("STATUS=MASTER <-> REPLICA sync: Partial Resynchronization accepted. Ready to accept connections.\n");
+            redisCommunicateSystemd("READY=1\n");
+        }
         return;
     }
 

From 324e22accf457edc996971bc97f5474349cd7c4c Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 20 Dec 2019 12:29:02 +0100
Subject: [PATCH 0010/1098] Fix ip and missing mode in RM_GetClusterNodeInfo().

---
 src/module.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/module.c b/src/module.c
index 33b69f8c2..705ffec2c 100644
--- a/src/module.c
+++ b/src/module.c
@@ -5082,10 +5082,13 @@ int RM_GetClusterNodeInfo(RedisModuleCtx *ctx, const char *id, char *ip, char *m
     UNUSED(ctx);
 
     clusterNode *node = clusterLookupNode(id);
-    if (node->flags & (CLUSTER_NODE_NOADDR|CLUSTER_NODE_HANDSHAKE))
+    if (node == NULL ||
+        node->flags & (CLUSTER_NODE_NOADDR|CLUSTER_NODE_HANDSHAKE))
+    {
         return REDISMODULE_ERR;
+    }
 
-    if (ip) memcpy(ip,node->name,REDISMODULE_NODE_ID_LEN);
+    if (ip) strncpy(ip,node->ip,NET_IP_STR_LEN);
 
     if (master_id) {
         /* If the information is not available, the function will set the

From 337dcde9fafa6c4bab78f1ec74f8e8abf253c86c Mon Sep 17 00:00:00 2001
From: "bodong.ybd" <bodong.ybd@alibaba-inc.com>
Date: Sat, 21 Dec 2019 21:27:38 +0800
Subject: [PATCH 0011/1098] Fix spop return nil #4709

---
 src/t_set.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_set.c b/src/t_set.c
index abbf82fde..60cf22d8c 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -415,7 +415,7 @@ void spopWithCountCommand(client *c) {
 
     /* Make sure a key with the name inputted exists, and that it's type is
      * indeed a set. Otherwise, return nil */
-    if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp]))
+    if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.emptyset[c->resp]))
         == NULL || checkType(c,set,OBJ_SET)) return;
 
     /* If count is zero, serve an empty set ASAP to avoid special

From af02478ba02da8320d80661ccbf91a5b125b03ab Mon Sep 17 00:00:00 2001
From: Khem Raj <raj.khem@gmail.com>
Date: Sat, 21 Dec 2019 11:17:50 -0800
Subject: [PATCH 0012/1098] Mark extern definition of SDS_NOINIT in sds.h

This helps in avoiding multiple definition of this variable, its also
defined globally in sds.c

Signed-off-by: Khem Raj <raj.khem@gmail.com>
---
 src/sds.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sds.h b/src/sds.h
index 1bdb60dec..adcc12c0a 100644
--- a/src/sds.h
+++ b/src/sds.h
@@ -34,7 +34,7 @@
 #define __SDS_H
 
 #define SDS_MAX_PREALLOC (1024*1024)
-const char *SDS_NOINIT;
+extern const char *SDS_NOINIT;
 
 #include <sys/types.h>
 #include <stdarg.h>

From 660645c7421fab93b69205ff7208e577ef225f0d Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 23 Dec 2019 10:15:52 +0200
Subject: [PATCH 0013/1098] modules don't signalModifiedKey in setKey() since
 that's done (optionally) in RM_CloseKey

---
 src/db.c       | 6 +++---
 src/module.c   | 6 +++---
 src/server.h   | 2 +-
 src/t_string.c | 2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/db.c b/src/db.c
index 32bb35652..bb93a6071 100644
--- a/src/db.c
+++ b/src/db.c
@@ -220,7 +220,7 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) {
  *    unless 'keepttl' is true.
  *
  * All the new keys in the database should be created via this interface. */
-void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl) {
+void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal) {
     if (lookupKeyWrite(db,key) == NULL) {
         dbAdd(db,key,val);
     } else {
@@ -228,12 +228,12 @@ void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl) {
     }
     incrRefCount(val);
     if (!keepttl) removeExpire(db,key);
-    signalModifiedKey(db,key);
+    if (signal) signalModifiedKey(db,key);
 }
 
 /* Common case for genericSetKey() where the TTL is not retained. */
 void setKey(redisDb *db, robj *key, robj *val) {
-    genericSetKey(db,key,val,0);
+    genericSetKey(db,key,val,0,1);
 }
 
 /* Return true if the specified key exists in the specified database.
diff --git a/src/module.c b/src/module.c
index 705ffec2c..afeb9c046 100644
--- a/src/module.c
+++ b/src/module.c
@@ -2128,7 +2128,7 @@ RedisModuleString *RM_RandomKey(RedisModuleCtx *ctx) {
 int RM_StringSet(RedisModuleKey *key, RedisModuleString *str) {
     if (!(key->mode & REDISMODULE_WRITE) || key->iter) return REDISMODULE_ERR;
     RM_DeleteKey(key);
-    setKey(key->db,key->key,str);
+    genericSetKey(key->db,key->key,str,0,0);
     key->value = str;
     return REDISMODULE_OK;
 }
@@ -2208,7 +2208,7 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) {
     if (key->value == NULL) {
         /* Empty key: create it with the new size. */
         robj *o = createObject(OBJ_STRING,sdsnewlen(NULL, newlen));
-        setKey(key->db,key->key,o);
+        genericSetKey(key->db,key->key,o,0,0);
         key->value = o;
         decrRefCount(o);
     } else {
@@ -3597,7 +3597,7 @@ int RM_ModuleTypeSetValue(RedisModuleKey *key, moduleType *mt, void *value) {
     if (!(key->mode & REDISMODULE_WRITE) || key->iter) return REDISMODULE_ERR;
     RM_DeleteKey(key);
     robj *o = createModuleObject(mt,value);
-    setKey(key->db,key->key,o);
+    genericSetKey(key->db,key->key,o,0,0);
     decrRefCount(o);
     key->value = o;
     return REDISMODULE_OK;
diff --git a/src/server.h b/src/server.h
index 8e354c03d..c6d40f704 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2032,7 +2032,7 @@ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
 #define LOOKUP_NOTOUCH (1<<0)
 void dbAdd(redisDb *db, robj *key, robj *val);
 void dbOverwrite(redisDb *db, robj *key, robj *val);
-void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl);
+void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal);
 void setKey(redisDb *db, robj *key, robj *val);
 int dbExists(redisDb *db, robj *key);
 robj *dbRandomKey(redisDb *db);
diff --git a/src/t_string.c b/src/t_string.c
index 8ccd69eb9..3174e9ccd 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -84,7 +84,7 @@ void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire,
         addReply(c, abort_reply ? abort_reply : shared.null[c->resp]);
         return;
     }
-    genericSetKey(c->db,key,val,flags & OBJ_SET_KEEPTTL);
+    genericSetKey(c->db,key,val,flags & OBJ_SET_KEEPTTL,1);
     server.dirty++;
     if (expire) setExpire(c,c->db,key,mstime()+milliseconds);
     notifyKeyspaceEvent(NOTIFY_STRING,"set",key,c->db->id);

From 0c898bdb07e7d5716db7a3b311a1d7586ffbdca0 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 6 Jun 2019 20:08:26 +0300
Subject: [PATCH 0014/1098] Add REDISMODULE_CTX_FLAGS_MULTI_DIRTY.

---
 src/module.c      | 6 ++++++
 src/redismodule.h | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/src/module.c b/src/module.c
index 705ffec2c..965bb4460 100644
--- a/src/module.c
+++ b/src/module.c
@@ -1842,6 +1842,12 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) {
          flags |= REDISMODULE_CTX_FLAGS_REPLICATED;
     }
 
+    /* For DIRTY flags, we need the blocked client if used */
+    client *c = ctx->blocked_client ? ctx->blocked_client->client : ctx->client;
+    if (c && (c->flags & (CLIENT_DIRTY_CAS|CLIENT_DIRTY_EXEC))) {
+        flags |= REDISMODULE_CTX_FLAGS_MULTI_DIRTY;
+    }
+
     if (server.cluster_enabled)
         flags |= REDISMODULE_CTX_FLAGS_CLUSTER;
 
diff --git a/src/redismodule.h b/src/redismodule.h
index 57f98fb93..637078f2b 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -110,6 +110,8 @@
 #define REDISMODULE_CTX_FLAGS_REPLICA_IS_ONLINE (1<<17)
 /* There is currently some background process active. */
 #define REDISMODULE_CTX_FLAGS_ACTIVE_CHILD (1<<18)
+/* The next EXEC will fail due to dirty CAS (touched keys). */
+#define REDISMODULE_CTX_FLAGS_MULTI_DIRTY (1<<19)
 
 /* Keyspace changes notification classes. Every class is associated with a
  * character for configuration purposes.

From 1f75ce30dfdb1c8c4df8474f9580f2aff032cfb4 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 26 Dec 2019 15:31:37 +0530
Subject: [PATCH 0015/1098] Stream: Handle streamID-related edge cases

This commit solves several edge cases that are related to
exhausting the streamID limits: We should correctly calculate
the succeeding streamID instead of blindly incrementing 'seq'
This affects both XREAD and XADD.

Other (unrelated) changes:
Reply with a better error message when trying to add an entry
to a stream that has exhausted last_id
---
 src/blocked.c              |  2 +-
 src/stream.h               |  1 +
 src/t_stream.c             | 28 +++++++++++++++++++++++++---
 tests/unit/type/stream.tcl | 27 +++++++++++++++++++++++++++
 4 files changed, 54 insertions(+), 4 deletions(-)

diff --git a/src/blocked.c b/src/blocked.c
index 20c0e760a..06aa5850e 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -388,7 +388,7 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) {
 
             if (streamCompareID(&s->last_id, gt) > 0) {
                 streamID start = *gt;
-                start.seq++; /* Can't overflow, it's an uint64_t */
+                streamIncrID(&start);
 
                 /* Lookup the consumer for the group, if any. */
                 streamConsumer *consumer = NULL;
diff --git a/src/stream.h b/src/stream.h
index 7de769ba1..b69073994 100644
--- a/src/stream.h
+++ b/src/stream.h
@@ -111,5 +111,6 @@ streamNACK *streamCreateNACK(streamConsumer *consumer);
 void streamDecodeID(void *buf, streamID *id);
 int streamCompareID(streamID *a, streamID *b);
 void streamFreeNACK(streamNACK *na);
+void streamIncrID(streamID *id);
 
 #endif
diff --git a/src/t_stream.c b/src/t_stream.c
index a499f7381..3d46ca0da 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -73,6 +73,21 @@ unsigned long streamLength(const robj *subject) {
     return s->length;
 }
 
+/* Set 'id' to be its successor streamID */
+void streamIncrID(streamID *id) {
+    if (id->seq == UINT64_MAX) {
+        if (id->ms == UINT64_MAX) {
+            /* Special case where 'id' is the last possible streamID... */
+            id->ms = id->seq = 0;
+        } else {
+            id->ms++;
+            id->seq = 0;
+        }
+    } else {
+        id->seq++;
+    }
+}
+
 /* Generate the next stream item ID given the previous one. If the current
  * milliseconds Unix time is greater than the previous one, just use this
  * as time part and start with sequence part of zero. Otherwise we use the
@@ -83,8 +98,8 @@ void streamNextID(streamID *last_id, streamID *new_id) {
         new_id->ms = ms;
         new_id->seq = 0;
     } else {
-        new_id->ms = last_id->ms;
-        new_id->seq = last_id->seq+1;
+        *new_id = *last_id;
+        streamIncrID(new_id);
     }
 }
 
@@ -1220,6 +1235,13 @@ void xaddCommand(client *c) {
     if ((o = streamTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return;
     s = o->ptr;
 
+    /* Return ASAP if the stream has reached the last possible ID */
+    if (s->last_id.ms == UINT64_MAX && s->last_id.seq == UINT64_MAX) {
+        addReplyError(c,"The stream has exhausted the last possible ID, "
+                        "unable to add more items");
+        return;
+    }
+
     /* Append using the low level function and return the ID. */
     if (streamAppendItem(s,c->argv+field_pos,(c->argc-field_pos)/2,
         &id, id_given ? &id : NULL)
@@ -1509,7 +1531,7 @@ void xreadCommand(client *c) {
              * so start from the next ID, since we want only messages with
              * IDs greater than start. */
             streamID start = *gt;
-            start.seq++; /* uint64_t can't overflow in this context. */
+            streamIncrID(&start);
 
             /* Emit the two elements sub-array consisting of the name
              * of the stream and the data we extracted from it. */
diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl
index 656bac5de..9840e3b74 100644
--- a/tests/unit/type/stream.tcl
+++ b/tests/unit/type/stream.tcl
@@ -328,6 +328,33 @@ start_server {
 
         assert_equal [r xrevrange teststream2 1234567891245 -] {{1234567891240-0 {key1 value2}} {1234567891230-0 {key1 value1}}}
     }
+
+    test {XREAD streamID edge (no-blocking)} {
+        r del x
+        r XADD x 1-1 f v
+        r XADD x 1-18446744073709551615 f v
+        r XADD x 2-1 f v
+        set res [r XREAD BLOCK 0 STREAMS x 1-18446744073709551615]
+        assert {[lindex $res 0 1 0] == {2-1 {f v}}}
+    }
+
+    test {XREAD streamID edge (blocking)} {
+        r del x
+        set rd [redis_deferring_client]
+        $rd XREAD BLOCK 0 STREAMS x 1-18446744073709551615
+        r XADD x 1-1 f v
+        r XADD x 1-18446744073709551615 f v
+        r XADD x 2-1 f v
+        set res [$rd read]
+        assert {[lindex $res 0 1 0] == {2-1 {f v}}}
+    }
+
+    test {XADD streamID edge} {
+        r del x
+        r XADD x 2577343934890-18446744073709551615 f v ;# we need the timestamp to be in the future
+        r XADD x * f2 v2
+        assert_equal [r XRANGE x - +] {{2577343934890-18446744073709551615 {f v}} {2577343934891-0 {f2 v2}}}
+    }
 }
 
 start_server {tags {"stream"} overrides {appendonly yes}} {

From 0c3fe52ef7e3acf135428be81dd8e27b4ef191e5 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 26 Dec 2019 13:59:58 +0200
Subject: [PATCH 0016/1098] config.c adjust config limits and mutable

- make lua-replicate-commands mutable (it never was, but i don't see why)
- make tcp-backlog immutable (fix a recent refactory mistake)
- increase the max limit of a few configs to match what they were before
the recent refactory
---
 src/config.c                 | 12 ++++++------
 tests/unit/introspection.tcl |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/config.c b/src/config.c
index 3d53e656b..9aa847183 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2137,7 +2137,7 @@ standardConfig configs[] = {
     createBoolConfig("rdbchecksum", NULL, IMMUTABLE_CONFIG, server.rdb_checksum, 1, NULL, NULL),
     createBoolConfig("daemonize", NULL, IMMUTABLE_CONFIG, server.daemonize, 0, NULL, NULL),
     createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, server.io_threads_do_reads, 0,NULL, NULL), /* Read + parse from threads? */
-    createBoolConfig("lua-replicate-commands", NULL, IMMUTABLE_CONFIG, server.lua_always_replicate_commands, 1, NULL, NULL),
+    createBoolConfig("lua-replicate-commands", NULL, MODIFIABLE_CONFIG, server.lua_always_replicate_commands, 1, NULL, NULL),
     createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, 0, NULL, NULL),
     createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, 1, NULL, NULL),
     createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, 1, NULL, NULL),
@@ -2209,7 +2209,7 @@ standardConfig configs[] = {
     createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.maxmemory_samples, 5, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.maxidletime, 0, INTEGER_CONFIG, NULL, NULL), /* Default client timeout: infinite */
     createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, server.slave_announce_port, 0, INTEGER_CONFIG, NULL, NULL),
-    createIntConfig("tcp-backlog", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, 511, INTEGER_CONFIG, NULL, NULL), /* TCP listen backlog. */
+    createIntConfig("tcp-backlog", NULL, IMMUTABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, 511, INTEGER_CONFIG, NULL, NULL), /* TCP listen backlog. */
     createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_bus_port, 0, INTEGER_CONFIG, NULL, NULL), /* Default: Use +10000 offset. */
     createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_port, 0, INTEGER_CONFIG, NULL, NULL), /* Use server.port */
     createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_timeout, 60, INTEGER_CONFIG, NULL, NULL),
@@ -2235,9 +2235,9 @@ standardConfig configs[] = {
     createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, 15000, INTEGER_CONFIG, NULL, NULL),
     createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, 10000, INTEGER_CONFIG, NULL, NULL),
     createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, 0, INTEGER_CONFIG, NULL, NULL),
-    createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */
-    createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL),
-    createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */
+    createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */
+    createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL),
+    createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */
 
     /* Unsigned Long Long configs */
     createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory),
@@ -2246,7 +2246,7 @@ standardConfig configs[] = {
     createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, 512, INTEGER_CONFIG, NULL, NULL),
     createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, 512, INTEGER_CONFIG, NULL, NULL),
     createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, 128, INTEGER_CONFIG, NULL, NULL),
-    createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_ignore_bytes, 100<<20, MEMORY_CONFIG, NULL, NULL), /* Default: don't defrag if frag overhead is below 100mb */
+    createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.active_defrag_ignore_bytes, 100<<20, MEMORY_CONFIG, NULL, NULL), /* Default: don't defrag if frag overhead is below 100mb */
     createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL),
     createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, 4096, MEMORY_CONFIG, NULL, NULL),
     createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL),
diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl
index 3bf3d367b..cd905084a 100644
--- a/tests/unit/introspection.tcl
+++ b/tests/unit/introspection.tcl
@@ -65,7 +65,7 @@ start_server {tags {"introspection"}} {
             rdbchecksum
             daemonize
             io-threads-do-reads
-            lua-replicate-commands
+            tcp-backlog
             always-show-logo
             syslog-enabled
             cluster-enabled

From 5521910de77d8a9d7587ec8497cc589c4245a599 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Sun, 29 Dec 2019 15:40:40 +0100
Subject: [PATCH 0017/1098] Inline protocol: handle empty strings well.

This bug is from the first version of Redis. Probably the problem here
is that before we used an SDS split function that created empty strings
for additional spaces, like in "SET    foo          bar".
AFAIK later we replaced it with the curretn sdssplitarg() API that has
no such a problem. As a result, we introduced a bug, where it is no
longer possible to do something like:

    SET foo ""

Using the inline protocol. Now it is fixed.
---
 src/networking.c | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 37f8fa9b7..dac45751d 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1453,12 +1453,8 @@ int processInlineBuffer(client *c) {
 
     /* Create redis objects for all arguments. */
     for (c->argc = 0, j = 0; j < argc; j++) {
-        if (sdslen(argv[j])) {
-            c->argv[c->argc] = createObject(OBJ_STRING,argv[j]);
-            c->argc++;
-        } else {
-            sdsfree(argv[j]);
-        }
+        c->argv[c->argc] = createObject(OBJ_STRING,argv[j]);
+        c->argc++;
     }
     zfree(argv);
     return C_OK;

From 6e4f70b8170e14ba8a2ada2fde4a75e073effee3 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Sun, 29 Dec 2019 15:44:59 +0100
Subject: [PATCH 0018/1098] Fix duplicated CLIENT SETNAME reply.

Happened when we set the name to "" to cancel the name.
Was introduced during the RESP3 refactoring.

See #6036.
---
 src/networking.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index dac45751d..a558ae91a 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1998,7 +1998,6 @@ int clientSetNameOrReply(client *c, robj *name) {
     if (len == 0) {
         if (c->name) decrRefCount(c->name);
         c->name = NULL;
-        addReply(c,shared.ok);
         return C_OK;
     }
 

From a351e74fe9e5d2da5b567520b485d55eff99771f Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 26 Dec 2019 12:19:24 +0530
Subject: [PATCH 0019/1098] Blocking XREAD[GROUP] should always reply with
 valid data (or timeout)

This commit solves the following bug:
127.0.0.1:6379> XGROUP CREATE x grp $ MKSTREAM
OK
127.0.0.1:6379> XADD x 666 f v
"666-0"
127.0.0.1:6379> XREADGROUP GROUP grp Alice BLOCK 0 STREAMS x >
1) 1) "x"
   2) 1) 1) "666-0"
         2) 1) "f"
            2) "v"
127.0.0.1:6379> XADD x 667 f v
"667-0"
127.0.0.1:6379> XDEL x 667
(integer) 1
127.0.0.1:6379> XREADGROUP GROUP grp Alice BLOCK 0 STREAMS x >
1) 1) "x"
   2) (empty array)

The root cause is that we use s->last_id in streamCompareID
while we should use the last *valid* ID
---
 src/t_stream.c                     | 29 +++++++++++++++++++----------
 tests/unit/type/stream-cgroups.tcl | 14 ++++++++++++++
 tests/unit/type/stream.tcl         | 11 +++++++++++
 3 files changed, 44 insertions(+), 10 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index a499f7381..d1b39d2f3 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -782,6 +782,16 @@ int streamDeleteItem(stream *s, streamID *id) {
     return deleted;
 }
 
+/* Get the last valid (non-tombstone) streamID of 's'. */
+void streamLastValidID(stream *s, streamID *maxid)
+{
+    streamIterator si;
+    streamIteratorStart(&si,s,NULL,NULL,1);
+    int64_t numfields;
+    streamIteratorGetID(&si,maxid,&numfields);
+    streamIteratorStop(&si);
+}
+
 /* Emit a reply in the client output buffer by formatting a Stream ID
  * in the standard <ms>-<seq> format, using the simple string protocol
  * of REPL. */
@@ -1484,20 +1494,23 @@ void xreadCommand(client *c) {
             {
                 serve_synchronously = 1;
                 serve_history = 1;
-            } else {
+            } else if (s->length) {
                 /* We also want to serve a consumer in a consumer group
                  * synchronously in case the group top item delivered is smaller
                  * than what the stream has inside. */
-                streamID *last = &groups[i]->last_id;
-                if (s->length && (streamCompareID(&s->last_id, last) > 0)) {
+                streamID maxid, *last = &groups[i]->last_id;
+                streamLastValidID(s, &maxid);
+                if (streamCompareID(&maxid, last) > 0) {
                     serve_synchronously = 1;
                     *gt = *last;
                 }
             }
-        } else {
+        } else if (s->length) {
             /* For consumers without a group, we serve synchronously if we can
              * actually provide at least one item from the stream. */
-            if (s->length && (streamCompareID(&s->last_id, gt) > 0)) {
+            streamID maxid;
+            streamLastValidID(s, &maxid);
+            if (streamCompareID(&maxid, gt) > 0) {
                 serve_synchronously = 1;
             }
         }
@@ -1849,11 +1862,7 @@ void xsetidCommand(client *c) {
      * item, otherwise the fundamental ID monotonicity assumption is violated. */
     if (s->length > 0) {
         streamID maxid;
-        streamIterator si;
-        streamIteratorStart(&si,s,NULL,NULL,1);
-        int64_t numfields;
-        streamIteratorGetID(&si,&maxid,&numfields);
-        streamIteratorStop(&si);
+        streamLastValidID(s,&maxid);
 
         if (streamCompareID(&id,&maxid) < 0) {
             addReplyError(c,"The ID specified in XSETID is smaller than the "
diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl
index 34d4061c2..a59e168ef 100644
--- a/tests/unit/type/stream-cgroups.tcl
+++ b/tests/unit/type/stream-cgroups.tcl
@@ -147,6 +147,20 @@ start_server {
         assert {[lindex $res 0 1 1] == {2-0 {field1 B}}}
     }
 
+    test {Blocking XREADGROUP will not reply with an empty array} {
+        r del mystream
+        r XGROUP CREATE mystream mygroup $ MKSTREAM
+        r XADD mystream 666 f v
+        set res [r XREADGROUP GROUP mygroup Alice BLOCK 10 STREAMS mystream ">"]
+        assert {[lindex $res 0 1 0] == {666-0 {f v}}}
+        r XADD mystream 667 f2 v2
+        r XDEL mystream 667
+        set rd [redis_deferring_client]
+        $rd XREADGROUP GROUP mygroup Alice BLOCK 10 STREAMS mystream ">"
+        after 20
+        assert {[$rd read] == {}} ;# before the fix, client didn't even block, but was served synchronously with {mystream {}}
+    }
+
     test {XCLAIM can claim PEL items from another consumer} {
         # Add 3 items into the stream, and create a consumer group
         r del mystream
diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl
index 656bac5de..585e2a634 100644
--- a/tests/unit/type/stream.tcl
+++ b/tests/unit/type/stream.tcl
@@ -191,6 +191,17 @@ start_server {
         assert {[lindex $res 0 1 0 1] eq {old abcd1234}}
     }
 
+    test {Blocking XREAD will not reply with an empty array} {
+        r del s1
+        r XADD s1 666 f v
+        r XADD s1 667 f2 v2
+        r XDEL s1 667
+        set rd [redis_deferring_client]
+        $rd XREAD BLOCK 10 STREAMS s1 666
+        after 20
+        assert {[$rd read] == {}} ;# before the fix, client didn't even block, but was served synchronously with {s1 {}}
+    }
+
     test "XREAD: XADD + DEL should not awake client" {
         set rd [redis_deferring_client]
         r del s1

From d7d13721d38f391f31ed0d9b1081b59245b468ab Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Tue, 24 Dec 2019 17:14:23 +0530
Subject: [PATCH 0020/1098] Modules: Fix blocked-client-related memory leak

If a blocked module client times-out (or disconnects, unblocked
by CLIENT command, etc.) we need to call moduleUnblockClient
in order to free memory allocated by the module sub-system
and blocked-client private data

Other changes:
Made blockedonkeys.tcl tests a bit more aggressive in order
to smoke-out potential memory leaks
---
 src/module.c                         |  9 +++++++
 tests/modules/blockonkeys.c          | 13 ++++++-----
 tests/unit/moduleapi/blockonkeys.tcl | 35 ++++++++++++++++++++++++++++
 3 files changed, 51 insertions(+), 6 deletions(-)

diff --git a/src/module.c b/src/module.c
index 705ffec2c..cbab735f1 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4271,6 +4271,15 @@ void unblockClientFromModule(client *c) {
         moduleFreeContext(&ctx);
     }
 
+    /* If we made it here and client is still blocked it means that the command
+     * timed-out, client was killed or disconnected and disconnect_callback was
+     * not implemented (or it was, but RM_UnblockClient was not called from
+     * within it, as it should).
+     * We must call moduleUnblockClient in order to free privdata and
+     * RedisModuleBlockedClient */
+    if (!bc->unblocked)
+        moduleUnblockClient(c);
+
     bc->client = NULL;
     /* Reset the client for a new query since, for blocking commands implemented
      * into modules, we do not it immediately after the command returns (and
diff --git a/tests/modules/blockonkeys.c b/tests/modules/blockonkeys.c
index 959918b1c..10dc65b1a 100644
--- a/tests/modules/blockonkeys.c
+++ b/tests/modules/blockonkeys.c
@@ -172,13 +172,13 @@ int bpopgt_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int arg
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
     RedisModuleString *keyname = RedisModule_GetBlockedClientReadyKey(ctx);
-    long long gt = (long long)RedisModule_GetBlockedClientPrivateData(ctx);
+    long long *pgt = RedisModule_GetBlockedClientPrivateData(ctx);
 
     fsl_t *fsl;
     if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0))
         return REDISMODULE_ERR;
 
-    if (!fsl || fsl->list[fsl->length-1] <= gt)
+    if (!fsl || fsl->list[fsl->length-1] <= *pgt)
         return REDISMODULE_ERR;
 
     RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]);
@@ -192,10 +192,8 @@ int bpopgt_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int a
 }
 
 void bpopgt_free_privdata(RedisModuleCtx *ctx, void *privdata) {
-    /* Nothing to do because privdata is actually a 'long long',
-     * not a pointer to the heap */
     REDISMODULE_NOT_USED(ctx);
-    REDISMODULE_NOT_USED(privdata);
+    RedisModule_Free(privdata);
 }
 
 /* FSL.BPOPGT <key> <gt> <timeout> - Block clients until list has an element greater than <gt>.
@@ -217,9 +215,12 @@ int fsl_bpopgt(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
         return REDISMODULE_OK;
 
     if (!fsl || fsl->list[fsl->length-1] <= gt) {
+        /* We use malloc so the tests in blockedonkeys.tcl can check for memory leaks */
+        long long *pgt = RedisModule_Alloc(sizeof(long long));
+        *pgt = gt;
         /* Key is empty or has <2 elements, we must block */
         RedisModule_BlockClientOnKeys(ctx, bpopgt_reply_callback, bpopgt_timeout_callback,
-                                      bpopgt_free_privdata, timeout, &argv[1], 1, (void*)gt);
+                                      bpopgt_free_privdata, timeout, &argv[1], 1, pgt);
     } else {
         RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]);
     }
diff --git a/tests/unit/moduleapi/blockonkeys.tcl b/tests/unit/moduleapi/blockonkeys.tcl
index cb99ab1c9..b380227e0 100644
--- a/tests/unit/moduleapi/blockonkeys.tcl
+++ b/tests/unit/moduleapi/blockonkeys.tcl
@@ -45,18 +45,24 @@ start_server {tags {"modules"}} {
     test {Module client blocked on keys (with metadata): Timeout} {
         r del k
         set rd [redis_deferring_client]
+        $rd client id
+        set cid [$rd read]
         r fsl.push k 33
         $rd fsl.bpopgt k 35 1
         assert_equal {Request timedout} [$rd read]
+        r client kill id $cid ;# try to smoke-out client-related memory leak
     }
 
     test {Module client blocked on keys (with metadata): Blocked, case 1} {
         r del k
         set rd [redis_deferring_client]
+        $rd client id
+        set cid [$rd read]
         r fsl.push k 33
         $rd fsl.bpopgt k 33 0
         r fsl.push k 34
         assert_equal {34} [$rd read]
+        r client kill id $cid ;# try to smoke-out client-related memory leak
     }
 
     test {Module client blocked on keys (with metadata): Blocked, case 2} {
@@ -70,6 +76,35 @@ start_server {tags {"modules"}} {
         assert_equal {36} [$rd read]
     }
 
+    test {Module client blocked on keys (with metadata): Blocked, CLIENT KILL} {
+        r del k
+        set rd [redis_deferring_client]
+        $rd client id
+        set cid [$rd read]
+        $rd fsl.bpopgt k 35 0
+        r client kill id $cid ;# try to smoke-out client-related memory leak
+    }
+
+    test {Module client blocked on keys (with metadata): Blocked, CLIENT UNBLOCK TIMEOUT} {
+        r del k
+        set rd [redis_deferring_client]
+        $rd client id
+        set cid [$rd read]
+        $rd fsl.bpopgt k 35 0
+        r client unblock $cid timeout ;# try to smoke-out client-related memory leak
+        assert_equal {Request timedout} [$rd read]
+    }
+
+    test {Module client blocked on keys (with metadata): Blocked, CLIENT UNBLOCK ERROR} {
+        r del k
+        set rd [redis_deferring_client]
+        $rd client id
+        set cid [$rd read]
+        $rd fsl.bpopgt k 35 0
+        r client unblock $cid error ;# try to smoke-out client-related memory leak
+        assert_error "*unblocked*" {$rd read}
+    }
+
     test {Module client blocked on keys does not wake up on wrong type} {
         r del k
         set rd [redis_deferring_client]

From a3b0d8631ff8852967e864e3dd91d6238fe6bda1 Mon Sep 17 00:00:00 2001
From: hayashier <goodheaven700@gmail.com>
Date: Tue, 31 Dec 2019 17:46:48 +0900
Subject: [PATCH 0021/1098] fix typo from fss to rss

---
 src/object.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/object.c b/src/object.c
index 2201a317a..3ae164919 100644
--- a/src/object.c
+++ b/src/object.c
@@ -1119,13 +1119,13 @@ sds getMemoryDoctorReport(void) {
             num_reports++;
         }
 
-        /* Allocator fss is higher than 1.1 and 10MB ? */
+        /* Allocator rss is higher than 1.1 and 10MB ? */
         if (mh->allocator_rss > 1.1 && mh->allocator_rss_bytes > 10<<20) {
             high_alloc_rss = 1;
             num_reports++;
         }
 
-        /* Non-Allocator fss is higher than 1.1 and 10MB ? */
+        /* Non-Allocator rss is higher than 1.1 and 10MB ? */
         if (mh->rss_extra > 1.1 && mh->rss_extra_bytes > 10<<20) {
             high_proc_rss = 1;
             num_reports++;

From 0992ada2fe1cbc8f5f25c0cdec67cb69bb8d3810 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Tue, 31 Dec 2019 18:15:21 +0800
Subject: [PATCH 0022/1098] Fix petential cluster link error.

Funcion adjustOpenFilesLimit() has an implicit parameter, which is server.maxclients.
This function aims to ajust maximum file descriptor number according to server.maxclients
by best effort, which is "bestlimit" could be lower than "maxfiles" but greater than "oldlimit".
When we try to increase "maxclients" using CONFIG SET command, we could increase maximum
file descriptor number to a bigger value without calling aeResizeSetSize the same time.
When later more and more clients connect to server, the allocated fd could be bigger and bigger,
and eventually exceeds events size of aeEventLoop.events. When new nodes joins the cluster,
new link is created, together with new fd, but when calling aeCreateFileEvent, we did not
check the return value. In this case, we have a non-null "link" but the associated fd is not
registered.

So when we dynamically set "maxclients" we could reach an inconsistency between maximum file
descriptor number of the process and server.maxclients. And later could cause cluster link and link
fd inconsistency.

While setting "maxclients" dynamically, we consider it as failed when resulting "maxclients" is not
the same as expected. We try to restore back the maximum file descriptor number when we failed to set
"maxclients" to the specified value, so that server.maxclients could act as a guard as before.
---
 src/config.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/config.c b/src/config.c
index 9aa847183..ed19d336b 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2098,6 +2098,10 @@ static int updateMaxclients(long long val, long long prev, char **err) {
             static char msg[128];
             sprintf(msg, "The operating system is not able to handle the specified number of clients, try with %d", server.maxclients);
             *err = msg;
+            if (server.maxclients > prev) {
+                server.maxclients = prev;
+                adjustOpenFilesLimit();
+            }
             return 0;
         }
         if ((unsigned int) aeGetSetSize(server.el) <

From 3848849013e6ec7efa244897f5fdebb843fcfaa4 Mon Sep 17 00:00:00 2001
From: wangyuan21 <wangyuan21@baidu.com>
Date: Tue, 31 Dec 2019 19:53:00 +0800
Subject: [PATCH 0023/1098] free time event when delete eventloop

---
 src/ae.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/ae.c b/src/ae.c
index 2c1dae512..248096e1f 100644
--- a/src/ae.c
+++ b/src/ae.c
@@ -135,6 +135,13 @@ void aeDeleteEventLoop(aeEventLoop *eventLoop) {
     aeApiFree(eventLoop);
     zfree(eventLoop->events);
     zfree(eventLoop->fired);
+    /* Free time event. */
+    aeTimeEvent *next_te, *te = eventLoop->timeEventHead;
+    while (te) {
+        next_te = te->next;
+        zfree(te);
+        te = next_te;
+    }
     zfree(eventLoop);
 }
 

From 2bc8db9ca5f80e7c69ca51932aed05745d727ec5 Mon Sep 17 00:00:00 2001
From: ShooterIT <wangyuan21@baidu.com>
Date: Tue, 31 Dec 2019 21:35:56 +0800
Subject: [PATCH 0024/1098] Rename rdb asynchronously

---
 src/replication.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/replication.c b/src/replication.c
index 68dc77a61..b7e77184a 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -31,6 +31,7 @@
 
 #include "server.h"
 #include "cluster.h"
+#include "bio.h"
 
 #include <sys/time.h>
 #include <unistd.h>
@@ -1616,14 +1617,20 @@ void readSyncBulkPayload(connection *conn) {
             killRDBChild();
         }
 
+        /* Rename rdb like renaming rewrite aof asynchronously. */
+        int old_rdb_fd = open(server.rdb_filename,O_RDONLY|O_NONBLOCK);
         if (rename(server.repl_transfer_tmpfile,server.rdb_filename) == -1) {
             serverLog(LL_WARNING,
                 "Failed trying to rename the temp DB into %s in "
                 "MASTER <-> REPLICA synchronization: %s",
                 server.rdb_filename, strerror(errno));
             cancelReplicationHandshake();
+            if (old_rdb_fd != -1) close(old_rdb_fd);
             return;
         }
+        /* Close old rdb asynchronously. */
+        if (old_rdb_fd != -1) bioCreateBackgroundJob(BIO_CLOSE_FILE,(void*)(long)old_rdb_fd,NULL,NULL);
+
         if (rdbLoad(server.rdb_filename,&rsi,RDBFLAGS_REPLICATION) != C_OK) {
             serverLog(LL_WARNING,
                 "Failed trying to load the MASTER synchronization "

From e5565a793e139f29970d46e3c681316f87fa16e8 Mon Sep 17 00:00:00 2001
From: John Sully <john@csquare.ca>
Date: Wed, 1 Jan 2020 10:33:02 -0500
Subject: [PATCH 0025/1098] Add support for incremental build with header files

---
 .gitignore   | 1 +
 src/Makefile | 6 +++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 717bf3c7c..8489dbacf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 .*.swp
 *.o
+*.d
 *.log
 dump.rdb
 redis-benchmark
diff --git a/src/Makefile b/src/Makefile
index 86a763e31..00b623a4b 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -283,14 +283,18 @@ $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ)
 dict-benchmark: dict.c zmalloc.c sds.c siphash.c
 	$(REDIS_CC) $(FINAL_CFLAGS) $^ -D DICT_BENCHMARK_MAIN -o $@ $(FINAL_LIBS)
 
+DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d)
+-include $(DEP)
+
 # Because the jemalloc.h header is generated as a part of the jemalloc build,
 # building it should complete before building any other object. Instead of
 # depending on a single artifact, build all dependencies first.
 %.o: %.c .make-prerequisites
-	$(REDIS_CC) -c $<
+	$(REDIS_CC) -MMD -o $@ -c $<
 
 clean:
 	rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html Makefile.dep dict-benchmark
+	rm -f $(DEP)
 
 .PHONY: clean
 

From 0af467d18f9d12b137af3b709c0af579c29d8414 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 1 Jan 2020 18:10:39 +0100
Subject: [PATCH 0026/1098] Fix active expire division by zero.

Likely fix #6723.

This is what happens AFAIK: we enter the main loop where we expire stuff
until a given percentage of keys is still found to be logically expired.
There are however other potential exit conditions.

However the "sampled" variable is not always incremented inside the
loop, because we may found no valid slot as we scan the hash table, but
just NULLs ad dict entries. So when the do/while loop condition is
triggered at the end, we do (expired*100/sampled), dividing by zero if
we sampled 0 keys.
---
 src/expire.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/expire.c b/src/expire.c
index b4ab9ab18..5aff72ee0 100644
--- a/src/expire.c
+++ b/src/expire.c
@@ -203,8 +203,10 @@ void activeExpireCycle(int type) {
          * distribute the time evenly across DBs. */
         current_db++;
 
-        /* Continue to expire if at the end of the cycle more than 25%
-         * of the keys were expired. */
+        /* Continue to expire if at the end of the cycle there are still
+         * a big percentage of keys to expire, compared to the number of keys
+         * we scanned. The percentage, stored in config_cycle_acceptable_stale
+         * is not fixed, but depends on the Redis configured "expire effort". */
         do {
             unsigned long num, slots;
             long long now, ttl_sum;
@@ -305,8 +307,9 @@ void activeExpireCycle(int type) {
             }
             /* We don't repeat the cycle for the current database if there are
              * an acceptable amount of stale keys (logically expired but yet
-             * not reclained). */
-        } while ((expired*100/sampled) > config_cycle_acceptable_stale);
+             * not reclaimed). */
+        } while (sampled == 0 ||
+                 (expired*100/sampled) > config_cycle_acceptable_stale);
     }
 
     elapsed = ustime()-start;

From 408e8e9f44b2f979e13559ba487d907974684846 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Sat, 4 Jan 2020 18:33:24 +0200
Subject: [PATCH 0027/1098] Adjusts 'io_threads_num' max to 128

Instead of 512, use the defined max from networking.c
---
 src/config.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/config.c b/src/config.c
index 9aa847183..54d14d38f 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2192,7 +2192,7 @@ standardConfig configs[] = {
     /* Integer configs */
     createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, 16, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.port, 6379, INTEGER_CONFIG, NULL, NULL), /* TCP port. */
-    createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 512, server.io_threads_num, 1, INTEGER_CONFIG, NULL, NULL), /* Single threaded by default */
+    createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 128, server.io_threads_num, 1, INTEGER_CONFIG, NULL, NULL), /* Single threaded by default */
     createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.aof_rewrite_perc, 100, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_slave_validity_factor, 10, INTEGER_CONFIG, NULL, NULL), /* Slave max data age factor. */
     createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.list_max_ziplist_size, -2, INTEGER_CONFIG, NULL, NULL),

From 2f8134a7ff9d3c435dad16a73bf96e3daa919a21 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Mon, 6 Jan 2020 19:56:50 +0800
Subject: [PATCH 0028/1098] Fix potential memory leak of
 rioWriteBulkStreamID().

---
 src/aof.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/aof.c b/src/aof.c
index 0ef59cfb6..102c39f3f 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1139,7 +1139,10 @@ int rioWriteBulkStreamID(rio *r,streamID *id) {
     int retval;
 
     sds replyid = sdscatfmt(sdsempty(),"%U-%U",id->ms,id->seq);
-    if ((retval = rioWriteBulkString(r,replyid,sdslen(replyid))) == 0) return 0;
+    if ((retval = rioWriteBulkString(r,replyid,sdslen(replyid))) == 0) {
+        sdsfree(replyid);
+        return 0;
+    }
     sdsfree(replyid);
     return retval;
 }

From 9387f7333e0ccc200dda10032e517e2fb93cc368 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Tue, 7 Jan 2020 10:28:36 +0800
Subject: [PATCH 0029/1098] Fix potential memory leak of clusterLoadConfig().

---
 src/cluster.c | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index f603361cd..f9d8ae151 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -157,7 +157,10 @@ int clusterLoadConfig(char *filename) {
         }
 
         /* Regular config lines have at least eight fields */
-        if (argc < 8) goto fmterr;
+        if (argc < 8) {
+            sdsfreesplitres(argv,argc);
+            goto fmterr;
+        }
 
         /* Create this node if it does not exist */
         n = clusterLookupNode(argv[0]);
@@ -166,7 +169,10 @@ int clusterLoadConfig(char *filename) {
             clusterAddNode(n);
         }
         /* Address and port */
-        if ((p = strrchr(argv[1],':')) == NULL) goto fmterr;
+        if ((p = strrchr(argv[1],':')) == NULL) {
+            sdsfreesplitres(argv,argc);
+            goto fmterr;
+        }
         *p = '\0';
         memcpy(n->ip,argv[1],strlen(argv[1])+1);
         char *port = p+1;
@@ -247,7 +253,10 @@ int clusterLoadConfig(char *filename) {
                 *p = '\0';
                 direction = p[1]; /* Either '>' or '<' */
                 slot = atoi(argv[j]+1);
-                if (slot < 0 || slot >= CLUSTER_SLOTS) goto fmterr;
+                if (slot < 0 || slot >= CLUSTER_SLOTS) {
+                    sdsfreesplitres(argv,argc);
+                    goto fmterr;
+                }
                 p += 3;
                 cn = clusterLookupNode(p);
                 if (!cn) {
@@ -267,8 +276,14 @@ int clusterLoadConfig(char *filename) {
             } else {
                 start = stop = atoi(argv[j]);
             }
-            if (start < 0 || start >= CLUSTER_SLOTS) goto fmterr;
-            if (stop < 0 || stop >= CLUSTER_SLOTS) goto fmterr;
+            if (start < 0 || start >= CLUSTER_SLOTS) {
+                sdsfreesplitres(argv,argc);
+                goto fmterr;
+            }
+            if (stop < 0 || stop >= CLUSTER_SLOTS) {
+                sdsfreesplitres(argv,argc);
+                goto fmterr;
+            }
             while(start <= stop) clusterAddSlot(n, start++);
         }
 

From 18c26760846fa51aee4ad5d6c4a6d06c64d7269e Mon Sep 17 00:00:00 2001
From: yz1509 <pro-765@qq.com>
Date: Tue, 7 Jan 2020 10:29:54 +0800
Subject: [PATCH 0030/1098] avoid sentinel changes promoted_slave to be its own
 replica.

---
 src/sentinel.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sentinel.c b/src/sentinel.c
index 10117252d..10c003d03 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -4308,7 +4308,7 @@ void sentinelFailoverDetectEnd(sentinelRedisInstance *master) {
             sentinelRedisInstance *slave = dictGetVal(de);
             int retval;
 
-            if (slave->flags & (SRI_RECONF_DONE|SRI_RECONF_SENT)) continue;
+            if (slave->flags & (SRI_PROMOTED|SRI_RECONF_DONE|SRI_RECONF_SENT)) continue;
             if (slave->link->disconnected) continue;
 
             retval = sentinelSendSlaveOf(slave,

From d6a13f444003e75e8ccbe97297bf32f0eb58cb23 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Tue, 7 Jan 2020 11:17:52 +0800
Subject: [PATCH 0031/1098] Free allocated sds in pfdebugCommand() to avoid
 memory leak.

---
 src/hyperloglog.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/hyperloglog.c b/src/hyperloglog.c
index a44d15646..facd99743 100644
--- a/src/hyperloglog.c
+++ b/src/hyperloglog.c
@@ -1535,6 +1535,7 @@ void pfdebugCommand(client *c) {
         sds decoded = sdsempty();
 
         if (hdr->encoding != HLL_SPARSE) {
+            sdsfree(decoded);
             addReplyError(c,"HLL encoding is not sparse");
             return;
         }

From 63e46e5f8db68f3b8b5e0ea6b3b852b1068c9894 Mon Sep 17 00:00:00 2001
From: Leo Murillo <leonardo.murillo@gmail.com>
Date: Tue, 7 Jan 2020 13:55:26 -0600
Subject: [PATCH 0032/1098] Fix bug on KEYS command where pattern starts with *
 followed by \x00 (null char).

---
 src/db.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/db.c b/src/db.c
index 32bb35652..ba7be2725 100644
--- a/src/db.c
+++ b/src/db.c
@@ -602,7 +602,7 @@ void keysCommand(client *c) {
     void *replylen = addReplyDeferredLen(c);
 
     di = dictGetSafeIterator(c->db->dict);
-    allkeys = (pattern[0] == '*' && pattern[1] == '\0');
+    allkeys = (pattern[0] == '*' && plen == 1);
     while((de = dictNext(di)) != NULL) {
         sds key = dictGetKey(de);
         robj *keyobj;

From fda2bf9c92fe1c4adbb5c968b9b234710a85a7ed Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Tue, 7 Jan 2020 21:09:44 -0500
Subject: [PATCH 0033/1098] typo fix in acl.c

---
 src/acl.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index db742c649..1f395bd3f 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -183,12 +183,12 @@ int ACLListMatchSds(void *a, void *b) {
     return sdscmp(a,b) == 0;
 }
 
-/* Method to free list elements from ACL users password/ptterns lists. */
+/* Method to free list elements from ACL users password/patterns lists. */
 void ACLListFreeSds(void *item) {
     sdsfree(item);
 }
 
-/* Method to duplicate list elements from ACL users password/ptterns lists. */
+/* Method to duplicate list elements from ACL users password/patterns lists. */
 void *ACLListDupSds(void *item) {
     return sdsdup(item);
 }

From 7080a28efc28bd3a3004bd37c9b85d58233fc1ed Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 8 Jan 2020 10:10:11 +0100
Subject: [PATCH 0034/1098] XCLAIM: Create the consumer only on successful
 claims.

Fixes #6744.
---
 src/t_stream.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 3d46ca0da..ae57202c1 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -2246,7 +2246,7 @@ void xclaimCommand(client *c) {
     }
 
     /* Do the actual claiming. */
-    streamConsumer *consumer = streamLookupConsumer(group,c->argv[3]->ptr,1);
+    streamConsumer *consumer = NULL;
     void *arraylenptr = addReplyDeferredLen(c);
     size_t arraylen = 0;
     for (int j = 5; j <= last_id_arg; j++) {
@@ -2298,9 +2298,11 @@ void xclaimCommand(client *c) {
             if (nack->consumer)
                 raxRemove(nack->consumer->pel,buf,sizeof(buf),NULL);
             /* Update the consumer and idle time. */
+            if (consumer == NULL)
+                consumer = streamLookupConsumer(group,c->argv[3]->ptr,1);
             nack->consumer = consumer;
             nack->delivery_time = deliverytime;
-            /* Set the delivery attempts counter if given, otherwise 
+            /* Set the delivery attempts counter if given, otherwise
              * autoincrement unless JUSTID option provided */
             if (retrycount >= 0) {
                 nack->delivery_count = retrycount;

From 5234bff5790581c7d52e668e65b87a5beac1863e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 10 Jan 2020 12:22:16 +0100
Subject: [PATCH 0035/1098] Git ignore: ignore more files.

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index 8489dbacf..de626d61b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,7 @@
 .*.swp
 *.o
+*.xo
+*.so
 *.d
 *.log
 dump.rdb

From 9466dae4e5377056cd27fcdb0f4b47a5497bb08d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 10 Jan 2020 13:02:45 +0100
Subject: [PATCH 0036/1098] Free fakeclient argv on AOF error.

We exit later, so no bug fixed, but it is more correct.

See #6054, thanks to @ShooterIT for finding the issue.
---
 src/aof.c | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index 0ef59cfb6..9c2fa838b 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -781,18 +781,24 @@ int loadAppendOnlyFile(char *filename) {
         argc = atoi(buf+1);
         if (argc < 1) goto fmterr;
 
+        /* Load the next command in the AOF as our fake client
+         * argv. */
         argv = zmalloc(sizeof(robj*)*argc);
         fakeClient->argc = argc;
         fakeClient->argv = argv;
 
         for (j = 0; j < argc; j++) {
-            if (fgets(buf,sizeof(buf),fp) == NULL) {
+            /* Parse the argument len. */
+            if (fgets(buf,sizeof(buf),fp) == NULL ||
+                buf[0] != '$')
+            {
                 fakeClient->argc = j; /* Free up to j-1. */
                 freeFakeClientArgv(fakeClient);
                 goto readerr;
             }
-            if (buf[0] != '$') goto fmterr;
             len = strtol(buf+1,NULL,10);
+
+            /* Read it into a string object. */
             argsds = sdsnewlen(SDS_NOINIT,len);
             if (len && fread(argsds,len,1,fp) == 0) {
                 sdsfree(argsds);
@@ -801,10 +807,12 @@ int loadAppendOnlyFile(char *filename) {
                 goto readerr;
             }
             argv[j] = createObject(OBJ_STRING,argsds);
+
+            /* Discard CRLF. */
             if (fread(buf,2,1,fp) == 0) {
                 fakeClient->argc = j+1; /* Free up to j. */
                 freeFakeClientArgv(fakeClient);
-                goto readerr; /* discard CRLF */
+                goto readerr;
             }
         }
 

From b5f52bf01cd99d0bb2d51d158fd009690a36d042 Mon Sep 17 00:00:00 2001
From: Vasyl Melnychuk <vasyl.melnychuk@toptal.com>
Date: Fri, 10 Jan 2020 23:34:15 +0200
Subject: [PATCH 0037/1098] Make error when submitting command in incorrect
 context more explicit

So error message `ERR only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT allowed in this context` will become
`ERR 'get' command submitted, but only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT allowed in this context`
---
 src/server.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index 5845a5485..3f64aaec9 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3498,7 +3498,10 @@ int processCommand(client *c) {
         c->cmd->proc != unsubscribeCommand &&
         c->cmd->proc != psubscribeCommand &&
         c->cmd->proc != punsubscribeCommand) {
-        addReplyError(c,"only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT allowed in this context");
+        addReplyErrorFormat(c,
+            "'%s' command submitted, but only (P)SUBSCRIBE / "
+            "(P)UNSUBSCRIBE / PING / QUIT allowed in this context",
+            c->cmd->name);
         return C_OK;
     }
 

From a8c912ead65e85b5ce3dae936320e77e047df9c9 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 13 Jan 2020 12:50:26 +0100
Subject: [PATCH 0038/1098] A few comments about main thread serving I/O as
 well.

Related to #6110.
---
 src/networking.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index f1a6b9910..96ab8e592 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2656,6 +2656,10 @@ pthread_mutex_t io_threads_mutex[IO_THREADS_MAX_NUM];
 _Atomic unsigned long io_threads_pending[IO_THREADS_MAX_NUM];
 int io_threads_active;  /* Are the threads currently spinning waiting I/O? */
 int io_threads_op;      /* IO_THREADS_OP_WRITE or IO_THREADS_OP_READ. */
+
+/* This is the list of clients each thread will serve when threaded I/O is
+ * used. We spawn N threads, and the N+1 list is used for the clients that
+ * are processed by the main thread itself (this is why ther is "+1"). */
 list *io_threads_list[IO_THREADS_MAX_NUM+1];
 
 void *IOThreadMain(void *myid) {
@@ -2729,7 +2733,7 @@ void initThreadedIO(void) {
         }
         io_threads[i] = tid;
     }
-    io_threads_list[server.io_threads_num] = listCreate();
+    io_threads_list[server.io_threads_num] = listCreate(); /* For main thread */
 }
 
 void startThreadedIO(void) {
@@ -2814,6 +2818,7 @@ int handleClientsWithPendingWritesUsingThreads(void) {
         io_threads_pending[j] = count;
     }
 
+    /* Also use the main thread to process a slide of clients. */
     listRewind(io_threads_list[server.io_threads_num],&li);
     while((ln = listNext(&li))) {
         client *c = listNodeValue(ln);
@@ -2898,6 +2903,7 @@ int handleClientsWithPendingReadsUsingThreads(void) {
         io_threads_pending[j] = count;
     }
 
+    /* Also use the main thread to process a slide of clients. */
     listRewind(io_threads_list[server.io_threads_num],&li);
     while((ln = listNext(&li))) {
         client *c = listNodeValue(ln);

From d28948b14303fdf00b331090ef177cc6cca54fe1 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 13 Jan 2020 12:54:39 +0100
Subject: [PATCH 0039/1098] Port PR #6110 to new connection object code.

---
 src/networking.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 96ab8e592..73dc4afca 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2822,7 +2822,7 @@ int handleClientsWithPendingWritesUsingThreads(void) {
     listRewind(io_threads_list[server.io_threads_num],&li);
     while((ln = listNext(&li))) {
         client *c = listNodeValue(ln);
-        writeToClient(c->fd,c,0);
+        writeToClient(c,0);
     }
     listEmpty(io_threads_list[server.io_threads_num]);
 
@@ -2907,7 +2907,7 @@ int handleClientsWithPendingReadsUsingThreads(void) {
     listRewind(io_threads_list[server.io_threads_num],&li);
     while((ln = listNext(&li))) {
         client *c = listNodeValue(ln);
-        readQueryFromClient(NULL,c->fd,c,0);
+        readQueryFromClient(c->conn);
     }
     listEmpty(io_threads_list[server.io_threads_num]);
 

From baa88a1c144954469e3762023fb41617242dc30e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 13 Jan 2020 13:16:13 +0100
Subject: [PATCH 0040/1098] Jump to right label on AOF parsing error.

Related to #6054.
---
 src/aof.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index 9c2fa838b..63b34b43f 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -789,12 +789,14 @@ int loadAppendOnlyFile(char *filename) {
 
         for (j = 0; j < argc; j++) {
             /* Parse the argument len. */
-            if (fgets(buf,sizeof(buf),fp) == NULL ||
-                buf[0] != '$')
-            {
+            char *readres = fgets(buf,sizeof(buf),fp);
+            if (readres == NULL || buf[0] != '$') {
                 fakeClient->argc = j; /* Free up to j-1. */
                 freeFakeClientArgv(fakeClient);
-                goto readerr;
+                if (readres == NULL)
+                    goto readerr;
+                else
+                    goto fmterr;
             }
             len = strtol(buf+1,NULL,10);
 

From 28d146bed69cd1d288ff435fe86ca9ad2c9cb879 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 13 Jan 2020 13:25:37 +0100
Subject: [PATCH 0041/1098] Simplify #6379 changes.

---
 src/aof.c     | 5 +----
 src/cluster.c | 8 +++-----
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index efdd68efa..9eeb3f1e2 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1149,10 +1149,7 @@ int rioWriteBulkStreamID(rio *r,streamID *id) {
     int retval;
 
     sds replyid = sdscatfmt(sdsempty(),"%U-%U",id->ms,id->seq);
-    if ((retval = rioWriteBulkString(r,replyid,sdslen(replyid))) == 0) {
-        sdsfree(replyid);
-        return 0;
-    }
+    retval = rioWriteBulkString(r,replyid,sdslen(replyid));
     sdsfree(replyid);
     return retval;
 }
diff --git a/src/cluster.c b/src/cluster.c
index f9d8ae151..c05e46f76 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -276,11 +276,9 @@ int clusterLoadConfig(char *filename) {
             } else {
                 start = stop = atoi(argv[j]);
             }
-            if (start < 0 || start >= CLUSTER_SLOTS) {
-                sdsfreesplitres(argv,argc);
-                goto fmterr;
-            }
-            if (stop < 0 || stop >= CLUSTER_SLOTS) {
+            if (start < 0 || start >= CLUSTER_SLOTS ||
+                stop < 0 || stop >= CLUSTER_SLOTS)
+            {
                 sdsfreesplitres(argv,argc);
                 goto fmterr;
             }

From 7d5fb5df3f2dff8102ead8542e7697a5bc312e94 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 13 Jan 2020 18:53:12 +0100
Subject: [PATCH 0042/1098] Setting N I/O threads should mean N-1 additional +
 1 main thread.

---
 src/networking.c | 47 +++++++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 22 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 73dc4afca..a2e454d4b 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2658,9 +2658,9 @@ int io_threads_active;  /* Are the threads currently spinning waiting I/O? */
 int io_threads_op;      /* IO_THREADS_OP_WRITE or IO_THREADS_OP_READ. */
 
 /* This is the list of clients each thread will serve when threaded I/O is
- * used. We spawn N threads, and the N+1 list is used for the clients that
- * are processed by the main thread itself (this is why ther is "+1"). */
-list *io_threads_list[IO_THREADS_MAX_NUM+1];
+ * used. We spawn io_threads_num-1 threads, since one is the main thread
+ * itself. */
+list *io_threads_list[IO_THREADS_MAX_NUM];
 
 void *IOThreadMain(void *myid) {
     /* The ID is the thread number (from 0 to server.iothreads_num-1), and is
@@ -2720,12 +2720,16 @@ void initThreadedIO(void) {
         exit(1);
     }
 
-    /* Spawn the I/O threads. */
+    /* Spawn and initialize the I/O threads. */
     for (int i = 0; i < server.io_threads_num; i++) {
+        /* Things we do for all the threads including the main thread. */
+        io_threads_list[i] = listCreate();
+        if (i == 0) continue; /* Thread 0 is the main thread. */
+
+        /* Things we do only for the additional threads. */
         pthread_t tid;
         pthread_mutex_init(&io_threads_mutex[i],NULL);
         io_threads_pending[i] = 0;
-        io_threads_list[i] = listCreate();
         pthread_mutex_lock(&io_threads_mutex[i]); /* Thread will be stopped. */
         if (pthread_create(&tid,NULL,IOThreadMain,(void*)(long)i) != 0) {
             serverLog(LL_WARNING,"Fatal: Can't initialize IO thread.");
@@ -2733,14 +2737,13 @@ void initThreadedIO(void) {
         }
         io_threads[i] = tid;
     }
-    io_threads_list[server.io_threads_num] = listCreate(); /* For main thread */
 }
 
 void startThreadedIO(void) {
     if (tio_debug) { printf("S"); fflush(stdout); }
     if (tio_debug) printf("--- STARTING THREADED IO ---\n");
     serverAssert(io_threads_active == 0);
-    for (int j = 0; j < server.io_threads_num; j++)
+    for (int j = 1; j < server.io_threads_num; j++)
         pthread_mutex_unlock(&io_threads_mutex[j]);
     io_threads_active = 1;
 }
@@ -2754,7 +2757,7 @@ void stopThreadedIO(void) {
         (int) listLength(server.clients_pending_read),
         (int) listLength(server.clients_pending_write));
     serverAssert(io_threads_active == 1);
-    for (int j = 0; j < server.io_threads_num; j++)
+    for (int j = 1; j < server.io_threads_num; j++)
         pthread_mutex_lock(&io_threads_mutex[j]);
     io_threads_active = 0;
 }
@@ -2805,7 +2808,7 @@ int handleClientsWithPendingWritesUsingThreads(void) {
     while((ln = listNext(&li))) {
         client *c = listNodeValue(ln);
         c->flags &= ~CLIENT_PENDING_WRITE;
-        int target_id = item_id % (server.io_threads_num+1);
+        int target_id = item_id % server.io_threads_num;
         listAddNodeTail(io_threads_list[target_id],c);
         item_id++;
     }
@@ -2813,23 +2816,23 @@ int handleClientsWithPendingWritesUsingThreads(void) {
     /* Give the start condition to the waiting threads, by setting the
      * start condition atomic var. */
     io_threads_op = IO_THREADS_OP_WRITE;
-    for (int j = 0; j < server.io_threads_num; j++) {
+    for (int j = 1; j < server.io_threads_num; j++) {
         int count = listLength(io_threads_list[j]);
         io_threads_pending[j] = count;
     }
 
-    /* Also use the main thread to process a slide of clients. */
-    listRewind(io_threads_list[server.io_threads_num],&li);
+    /* Also use the main thread to process a slice of clients. */
+    listRewind(io_threads_list[0],&li);
     while((ln = listNext(&li))) {
         client *c = listNodeValue(ln);
         writeToClient(c,0);
     }
-    listEmpty(io_threads_list[server.io_threads_num]);
+    listEmpty(io_threads_list[0]);
 
-    /* Wait for all threads to end their work. */
+    /* Wait for all the other threads to end their work. */
     while(1) {
         unsigned long pending = 0;
-        for (int j = 0; j < server.io_threads_num; j++)
+        for (int j = 1; j < server.io_threads_num; j++)
             pending += io_threads_pending[j];
         if (pending == 0) break;
     }
@@ -2890,7 +2893,7 @@ int handleClientsWithPendingReadsUsingThreads(void) {
     int item_id = 0;
     while((ln = listNext(&li))) {
         client *c = listNodeValue(ln);
-        int target_id = item_id % (server.io_threads_num+1);
+        int target_id = item_id % server.io_threads_num;
         listAddNodeTail(io_threads_list[target_id],c);
         item_id++;
     }
@@ -2898,23 +2901,23 @@ int handleClientsWithPendingReadsUsingThreads(void) {
     /* Give the start condition to the waiting threads, by setting the
      * start condition atomic var. */
     io_threads_op = IO_THREADS_OP_READ;
-    for (int j = 0; j < server.io_threads_num; j++) {
+    for (int j = 1; j < server.io_threads_num; j++) {
         int count = listLength(io_threads_list[j]);
         io_threads_pending[j] = count;
     }
 
-    /* Also use the main thread to process a slide of clients. */
-    listRewind(io_threads_list[server.io_threads_num],&li);
+    /* Also use the main thread to process a slice of clients. */
+    listRewind(io_threads_list[0],&li);
     while((ln = listNext(&li))) {
         client *c = listNodeValue(ln);
         readQueryFromClient(c->conn);
     }
-    listEmpty(io_threads_list[server.io_threads_num]);
+    listEmpty(io_threads_list[0]);
 
-    /* Wait for all threads to end their work. */
+    /* Wait for all the other threads to end their work. */
     while(1) {
         unsigned long pending = 0;
-        for (int j = 0; j < server.io_threads_num; j++)
+        for (int j = 1; j < server.io_threads_num; j++)
             pending += io_threads_pending[j];
         if (pending == 0) break;
     }

From 2d7dc71b6fcf1638338f2d24d2d9c94297157416 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 13 Jan 2020 19:10:42 +0100
Subject: [PATCH 0043/1098] Document I/O threads in redis.conf.

---
 redis.conf | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/redis.conf b/redis.conf
index 5f9547c1d..07005cffe 100644
--- a/redis.conf
+++ b/redis.conf
@@ -938,6 +938,52 @@ lazyfree-lazy-expire no
 lazyfree-lazy-server-del no
 replica-lazy-flush no
 
+################################ THREADED I/O #################################
+
+# Redis is mostly single threaded, however there are certain threaded
+# operations such as UNLINK, slow I/O accesses and other things that are
+# performed on side threads.
+#
+# Now it is also possible to handle Redis clients socket reads and writes
+# in different I/O threads. Since especially writing is so slow, normally
+# Redis users use pipelining in order to speedup the Redis performances per
+# core, and spawn multiple instances in order to scale more. Using I/O
+# threads it is possible to easily speedup two times Redis without resorting
+# to pipelining nor sharding of the instance.
+#
+# By default threading is disabled, we suggest enabling it only in machines
+# that have at least 4 or more cores, leaving at least one spare core.
+# Using more than 8 threads is unlikely to help much. We also recommend using
+# threaded I/O only if you actually have performance problems, with Redis
+# instances being able to use a quite big percentage of CPU time, otherwise
+# there is no point in using this feature.
+#
+# So for instance if you have a four cores boxes, try to use 2 or 3 I/O
+# threads, if you have a 8 cores, try to use 6 threads. In order to
+# enable I/O threads use the following configuration directive:
+#
+# io-threads 4
+#
+# Setting io-threads to 1 will just use the main thread as usually.
+# When I/O threads are enabled, we only use threads for writes, that is
+# to thread the write(2) syscall and transfer the client buffers to the
+# socket. However it is also possible to enable threading of reads and
+# protocol parsing using the following configuration directive, by setting
+# it to yes:
+#
+# io-threads-do-reads no
+#
+# Usually threading reads doesn't help much.
+#
+# NOTE 1: This configuration directive cannot be changed at runtime via
+# CONFIG SET. Aso this feature currently does not work when SSL is
+# enabled.
+#
+# NOTE 2: If you want to test the Redis speedup using redis-benchmark, make
+# sure you also run the benchmark itself in threaded mode, using the
+# --threads option to match the number of Redis theads, otherwise you'll not
+# be able to notice the improvements.
+
 ############################## APPEND ONLY MODE ###############################
 
 # By default Redis asynchronously dumps the dataset on disk. This mode is

From abf8150b2e91927f73dccd670e49b655615b8b5a Mon Sep 17 00:00:00 2001
From: Ponnuvel Palaniyappan <pponnuvel@gmail.com>
Date: Tue, 14 Jan 2020 07:38:57 +0000
Subject: [PATCH 0044/1098] Fix typos in README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 3442659e6..9d7c3fbcb 100644
--- a/README.md
+++ b/README.md
@@ -318,7 +318,7 @@ There are two special functions called periodically by the event loop:
 Inside server.c you can find code that handles other vital things of the Redis server:
 
 * `call()` is used in order to call a given command in the context of a given client.
-* `activeExpireCycle()` handles eviciton of keys with a time to live set via the `EXPIRE` command.
+* `activeExpireCycle()` handles eviction of keys with a time to live set via the `EXPIRE` command.
 * `freeMemoryIfNeeded()` is called when a new write command should be performed but Redis is out of memory according to the `maxmemory` directive.
 * The global variable `redisCommandTable` defines all the Redis commands, specifying the name of the command, the function implementing the command, the number of arguments required, and other properties of each command.
 
@@ -411,7 +411,7 @@ Other C files
 * `sds.c` is the Redis string library, check http://github.com/antirez/sds for more information.
 * `anet.c` is a library to use POSIX networking in a simpler way compared to the raw interface exposed by the kernel.
 * `dict.c` is an implementation of a non-blocking hash table which rehashes incrementally.
-* `scripting.c` implements Lua scripting. It is completely self contained from the rest of the Redis implementation and is simple enough to understand if you are familar with the Lua API.
+* `scripting.c` implements Lua scripting. It is completely self contained from the rest of the Redis implementation and is simple enough to understand if you are familiar with the Lua API.
 * `cluster.c` implements the Redis Cluster. Probably a good read only after being very familiar with the rest of the Redis code base. If you want to read `cluster.c` make sure to read the [Redis Cluster specification][3].
 
 [3]: http://redis.io/topics/cluster-spec

From 1d4ea00d12885108d936b76cd31097dc4894f5ca Mon Sep 17 00:00:00 2001
From: Ponnuvel Palaniyappan <pponnuvel@gmail.com>
Date: Tue, 14 Jan 2020 08:10:39 +0000
Subject: [PATCH 0045/1098] Fix a potential overflow with strncpy

---
 src/config.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/config.c b/src/config.c
index 0526de84d..e4b1bf869 100644
--- a/src/config.c
+++ b/src/config.c
@@ -108,12 +108,12 @@ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = {
 /* Generic config infrastructure function pointers
  * int is_valid_fn(val, err)
  *     Return 1 when val is valid, and 0 when invalid.
- *     Optionslly set err to a static error string.
+ *     Optionally set err to a static error string.
  * int update_fn(val, prev, err)
  *     This function is called only for CONFIG SET command (not at config file parsing)
  *     It is called after the actual config is applied,
  *     Return 1 for success, and 0 for failure.
- *     Optionslly set err to a static error string.
+ *     Optionally set err to a static error string.
  *     On failure the config change will be reverted.
  */
 
@@ -720,7 +720,7 @@ void configSetCommand(client *c) {
      * config_set_memory_field(name,var) */
     } config_set_memory_field(
       "client-query-buffer-limit",server.client_max_querybuf_len) {
-    /* Everyhing else is an error... */
+    /* Everything else is an error... */
     } config_set_else {
         addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s",
             (char*)c->argv[2]->ptr);
@@ -1699,9 +1699,9 @@ static int configEnumLoad(typeData data, sds *argv, int argc, char **err) {
 
         enumerr[sdslen(enumerr) - 2] = '\0';
 
-        /* Make sure we don't overrun the fixed buffer */
-        enumerr[LOADBUF_SIZE - 1] = '\0';
         strncpy(loadbuf, enumerr, LOADBUF_SIZE);
+        /* strncpy does not if null terminate if source string length is >= destination buffer. */
+        loadbuf[LOADBUF_SIZE - 1] = '\0';
 
         sdsfree(enumerr);
         *err = loadbuf;

From 7ef2270ee73e75b00b0c1bde40fb1df55c7ec6ff Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 15 Jan 2020 17:55:24 +0100
Subject: [PATCH 0046/1098] Change error message for #6775.

---
 src/server.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/server.c b/src/server.c
index 3f64aaec9..2b226f568 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3499,8 +3499,8 @@ int processCommand(client *c) {
         c->cmd->proc != psubscribeCommand &&
         c->cmd->proc != punsubscribeCommand) {
         addReplyErrorFormat(c,
-            "'%s' command submitted, but only (P)SUBSCRIBE / "
-            "(P)UNSUBSCRIBE / PING / QUIT allowed in this context",
+            "Can't execute '%s': only (P)SUBSCRIBE / "
+            "(P)UNSUBSCRIBE / PING / QUIT are allowed in this context",
             c->cmd->name);
         return C_OK;
     }

From 3f6d00d387b85b0eaa459427c2891ca979dc8363 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Thu, 16 Jan 2020 17:33:23 -0500
Subject: [PATCH 0047/1098] fix potentical memory leaks

---
 src/debug.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/debug.c b/src/debug.c
index a2d37337d..b441fa8f8 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -683,9 +683,12 @@ NULL
         sds stats = sdsempty();
         char buf[4096];
 
-        if (getLongFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK)
+        if (getLongFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK){
+            sdsfree(stats);
             return;
+        }
         if (dbid < 0 || dbid >= server.dbnum) {
+            sdsfree(stats);
             addReplyError(c,"Out of range database");
             return;
         }

From 67ee87522ab5b3c3bae8ee4c84808664b14de2cc Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Thu, 16 Jan 2020 17:35:26 -0500
Subject: [PATCH 0048/1098] format fix

---
 src/debug.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/debug.c b/src/debug.c
index b441fa8f8..5ffbab7b4 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -683,7 +683,7 @@ NULL
         sds stats = sdsempty();
         char buf[4096];
 
-        if (getLongFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK){
+        if (getLongFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK) {
             sdsfree(stats);
             return;
         }

From dd05b7f67f8191aacbbe73a1b415658233715926 Mon Sep 17 00:00:00 2001
From: srzhao <srzhao@sysnew.com>
Date: Fri, 17 Jan 2020 11:46:19 +0800
Subject: [PATCH 0049/1098] fix impl of aof-child whitelist SIGUSR1 feature.

---
 src/aof.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index 9eeb3f1e2..ce1a440f8 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1797,14 +1797,15 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
         serverLog(LL_VERBOSE,
             "Background AOF rewrite signal handler took %lldus", ustime()-now);
     } else if (!bysignal && exitcode != 0) {
+        server.aof_lastbgrewrite_status = C_ERR;
+
+        serverLog(LL_WARNING,
+            "Background AOF rewrite terminated with error");
+    } else {
         /* SIGUSR1 is whitelisted, so we have a way to kill a child without
          * tirggering an error condition. */
         if (bysignal != SIGUSR1)
             server.aof_lastbgrewrite_status = C_ERR;
-        serverLog(LL_WARNING,
-            "Background AOF rewrite terminated with error");
-    } else {
-        server.aof_lastbgrewrite_status = C_ERR;
 
         serverLog(LL_WARNING,
             "Background AOF rewrite terminated by signal %d", bysignal);

From 38f6207f884f514e928513acb6560fdb375daa2e Mon Sep 17 00:00:00 2001
From: srzhao <srzhao@sysnew.com>
Date: Mon, 20 Jan 2020 21:17:02 +0800
Subject: [PATCH 0050/1098] Check OOM at script start to get stable lua OOM
 state.

Checking OOM by `getMaxMemoryState` inside script might get different result
with `freeMemoryIfNeededAndSafe` at script start, because lua stack and
arguments also consume memory.

This leads to memory `borderline` when memory grows near server.maxmemory:

- `freeMemoryIfNeededAndSafe` at script start detects no OOM, no memory freed
- `getMaxMemoryState` inside script detects OOM, script aborted

We solve this 'borderline' issue by saving OOM state at script start to get
stable lua OOM state.

related to issue #6565 and #5250.
---
 src/scripting.c | 7 +++----
 src/server.c    | 7 +++++++
 src/server.h    | 1 +
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/scripting.c b/src/scripting.c
index 9282b7fd9..5ed24ffad 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -655,12 +655,11 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) {
         !server.loading &&              /* Don't care about mem if loading. */
         !server.masterhost &&           /* Slave must execute the script. */
         server.lua_write_dirty == 0 &&  /* Script had no side effects so far. */
+        server.lua_oom &&               /* Detected OOM when script start. */
         (cmd->flags & CMD_DENYOOM))
     {
-        if (getMaxmemoryState(NULL,NULL,NULL,NULL) != C_OK) {
-            luaPushError(lua, shared.oomerr->ptr);
-            goto cleanup;
-        }
+        luaPushError(lua, shared.oomerr->ptr);
+        goto cleanup;
     }
 
     if (cmd->flags & CMD_RANDOM) server.lua_random_dirty = 1;
diff --git a/src/server.c b/src/server.c
index 2b226f568..21d6aab27 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3442,6 +3442,13 @@ int processCommand(client *c) {
             addReply(c, shared.oomerr);
             return C_OK;
         }
+
+        /* Save out_of_memory result at script start, otherwise if we check OOM
+         * untill first write within script, memory used by lua stack and
+         * arguments might interfere. */
+        if (c->cmd->proc == evalCommand || c->cmd->proc == evalShaCommand) {
+            server.lua_oom = out_of_memory;
+        }
     }
 
     /* Make sure to use a reasonable amount of memory for client side
diff --git a/src/server.h b/src/server.h
index 8e354c03d..b440beba3 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1376,6 +1376,7 @@ struct redisServer {
                              execution. */
     int lua_kill;         /* Kill the script if true. */
     int lua_always_replicate_commands; /* Default replication type. */
+    int lua_oom;          /* OOM detected when script start? */
     /* Lazy free */
     int lazyfree_lazy_eviction;
     int lazyfree_lazy_expire;

From bd60c11bd8ca652f6cb6ecc57d0d10bc8aaa3481 Mon Sep 17 00:00:00 2001
From: qetu3790 <qetu3790@163.com>
Date: Thu, 23 Jan 2020 17:18:07 +0800
Subject: [PATCH 0051/1098] Fix not used constant in lru_test_mode.

LRU_CYCLE_PERIOD is defined,but not used.
---
 src/redis-cli.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 065c389c6..36ceb16e8 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -7735,7 +7735,7 @@ static void LRUTestMode(void) {
          * to fill the target instance easily. */
         start_cycle = mstime();
         long long hits = 0, misses = 0;
-        while(mstime() - start_cycle < 1000) {
+        while(mstime() - start_cycle < LRU_CYCLE_PERIOD) {
             /* Write cycle. */
             for (j = 0; j < LRU_CYCLE_PIPELINE_SIZE; j++) {
                 char val[6];

From 3e9e27e98fa7ecdbb5a34676e51cda54de671d8a Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 27 Jan 2020 18:37:52 +0100
Subject: [PATCH 0052/1098] ACL LOG: data structures and initial functions.

---
 src/acl.c       | 51 ++++++++++++++++++++++++++++++++++++++++++++++++-
 src/multi.c     |  2 +-
 src/scripting.c |  2 +-
 src/server.c    |  2 +-
 src/server.h    |  2 +-
 5 files changed, 54 insertions(+), 5 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 1f395bd3f..4391382a6 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -49,6 +49,8 @@ list *UsersToLoad;  /* This is a list of users found in the configuration file
                        array of SDS pointers: the first is the user name,
                        all the remaining pointers are ACL rules in the same
                        format as ACLSetUser(). */
+list *ACLLog;       /* Our security log, the user is able to inspect that
+                       using the ACL LOG command .*/
 
 struct ACLCategoryItem {
     const char *name;
@@ -920,6 +922,7 @@ void ACLInitDefaultUser(void) {
 void ACLInit(void) {
     Users = raxNew();
     UsersToLoad = listCreate();
+    ACLLog = listCreate();
     ACLInitDefaultUser();
 }
 
@@ -1034,7 +1037,7 @@ user *ACLGetUserByName(const char *name, size_t namelen) {
  * command cannot be executed because the user is not allowed to run such
  * command, the second if the command is denied because the user is trying
  * to access keys that are not among the specified patterns. */
-int ACLCheckCommandPerm(client *c) {
+int ACLCheckCommandPerm(client *c, int *keyidxptr) {
     user *u = c->user;
     uint64_t id = c->cmd->id;
 
@@ -1094,6 +1097,7 @@ int ACLCheckCommandPerm(client *c) {
                 }
             }
             if (!match) {
+                if (keyidxptr) *keyidxptr = keyidx[j];
                 getKeysFreeResult(keyidx);
                 return ACL_DENIED_KEY;
             }
@@ -1454,6 +1458,51 @@ void ACLLoadUsersAtStartup(void) {
     }
 }
 
+/* =============================================================================
+ * ACL log
+ * ==========================================================================*/
+
+#define ACL_LOG_CTX_TOPLEVEL 0
+#define ACL_LOG_CTX_LUA 1
+#define ACL_LOG_CTX_MULTI 2
+
+/* This structure defines an entry inside the ACL log. */
+typedef struct aclLogEntry {
+    uint64_t count;     /* Number of times this happened recently. */
+    int reason;         /* Reason for denying the command. ACL_DENIED_*. */
+    int context;        /* Toplevel, Lua or MULTI/EXEC? ACL_LOG_CTX_*. */
+    sds object;         /* The key name or command name. */
+    sds username;       /* User the client is authenticated with. */
+    mstime_t ctime;     /* Milliseconds time of last update to this entry. */
+    sds cinfo;          /* Client info (last client if updated). */
+} aclLogEntry;
+
+void addACLLogEntry(client *c, int reason, int keypos) {
+    /* Create a new entry. */
+    struct aclLogEntry *le = zmalloc(sizeof(*le));
+    le->count = 1;
+    le->object = (reason == ACL_DENIED_CMD) ? sdsnew(c->cmd->name) :
+                                              sdsdup(c->argv[keypos]->ptr);
+    le->username = sdsdup(c->user->name);
+    le->ctime = mstime();
+
+    client *realclient = c;
+    if (realclient->flags & CLIENT_LUA) realclient = server.lua_caller;
+
+    le->cinfo = catClientInfoString(sdsempty(),realclient);
+    if (c->flags & CLIENT_MULTI) {
+        le->context = ACL_LOG_CTX_MULTI;
+    } else if (c->flags & CLIENT_LUA) {
+        le->context = ACL_LOG_CTX_LUA;
+    } else {
+        le->context = ACL_LOG_CTX_TOPLEVEL;
+    }
+
+    /* Add it to our list of entires. We'll have to trim the list
+     * to its maximum size. */
+    listAddNodeHead(ACLLog, le);
+}
+
 /* =============================================================================
  * ACL related commands
  * ==========================================================================*/
diff --git a/src/multi.c b/src/multi.c
index df11225bd..640149870 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -177,7 +177,7 @@ void execCommand(client *c) {
             must_propagate = 1;
         }
 
-        int acl_retval = ACLCheckCommandPerm(c);
+        int acl_retval = ACLCheckCommandPerm(c,NULL);
         if (acl_retval != ACL_OK) {
             addReplyErrorFormat(c,
                 "-NOPERM ACLs rules changed between the moment the "
diff --git a/src/scripting.c b/src/scripting.c
index 9282b7fd9..0e47ebcb0 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -606,7 +606,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) {
     }
 
     /* Check the ACLs. */
-    int acl_retval = ACLCheckCommandPerm(c);
+    int acl_retval = ACLCheckCommandPerm(c,NULL);
     if (acl_retval != ACL_OK) {
         if (acl_retval == ACL_DENIED_CMD)
             luaPushError(lua, "The user executing the script can't run this "
diff --git a/src/server.c b/src/server.c
index 2b226f568..b5e27e238 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3377,7 +3377,7 @@ int processCommand(client *c) {
 
     /* Check if the user can run this command according to the current
      * ACLs. */
-    int acl_retval = ACLCheckCommandPerm(c);
+    int acl_retval = ACLCheckCommandPerm(c,NULL);
     if (acl_retval != ACL_OK) {
         flagTransaction(c);
         if (acl_retval == ACL_DENIED_CMD)
diff --git a/src/server.h b/src/server.h
index 8e354c03d..6fd0ffc5b 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1824,7 +1824,7 @@ int ACLCheckUserCredentials(robj *username, robj *password);
 int ACLAuthenticateUser(client *c, robj *username, robj *password);
 unsigned long ACLGetCommandID(const char *cmdname);
 user *ACLGetUserByName(const char *name, size_t namelen);
-int ACLCheckCommandPerm(client *c);
+int ACLCheckCommandPerm(client *c, int *keyidxptr);
 int ACLSetUser(user *u, const char *op, ssize_t oplen);
 sds ACLDefaultUserFirstPassword(void);
 uint64_t ACLGetCommandCategoryFlagByName(const char *name);

From e8d0057710d11b3361daf37b93e479947aebee5a Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 28 Jan 2020 17:30:50 +0100
Subject: [PATCH 0053/1098] ACL LOG: implement ACL LOG subcommadn skeleton.

---
 src/acl.c | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/src/acl.c b/src/acl.c
index 4391382a6..a166469d0 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1477,6 +1477,11 @@ typedef struct aclLogEntry {
     sds cinfo;          /* Client info (last client if updated). */
 } aclLogEntry;
 
+/* Adds a new entry in the ACL log, making sure to delete the old entry
+ * if we reach the maximum length allowed for the log. This function attempts
+ * to find similar entries in the current log in order to bump the counter of
+ * the log entry instead of creating many entries for very similar ACL
+ * rules issues. */
 void addACLLogEntry(client *c, int reason, int keypos) {
     /* Create a new entry. */
     struct aclLogEntry *le = zmalloc(sizeof(*le));
@@ -1518,6 +1523,7 @@ void addACLLogEntry(client *c, int reason, int keypos) {
  * ACL GETUSER <username>
  * ACL GENPASS
  * ACL WHOAMI
+ * ACL LOG [<count> | RESET]
  */
 void aclCommand(client *c) {
     char *sub = c->argv[1]->ptr;
@@ -1704,6 +1710,36 @@ void aclCommand(client *c) {
         char pass[32]; /* 128 bits of actual pseudo random data. */
         getRandomHexChars(pass,sizeof(pass));
         addReplyBulkCBuffer(c,pass,sizeof(pass));
+    } else if (!strcasecmp(sub,"log") && (c->argc == 2 || c->argc ==3)) {
+        long count = 10; /* Number of entries to emit by default. */
+
+        /* Parse the only argument that LOG may have: it could be either
+         * the number of entires the user wants to display, or alternatively
+         * the "RESET" command in order to flush the old entires. */
+        if (c->argc == 3) {
+            if (!strcasecmp(c->argv[2]->ptr,"reset")) {
+                /* TODO: reset the log. */
+                addReply(c,shared.ok);
+                return;
+            } else if (getLongFromObjectOrReply(c,c->argv[2],&count,NULL)
+                       != C_OK)
+            {
+                return;
+            }
+            if (count < 0) count = 0;
+        }
+
+        /* Fix the count according to the number of entries we got. */
+        if ((size_t)count > listLength(ACLLog))
+            count = listLength(ACLLog);
+
+        addReplyArrayLen(c,count);
+        listIter li;
+        listNode *ln;
+        listRewind(ACLLog,&li);
+        while (count-- && (ln = listNext(&li)) != NULL) {
+            addReplyLongLong(c,1234);
+        }
     } else if (!strcasecmp(sub,"help")) {
         const char *help[] = {
 "LOAD                              -- Reload users from the ACL file.",
@@ -1716,6 +1752,7 @@ void aclCommand(client *c) {
 "CAT <category>                    -- List commands inside category.",
 "GENPASS                           -- Generate a secure user password.",
 "WHOAMI                            -- Return the current connection username.",
+"LOG [<count> | RESET]             -- Show the ACL log entries.",
 NULL
         };
         addReplyHelp(c,help);

From 61dffd8669bdc16f26c67c0ec8955a0d8a627ef8 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 28 Jan 2020 18:04:20 +0100
Subject: [PATCH 0054/1098] ACL LOG: actually emit entries.

---
 src/acl.c    | 34 ++++++++++++++++++++++++++++++----
 src/server.c |  4 +++-
 src/server.h |  1 +
 3 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index a166469d0..5937c069c 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1467,7 +1467,7 @@ void ACLLoadUsersAtStartup(void) {
 #define ACL_LOG_CTX_MULTI 2
 
 /* This structure defines an entry inside the ACL log. */
-typedef struct aclLogEntry {
+typedef struct ACLLogEntry {
     uint64_t count;     /* Number of times this happened recently. */
     int reason;         /* Reason for denying the command. ACL_DENIED_*. */
     int context;        /* Toplevel, Lua or MULTI/EXEC? ACL_LOG_CTX_*. */
@@ -1475,7 +1475,7 @@ typedef struct aclLogEntry {
     sds username;       /* User the client is authenticated with. */
     mstime_t ctime;     /* Milliseconds time of last update to this entry. */
     sds cinfo;          /* Client info (last client if updated). */
-} aclLogEntry;
+} ACLLogEntry;
 
 /* Adds a new entry in the ACL log, making sure to delete the old entry
  * if we reach the maximum length allowed for the log. This function attempts
@@ -1484,8 +1484,9 @@ typedef struct aclLogEntry {
  * rules issues. */
 void addACLLogEntry(client *c, int reason, int keypos) {
     /* Create a new entry. */
-    struct aclLogEntry *le = zmalloc(sizeof(*le));
+    struct ACLLogEntry *le = zmalloc(sizeof(*le));
     le->count = 1;
+    le->reason = reason;
     le->object = (reason == ACL_DENIED_CMD) ? sdsnew(c->cmd->name) :
                                               sdsdup(c->argv[keypos]->ptr);
     le->username = sdsdup(c->user->name);
@@ -1737,8 +1738,33 @@ void aclCommand(client *c) {
         listIter li;
         listNode *ln;
         listRewind(ACLLog,&li);
+        mstime_t now = mstime();
         while (count-- && (ln = listNext(&li)) != NULL) {
-            addReplyLongLong(c,1234);
+            ACLLogEntry *le = listNodeValue(ln);
+            addReplyMapLen(c,7);
+            addReplyBulkCString(c,"count");
+            addReplyLongLong(c,le->count);
+            addReplyBulkCString(c,"reason");
+            addReplyBulkCString(c,(le->reason == ACL_DENIED_CMD) ?
+                                  "command" : "key");
+            char *ctxstr;
+            switch(le->context) {
+            case ACL_LOG_CTX_TOPLEVEL: ctxstr="toplevel"; break;
+            case ACL_LOG_CTX_MULTI: ctxstr="multi"; break;
+            case ACL_LOG_CTX_LUA: ctxstr="lua"; break;
+            default: ctxstr="unknown";
+            }
+            addReplyBulkCString(c,"context");
+            addReplyBulkCString(c,ctxstr);
+            addReplyBulkCString(c,"object");
+            addReplyBulkCBuffer(c,le->object,sdslen(le->object));
+            addReplyBulkCString(c,"username");
+            addReplyBulkCBuffer(c,le->username,sdslen(le->username));
+            addReplyBulkCString(c,"age-seconds");
+            double age = (double)(now - le->ctime)/1000;
+            addReplyDouble(c,age);
+            addReplyBulkCString(c,"client-info");
+            addReplyBulkCBuffer(c,le->cinfo,sdslen(le->cinfo));
         }
     } else if (!strcasecmp(sub,"help")) {
         const char *help[] = {
diff --git a/src/server.c b/src/server.c
index b5e27e238..6968f311f 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3377,8 +3377,10 @@ int processCommand(client *c) {
 
     /* Check if the user can run this command according to the current
      * ACLs. */
-    int acl_retval = ACLCheckCommandPerm(c,NULL);
+    int acl_keypos;
+    int acl_retval = ACLCheckCommandPerm(c,&acl_keypos);
     if (acl_retval != ACL_OK) {
+        addACLLogEntry(c,acl_retval,acl_keypos);
         flagTransaction(c);
         if (acl_retval == ACL_DENIED_CMD)
             addReplyErrorFormat(c,
diff --git a/src/server.h b/src/server.h
index 6fd0ffc5b..bd4aed192 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1836,6 +1836,7 @@ void ACLLoadUsersAtStartup(void);
 void addReplyCommandCategories(client *c, struct redisCommand *cmd);
 user *ACLCreateUnlinkedUser();
 void ACLFreeUserAndKillClients(user *u);
+void addACLLogEntry(client *c, int reason, int keypos);
 
 /* Sorted sets data type */
 

From 17ff3173d62aa420c5970ccfa0f7dbca7c64babd Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 29 Jan 2020 12:47:50 +0100
Subject: [PATCH 0055/1098] Add more info in the unblockClientFromModule()
 function.

---
 src/module.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/module.c b/src/module.c
index d2b267be2..7fdab1b34 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4282,7 +4282,13 @@ void unblockClientFromModule(client *c) {
      * not implemented (or it was, but RM_UnblockClient was not called from
      * within it, as it should).
      * We must call moduleUnblockClient in order to free privdata and
-     * RedisModuleBlockedClient */
+     * RedisModuleBlockedClient.
+     *
+     * Note that clients implementing threads and working with private data,
+     * should make sure to stop the threads or protect the private data
+     * in some other way in the disconnection and timeout callback, because
+     * here we are going to free the private data associated with the
+     * blocked client. */
     if (!bc->unblocked)
         moduleUnblockClient(c);
 

From 6671032fafc82be04b4ca564790753b26f96780e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 29 Jan 2020 18:40:32 +0100
Subject: [PATCH 0056/1098] ACL LOG: group similar entries in a given time
 delta.

---
 src/acl.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 58 insertions(+), 3 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 5937c069c..d03599a79 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1465,6 +1465,7 @@ void ACLLoadUsersAtStartup(void) {
 #define ACL_LOG_CTX_TOPLEVEL 0
 #define ACL_LOG_CTX_LUA 1
 #define ACL_LOG_CTX_MULTI 2
+#define ACL_LOG_GROUPING_MAX_TIME_DELTA 60000
 
 /* This structure defines an entry inside the ACL log. */
 typedef struct ACLLogEntry {
@@ -1477,6 +1478,28 @@ typedef struct ACLLogEntry {
     sds cinfo;          /* Client info (last client if updated). */
 } ACLLogEntry;
 
+/* This function will check if ACL entries 'a' and 'b' are similar enough
+ * that we should actually update the existing entry in our ACL log instead
+ * of creating a new one. */
+int ACLLogMatchEntry(ACLLogEntry *a, ACLLogEntry *b) {
+    if (a->reason != b->reason) return 0;
+    if (a->context != b->context) return 0;
+    mstime_t delta = a->ctime - b->ctime;
+    if (delta < 0) delta = -delta;
+    if (delta > ACL_LOG_GROUPING_MAX_TIME_DELTA) return 0;
+    if (sdscmp(a->object,b->object) != 0) return 0;
+    if (sdscmp(a->username,b->username) != 0) return 0;
+    return 1;
+}
+
+/* Release an ACL log entry. */
+void ACLFreeLogEntry(ACLLogEntry *le) {
+    sdsfree(le->object);
+    sdsfree(le->username);
+    sdsfree(le->cinfo);
+    zfree(le);
+}
+
 /* Adds a new entry in the ACL log, making sure to delete the old entry
  * if we reach the maximum length allowed for the log. This function attempts
  * to find similar entries in the current log in order to bump the counter of
@@ -1504,9 +1527,41 @@ void addACLLogEntry(client *c, int reason, int keypos) {
         le->context = ACL_LOG_CTX_TOPLEVEL;
     }
 
-    /* Add it to our list of entires. We'll have to trim the list
-     * to its maximum size. */
-    listAddNodeHead(ACLLog, le);
+    /* Try to match this entry with past ones, to see if we can just
+     * update an existing entry instead of creating a new one. */
+    long toscan = 10; /* Do a limited work trying to find duplicated. */
+    listIter li;
+    listNode *ln;
+    listRewind(ACLLog,&li);
+    ACLLogEntry *match = NULL;
+    while (toscan-- && (ln = listNext(&li)) != NULL) {
+        ACLLogEntry *current = listNodeValue(ln);
+        if (ACLLogMatchEntry(current,le)) {
+            match = current;
+            listDelNode(ACLLog,ln);
+            listAddNodeHead(ACLLog,current);
+            break;
+        }
+    }
+
+    /* If there is a match update the entry, otherwise add it as a
+     * new one. */
+    if (match) {
+        /* We update a few fields of the existing entry and bump the
+         * counter of events for this entry. */
+        sdsfree(match->cinfo);
+        match->cinfo = le->cinfo;
+        match->ctime = le->ctime;
+        match->count++;
+
+        /* Release the old entry. */
+        le->cinfo = NULL;
+        ACLFreeLogEntry(le);
+    } else {
+        /* Add it to our list of entires. We'll have to trim the list
+         * to its maximum size. */
+        listAddNodeHead(ACLLog, le);
+    }
 }
 
 /* =============================================================================

From 30a466ba38759e8deb5ed79c18bb3d3dec352ad7 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 29 Jan 2020 18:51:04 +0100
Subject: [PATCH 0057/1098] ACL LOG: implement LOG RESET.

---
 src/acl.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index d03599a79..16acd4d3e 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -95,6 +95,7 @@ struct ACLUserFlag {
 void ACLResetSubcommandsForCommand(user *u, unsigned long id);
 void ACLResetSubcommands(user *u);
 void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub);
+void ACLFreeLogEntry(void *le);
 
 /* The length of the string representation of a hashed password. */
 #define HASH_PASSWORD_LEN SHA256_BLOCK_SIZE*2
@@ -1493,7 +1494,8 @@ int ACLLogMatchEntry(ACLLogEntry *a, ACLLogEntry *b) {
 }
 
 /* Release an ACL log entry. */
-void ACLFreeLogEntry(ACLLogEntry *le) {
+void ACLFreeLogEntry(void *leptr) {
+    ACLLogEntry *le = leptr;
     sdsfree(le->object);
     sdsfree(le->username);
     sdsfree(le->cinfo);
@@ -1774,7 +1776,9 @@ void aclCommand(client *c) {
          * the "RESET" command in order to flush the old entires. */
         if (c->argc == 3) {
             if (!strcasecmp(c->argv[2]->ptr,"reset")) {
-                /* TODO: reset the log. */
+                listSetFreeMethod(ACLLog,ACLFreeLogEntry);
+                listEmpty(ACLLog);
+                listSetFreeMethod(ACLLog,NULL);
                 addReply(c,shared.ok);
                 return;
             } else if (getLongFromObjectOrReply(c,c->argv[2],&count,NULL)

From 93edb3ff3a800a701e2c33eb8f20330569a0a134 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 29 Jan 2020 21:40:02 +0200
Subject: [PATCH 0058/1098] TLS: Fix missing initialization in redis-cli.

---
 src/redis-cli.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 065c389c6..1919829e1 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -49,6 +49,7 @@
 #include <hiredis.h>
 #ifdef USE_OPENSSL
 #include <openssl/ssl.h>
+#include <openssl/err.h>
 #include <hiredis_ssl.h>
 #endif
 #include <sds.h> /* use sds.h from hiredis, so that only one set of sds functions will be present in the binary */
@@ -7933,6 +7934,14 @@ int main(int argc, char **argv) {
 
     parseEnv();
 
+#ifdef USE_OPENSSL
+    if (config.tls) {
+        ERR_load_crypto_strings();
+        SSL_load_error_strings();
+        SSL_library_init();
+    }
+#endif
+
     /* Cluster Manager mode */
     if (CLUSTER_MANAGER_MODE()) {
         clusterManagerCommandProc *proc = validateClusterManagerCommand();

From 396161765b4f44f84fe428576a4272003669cec9 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 30 Jan 2020 10:50:32 +0100
Subject: [PATCH 0059/1098] ACL LOG: also log ACL errors in the scripting/MULTI
 ctx.

---
 src/multi.c     | 4 +++-
 src/scripting.c | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/multi.c b/src/multi.c
index 640149870..a88e5336b 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -177,8 +177,10 @@ void execCommand(client *c) {
             must_propagate = 1;
         }
 
-        int acl_retval = ACLCheckCommandPerm(c,NULL);
+        int acl_keypos;
+        int acl_retval = ACLCheckCommandPerm(c,&acl_keypos);
         if (acl_retval != ACL_OK) {
+            addACLLogEntry(c,acl_retval,acl_keypos);
             addReplyErrorFormat(c,
                 "-NOPERM ACLs rules changed between the moment the "
                 "transaction was accumulated and the EXEC call. "
diff --git a/src/scripting.c b/src/scripting.c
index 0e47ebcb0..916b90cf1 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -606,8 +606,10 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) {
     }
 
     /* Check the ACLs. */
-    int acl_retval = ACLCheckCommandPerm(c,NULL);
+    int acl_keypos;
+    int acl_retval = ACLCheckCommandPerm(c,&acl_keypos);
     if (acl_retval != ACL_OK) {
+        addACLLogEntry(c,acl_retval,acl_keypos);
         if (acl_retval == ACL_DENIED_CMD)
             luaPushError(lua, "The user executing the script can't run this "
                               "command or subcommand");

From b189a2197489d20cf233d447f961e8a9c3b4009a Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 30 Jan 2020 11:09:50 +0100
Subject: [PATCH 0060/1098] ACL LOG: implement a few basic tests.

---
 tests/unit/acl.tcl | 87 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 87 insertions(+)

diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl
index 2205d2d86..bd909d36c 100644
--- a/tests/unit/acl.tcl
+++ b/tests/unit/acl.tcl
@@ -141,4 +141,91 @@ start_server {tags {"acl"}} {
         r ACL setuser newuser -debug
         # The test framework will detect a leak if any.
     }
+
+    test {ACL LOG shows failed command executions at toplevel} {
+        r ACL LOG RESET
+        r ACL setuser antirez >foo on +set ~object:1234
+        r ACL setuser antirez +eval +multi +exec
+        r AUTH antirez foo
+        catch {r GET foo}
+        r AUTH default ""
+        set entry [lindex [r ACL LOG] 0]
+        assert {[dict get $entry username] eq {antirez}}
+        assert {[dict get $entry context] eq {toplevel}}
+        assert {[dict get $entry reason] eq {command}}
+        assert {[dict get $entry object] eq {get}}
+    }
+
+    test {ACL LOG is able to test similar events} {
+        r AUTH antirez foo
+        catch {r GET foo}
+        catch {r GET foo}
+        catch {r GET foo}
+        r AUTH default ""
+        set entry [lindex [r ACL LOG] 0]
+        assert {[dict get $entry count] == 4}
+    }
+
+    test {ACL LOG is able to log keys access violations and key name} {
+        r AUTH antirez foo
+        catch {r SET somekeynotallowed 1234}
+        r AUTH default ""
+        set entry [lindex [r ACL LOG] 0]
+        assert {[dict get $entry reason] eq {key}}
+        assert {[dict get $entry object] eq {somekeynotallowed}}
+    }
+
+    test {ACL LOG RESET is able to flush the entries in the log} {
+        r ACL LOG RESET
+        assert {[llength [r ACL LOG]] == 0}
+    }
+
+    test {ACL LOG can distinguish the transaction context (1)} {
+        r AUTH antirez foo
+        r MULTI
+        catch {r INCR foo}
+        catch {r EXEC}
+        r AUTH default ""
+        set entry [lindex [r ACL LOG] 0]
+        assert {[dict get $entry context] eq {multi}}
+        assert {[dict get $entry object] eq {incr}}
+    }
+
+    test {ACL LOG can distinguish the transaction context (2)} {
+        set rd1 [redis_deferring_client]
+        r ACL SETUSER antirez +incr
+
+        r AUTH antirez foo
+        r MULTI
+        r INCR object:1234
+        $rd1 ACL SETUSER antirez -incr
+        $rd1 read
+        catch {r EXEC}
+        $rd1 close
+        r AUTH default ""
+        set entry [lindex [r ACL LOG] 0]
+        assert {[dict get $entry context] eq {multi}}
+        assert {[dict get $entry object] eq {incr}}
+        r ACL SETUSER antirez -incr
+    }
+
+    test {ACL can log errors in the context of Lua scripting} {
+        r AUTH antirez foo
+        catch {r EVAL {redis.call('incr','foo')} 0}
+        r AUTH default ""
+        set entry [lindex [r ACL LOG] 0]
+        assert {[dict get $entry context] eq {lua}}
+        assert {[dict get $entry object] eq {incr}}
+    }
+
+    test {ACL LOG can accept a numerical argument to show less entries} {
+        r AUTH antirez foo
+        catch {r INCR foo}
+        catch {r INCR foo}
+        catch {r INCR foo}
+        catch {r INCR foo}
+        r AUTH default ""
+        assert {[llength [r ACL LOG]] > 1}
+        assert {[llength [r ACL LOG 2]] == 2}
+    }
 }

From 2deb55512ffd86e494d376586cc0d402db21ca89 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 30 Jan 2020 18:14:45 +0530
Subject: [PATCH 0061/1098] ld2string should fail if string contains \0 in the
 middle

This bug affected RM_StringToLongDouble and HINCRBYFLOAT.
I added tests for both cases.

Main changes:
1. Fixed string2ld to fail if string contains \0 in the middle
2. Use string2ld in getLongDoubleFromObject - No point of
   having duplicated code here

The two changes above broke RM_SaveLongDouble/RM_LoadLongDouble
because the long double string was saved with length+1 (An innocent
mistake, but it's actually a bug - The length passed to
RM_SaveLongDouble should not include the last \0).
---
 src/module.c             |  2 +-
 src/object.c             | 10 +---------
 src/util.c               |  3 ++-
 tests/modules/misc.c     |  9 +++++++++
 tests/unit/type/hash.tcl |  7 +++++++
 5 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/src/module.c b/src/module.c
index d2b267be2..584912fbe 100644
--- a/src/module.c
+++ b/src/module.c
@@ -3901,7 +3901,7 @@ void RM_SaveLongDouble(RedisModuleIO *io, long double value) {
     /* Long double has different number of bits in different platforms, so we
      * save it as a string type. */
     size_t len = ld2string(buf,sizeof(buf),value,LD_STR_HEX);
-    RM_SaveStringBuffer(io,buf,len+1); /* len+1 for '\0' */
+    RM_SaveStringBuffer(io,buf,len);
 }
 
 /* In the context of the rdb_save method of a module data type, loads back the
diff --git a/src/object.c b/src/object.c
index 2201a317a..52007b474 100644
--- a/src/object.c
+++ b/src/object.c
@@ -640,21 +640,13 @@ int getDoubleFromObjectOrReply(client *c, robj *o, double *target, const char *m
 
 int getLongDoubleFromObject(robj *o, long double *target) {
     long double value;
-    char *eptr;
 
     if (o == NULL) {
         value = 0;
     } else {
         serverAssertWithInfo(NULL,o,o->type == OBJ_STRING);
         if (sdsEncodedObject(o)) {
-            errno = 0;
-            value = strtold(o->ptr, &eptr);
-            if (sdslen(o->ptr) == 0 ||
-                isspace(((const char*)o->ptr)[0]) ||
-                (size_t)(eptr-(char*)o->ptr) != sdslen(o->ptr) ||
-                (errno == ERANGE &&
-                    (value == HUGE_VAL || value == -HUGE_VAL || value == 0)) ||
-                isnan(value))
+            if (!string2ld(o->ptr, sdslen(o->ptr), &value))
                 return C_ERR;
         } else if (o->encoding == OBJ_ENCODING_INT) {
             value = (long)o->ptr;
diff --git a/src/util.c b/src/util.c
index 20471b539..2be42a0df 100644
--- a/src/util.c
+++ b/src/util.c
@@ -471,13 +471,14 @@ int string2ld(const char *s, size_t slen, long double *dp) {
     long double value;
     char *eptr;
 
-    if (slen >= sizeof(buf)) return 0;
+    if (slen == 0 || slen >= sizeof(buf)) return 0;
     memcpy(buf,s,slen);
     buf[slen] = '\0';
 
     errno = 0;
     value = strtold(buf, &eptr);
     if (isspace(buf[0]) || eptr[0] != '\0' ||
+        (size_t)(eptr-buf) != slen ||
         (errno == ERANGE &&
             (value == HUGE_VAL || value == -HUGE_VAL || value == 0)) ||
         errno == EINVAL ||
diff --git a/tests/modules/misc.c b/tests/modules/misc.c
index b5a032f60..41bec06ed 100644
--- a/tests/modules/misc.c
+++ b/tests/modules/misc.c
@@ -74,6 +74,15 @@ int test_ld_conv(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
         RedisModule_ReplyWithError(ctx, err);
         goto final;
     }
+    /* Make sure we can't convert a string that has \0 in it */
+    char buf[4] = "123";
+    buf[1] = '\0';
+    RedisModuleString *s3 = RedisModule_CreateString(ctx, buf, 3);
+    long double ld3;
+    if (RedisModule_StringToLongDouble(s3, &ld3) == REDISMODULE_OK) {
+        RedisModule_ReplyWithError(ctx, "Invalid string successfully converted to long double");
+        goto final;
+    }
     RedisModule_ReplyWithLongDouble(ctx, ld2);
 final:
     RedisModule_FreeString(ctx, s1);
diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl
index d2c679d32..9f8a21b1c 100644
--- a/tests/unit/type/hash.tcl
+++ b/tests/unit/type/hash.tcl
@@ -390,6 +390,13 @@ start_server {tags {"hash"}} {
         lappend rv [string match "ERR*not*float*" $bigerr]
     } {1 1}
 
+    test {HINCRBYFLOAT fails against hash value that contains a null-terminator in the middle} {
+        r hset h f "1\x002"
+        catch {r hincrbyfloat h f 1} err
+        set rv {}
+        lappend rv [string match "ERR*not*float*" $err]
+    } {1}
+
     test {HSTRLEN against the small hash} {
         set err {}
         foreach k [array names smallhash *] {

From d72b7ed4fbcf8f16196d78267b469204741121db Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 30 Jan 2020 19:15:12 +0530
Subject: [PATCH 0062/1098] DEBUG OBJECT should pass keyname to module when
 loading

---
 src/debug.c | 2 +-
 src/rdb.c   | 4 ++--
 src/rdb.h   | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/debug.c b/src/debug.c
index a2d37337d..fecee25e1 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -488,7 +488,7 @@ NULL
             "encoding:%s serializedlength:%zu "
             "lru:%d lru_seconds_idle:%llu%s",
             (void*)val, val->refcount,
-            strenc, rdbSavedObjectLen(val),
+            strenc, rdbSavedObjectLen(val, c->argv[2]),
             val->lru, estimateObjectIdleTime(val)/1000, extra);
     } else if (!strcasecmp(c->argv[1]->ptr,"sdslen") && c->argc == 3) {
         dictEntry *de;
diff --git a/src/rdb.c b/src/rdb.c
index 27e1b3135..de3cdb459 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1002,8 +1002,8 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key) {
  * the rdbSaveObject() function. Currently we use a trick to get
  * this length with very little changes to the code. In the future
  * we could switch to a faster solution. */
-size_t rdbSavedObjectLen(robj *o) {
-    ssize_t len = rdbSaveObject(NULL,o,NULL);
+size_t rdbSavedObjectLen(robj *o, robj *key) {
+    ssize_t len = rdbSaveObject(NULL,o,key);
     serverAssertWithInfo(NULL,o,len != -1);
     return len;
 }
diff --git a/src/rdb.h b/src/rdb.h
index 4229beea8..b276a978b 100644
--- a/src/rdb.h
+++ b/src/rdb.h
@@ -143,7 +143,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi);
 void rdbRemoveTempFile(pid_t childpid);
 int rdbSave(char *filename, rdbSaveInfo *rsi);
 ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key);
-size_t rdbSavedObjectLen(robj *o);
+size_t rdbSavedObjectLen(robj *o, robj *key);
 robj *rdbLoadObject(int type, rio *rdb, robj *key);
 void backgroundSaveDoneHandler(int exitcode, int bysignal);
 int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime);

From 560e1e6c4b551a950699232074f7a6818bd4666e Mon Sep 17 00:00:00 2001
From: Leo Murillo <leonardo.murillo@gmail.com>
Date: Sun, 2 Feb 2020 02:48:00 -0600
Subject: [PATCH 0063/1098] Set ZSKIPLIST_MAXLEVEL to optimal value given 2^64
 elements and p=0.25

---
 src/server.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.h b/src/server.h
index 8e354c03d..5d63e9b55 100644
--- a/src/server.h
+++ b/src/server.h
@@ -335,7 +335,7 @@ typedef long long ustime_t; /* microsecond time type. */
 /* Anti-warning macro... */
 #define UNUSED(V) ((void) V)
 
-#define ZSKIPLIST_MAXLEVEL 64 /* Should be enough for 2^64 elements */
+#define ZSKIPLIST_MAXLEVEL 32 /* Should be enough for 2^64 elements */
 #define ZSKIPLIST_P 0.25      /* Skiplist P = 1/4 */
 
 /* Append only defines */

From 2fda5f5c98ca0f0ce426c7dff3951804db62e0fe Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Mon, 3 Feb 2020 15:19:44 +0530
Subject: [PATCH 0064/1098] Exclude "keymiss" notification from NOTIFY_ALL

Because "keymiss" is "special" compared to the rest of
the notifications (Trying not to break existing apps
using the 'A' format for notifications)

Also updated redis.conf and module.c docs
---
 redis.conf        | 6 +++++-
 src/module.c      | 3 ++-
 src/notify.c      | 2 +-
 src/redismodule.h | 4 ++--
 src/server.h      | 4 ++--
 5 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/redis.conf b/redis.conf
index 07005cffe..2b8711eea 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1362,7 +1362,11 @@ latency-monitor-threshold 0
 #  z     Sorted set commands
 #  x     Expired events (events generated every time a key expires)
 #  e     Evicted events (events generated when a key is evicted for maxmemory)
-#  A     Alias for g$lshzxe, so that the "AKE" string means all the events.
+#  t     Stream commands
+#  m     Key-miss events (Note: It is not included in the 'A' class)
+#  A     Alias for g$lshzxet, so that the "AKE" string means all the events
+#        (Except key-miss events which are excluded from 'A' due to their
+#         unique nature).
 #
 #  The "notify-keyspace-events" takes as argument a string that is composed
 #  of zero or multiple characters. The empty string means that notifications
diff --git a/src/module.c b/src/module.c
index d2b267be2..54072af57 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4798,7 +4798,8 @@ void moduleReleaseGIL(void) {
  *  - REDISMODULE_NOTIFY_EXPIRED: Expiration events
  *  - REDISMODULE_NOTIFY_EVICTED: Eviction events
  *  - REDISMODULE_NOTIFY_STREAM: Stream events
- *  - REDISMODULE_NOTIFY_ALL: All events
+ *  - REDISMODULE_NOTIFY_KEYMISS: Key-miss events
+ *  - REDISMODULE_NOTIFY_ALL: All events (Excluding REDISMODULE_NOTIFY_KEYMISS)
  *
  * We do not distinguish between key events and keyspace events, and it is up
  * to the module to filter the actions taken based on the key.
diff --git a/src/notify.c b/src/notify.c
index d6c3ad403..bb1055724 100644
--- a/src/notify.c
+++ b/src/notify.c
@@ -82,10 +82,10 @@ sds keyspaceEventsFlagsToString(int flags) {
         if (flags & NOTIFY_EXPIRED) res = sdscatlen(res,"x",1);
         if (flags & NOTIFY_EVICTED) res = sdscatlen(res,"e",1);
         if (flags & NOTIFY_STREAM) res = sdscatlen(res,"t",1);
-        if (flags & NOTIFY_KEY_MISS) res = sdscatlen(res,"m",1);
     }
     if (flags & NOTIFY_KEYSPACE) res = sdscatlen(res,"K",1);
     if (flags & NOTIFY_KEYEVENT) res = sdscatlen(res,"E",1);
+    if (flags & NOTIFY_KEY_MISS) res = sdscatlen(res,"m",1);
     return res;
 }
 
diff --git a/src/redismodule.h b/src/redismodule.h
index 637078f2b..e74611f13 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -127,8 +127,8 @@
 #define REDISMODULE_NOTIFY_EXPIRED (1<<8)     /* x */
 #define REDISMODULE_NOTIFY_EVICTED (1<<9)     /* e */
 #define REDISMODULE_NOTIFY_STREAM (1<<10)     /* t */
-#define REDISMODULE_NOTIFY_KEY_MISS (1<<11)   /* m */
-#define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM | REDISMODULE_NOTIFY_KEY_MISS)      /* A */
+#define REDISMODULE_NOTIFY_KEY_MISS (1<<11)   /* m (Note: This one is excluded from REDISMODULE_NOTIFY_ALL on purpose) */
+#define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM)      /* A */
 
 
 /* A special pointer that we can use between the core and the module to signal
diff --git a/src/server.h b/src/server.h
index 8e354c03d..257f22985 100644
--- a/src/server.h
+++ b/src/server.h
@@ -413,8 +413,8 @@ typedef long long ustime_t; /* microsecond time type. */
 #define NOTIFY_EXPIRED (1<<8)     /* x */
 #define NOTIFY_EVICTED (1<<9)     /* e */
 #define NOTIFY_STREAM (1<<10)     /* t */
-#define NOTIFY_KEY_MISS (1<<11)   /* m */
-#define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED | NOTIFY_STREAM | NOTIFY_KEY_MISS) /* A flag */
+#define NOTIFY_KEY_MISS (1<<11)   /* m (Note: This one is excluded from NOTIFY_ALL on purpose) */
+#define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED | NOTIFY_STREAM) /* A flag */
 
 /* Get the first bind addr or NULL */
 #define NET_FIRST_BIND_ADDR (server.bindaddr_count ? server.bindaddr[0] : NULL)

From 138ed120dd6ca084e2d2e7fa78a4a1e9dd87d2a3 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 3 Feb 2020 15:58:28 +0200
Subject: [PATCH 0065/1098] fix uninitialized info_cb var in module.c

---
 src/module.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/module.c b/src/module.c
index d922c5c20..914c50df3 100644
--- a/src/module.c
+++ b/src/module.c
@@ -859,6 +859,7 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api
     module->in_call = 0;
     module->in_hook = 0;
     module->options = 0;
+    module->info_cb = 0;
     ctx->module = module;
 }
 

From 01eaf53bb37506ddd922da89144b059ff85cd7fb Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Tue, 4 Feb 2020 16:34:11 +0800
Subject: [PATCH 0066/1098] Add tcl regression test in scripting.tcl to
 reproduce memory leak.

---
 tests/unit/scripting.tcl | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl
index 2543a0377..fb36d0b80 100644
--- a/tests/unit/scripting.tcl
+++ b/tests/unit/scripting.tcl
@@ -741,3 +741,8 @@ start_server {tags {"scripting repl"}} {
     }
 }
 
+start_server {tags {"scripting"}} {
+    r script debug sync
+    r eval {return 'hello'} 0
+    r eval {return 'hello'} 0
+}

From eb2196f5ce68ee72dd64dffa75b9ecb3e930e0cd Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Tue, 4 Feb 2020 16:38:46 +0800
Subject: [PATCH 0067/1098] Fix lua related memory leak.

---
 src/scripting.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/scripting.c b/src/scripting.c
index 9282b7fd9..a5c59b113 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -2473,6 +2473,7 @@ void ldbEval(lua_State *lua, sds *argv, int argc) {
             ldbLog(sdscatfmt(sdsempty(),"<error> %s",lua_tostring(lua,-1)));
             lua_pop(lua,1);
             sdsfree(code);
+            sdsfree(expr);
             return;
         }
     }

From 540b917a262c376f187dda51b4a7988aadef7ef1 Mon Sep 17 00:00:00 2001
From: lifubang <lifubang@acmcoder.com>
Date: Tue, 4 Feb 2020 17:32:30 +0800
Subject: [PATCH 0068/1098] fix ssl flag check for redis-cli

Signed-off-by: lifubang <lifubang@acmcoder.com>
---
 src/redis-cli.c | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 1919829e1..c1bda0a00 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -1595,15 +1595,15 @@ static int parseOptions(int argc, char **argv) {
 #ifdef USE_OPENSSL
         } else if (!strcmp(argv[i],"--tls")) {
             config.tls = 1;
-        } else if (!strcmp(argv[i],"--sni")) {
+        } else if (!strcmp(argv[i],"--sni") && !lastarg) {
             config.sni = argv[++i];
-        } else if (!strcmp(argv[i],"--cacertdir")) {
+        } else if (!strcmp(argv[i],"--cacertdir") && !lastarg) {
             config.cacertdir = argv[++i];
-        } else if (!strcmp(argv[i],"--cacert")) {
+        } else if (!strcmp(argv[i],"--cacert") && !lastarg) {
             config.cacert = argv[++i];
-        } else if (!strcmp(argv[i],"--cert")) {
+        } else if (!strcmp(argv[i],"--cert") && !lastarg) {
             config.cert = argv[++i];
-        } else if (!strcmp(argv[i],"--key")) {
+        } else if (!strcmp(argv[i],"--key") && !lastarg) {
             config.key = argv[++i];
 #endif
         } else if (!strcmp(argv[i],"-v") || !strcmp(argv[i], "--version")) {
@@ -1701,12 +1701,13 @@ static void usage(void) {
 "  -c                 Enable cluster mode (follow -ASK and -MOVED redirections).\n"
 #ifdef USE_OPENSSL
 "  --tls              Establish a secure TLS connection.\n"
-"  --cacert           CA Certificate file to verify with.\n"
-"  --cacertdir        Directory where trusted CA certificates are stored.\n"
+"  --sni <host>       Server name indication for TLS.\n"
+"  --cacert <file>    CA Certificate file to verify with.\n"
+"  --cacertdir <dir>  Directory where trusted CA certificates are stored.\n"
 "                     If neither cacert nor cacertdir are specified, the default\n"
 "                     system-wide trusted root certs configuration will apply.\n"
-"  --cert             Client certificate to authenticate with.\n"
-"  --key              Private key file to authenticate with.\n"
+"  --cert <file>      Client certificate to authenticate with.\n"
+"  --key <file>       Private key file to authenticate with.\n"
 #endif
 "  --raw              Use raw formatting for replies (default when STDOUT is\n"
 "                     not a tty).\n"

From 0c1a4b557691df2196d41835aeef5c1f29914b82 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 4 Feb 2020 12:55:26 +0100
Subject: [PATCH 0069/1098] ACL LOG: log failed auth attempts.

---
 src/acl.c       | 37 +++++++++++++++++++++++++++++--------
 src/multi.c     |  2 +-
 src/scripting.c |  2 +-
 src/server.c    |  2 +-
 src/server.h    |  3 ++-
 5 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 16acd4d3e..97c00d4f8 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -982,6 +982,7 @@ int ACLAuthenticateUser(client *c, robj *username, robj *password) {
         moduleNotifyUserChanged(c);
         return C_OK;
     } else {
+        addACLLogEntry(c,ACL_DENIED_AUTH,0,username->ptr);
         return C_ERR;
     }
 }
@@ -1506,17 +1507,29 @@ void ACLFreeLogEntry(void *leptr) {
  * if we reach the maximum length allowed for the log. This function attempts
  * to find similar entries in the current log in order to bump the counter of
  * the log entry instead of creating many entries for very similar ACL
- * rules issues. */
-void addACLLogEntry(client *c, int reason, int keypos) {
+ * rules issues.
+ *
+ * The keypos argument is only used when the reason is ACL_DENIED_KEY, since
+ * it allows the function to log the key name that caused the problem.
+ * Similarly the username is only passed when we failed to authenticate the
+ * user with AUTH or HELLO, for the ACL_DENIED_AUTH reason. Otherwise
+ * it will just be NULL.
+ */
+void addACLLogEntry(client *c, int reason, int keypos, sds username) {
     /* Create a new entry. */
     struct ACLLogEntry *le = zmalloc(sizeof(*le));
     le->count = 1;
     le->reason = reason;
-    le->object = (reason == ACL_DENIED_CMD) ? sdsnew(c->cmd->name) :
-                                              sdsdup(c->argv[keypos]->ptr);
-    le->username = sdsdup(c->user->name);
+    le->username = sdsdup(reason == ACL_DENIED_AUTH ? username : c->user->name);
     le->ctime = mstime();
 
+    switch(reason) {
+    case ACL_DENIED_CMD: le->object = sdsnew(c->cmd->name); break;
+    case ACL_DENIED_KEY: le->object = sdsnew(c->argv[keypos]->ptr); break;
+    case ACL_DENIED_AUTH: le->object = sdsnew(c->argv[0]->ptr); break;
+    default: le->object = sdsempty();
+    }
+
     client *realclient = c;
     if (realclient->flags & CLIENT_LUA) realclient = server.lua_caller;
 
@@ -1803,9 +1816,17 @@ void aclCommand(client *c) {
             addReplyMapLen(c,7);
             addReplyBulkCString(c,"count");
             addReplyLongLong(c,le->count);
+
             addReplyBulkCString(c,"reason");
-            addReplyBulkCString(c,(le->reason == ACL_DENIED_CMD) ?
-                                  "command" : "key");
+            char *reasonstr;
+            switch(le->reason) {
+            case ACL_DENIED_CMD: reasonstr="command"; break;
+            case ACL_DENIED_KEY: reasonstr="key"; break;
+            case ACL_DENIED_AUTH: reasonstr="auth"; break;
+            }
+            addReplyBulkCString(c,reasonstr);
+
+            addReplyBulkCString(c,"context");
             char *ctxstr;
             switch(le->context) {
             case ACL_LOG_CTX_TOPLEVEL: ctxstr="toplevel"; break;
@@ -1813,8 +1834,8 @@ void aclCommand(client *c) {
             case ACL_LOG_CTX_LUA: ctxstr="lua"; break;
             default: ctxstr="unknown";
             }
-            addReplyBulkCString(c,"context");
             addReplyBulkCString(c,ctxstr);
+
             addReplyBulkCString(c,"object");
             addReplyBulkCBuffer(c,le->object,sdslen(le->object));
             addReplyBulkCString(c,"username");
diff --git a/src/multi.c b/src/multi.c
index a88e5336b..cbbd2c513 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -180,7 +180,7 @@ void execCommand(client *c) {
         int acl_keypos;
         int acl_retval = ACLCheckCommandPerm(c,&acl_keypos);
         if (acl_retval != ACL_OK) {
-            addACLLogEntry(c,acl_retval,acl_keypos);
+            addACLLogEntry(c,acl_retval,acl_keypos,NULL);
             addReplyErrorFormat(c,
                 "-NOPERM ACLs rules changed between the moment the "
                 "transaction was accumulated and the EXEC call. "
diff --git a/src/scripting.c b/src/scripting.c
index 916b90cf1..2eac56b47 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -609,7 +609,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) {
     int acl_keypos;
     int acl_retval = ACLCheckCommandPerm(c,&acl_keypos);
     if (acl_retval != ACL_OK) {
-        addACLLogEntry(c,acl_retval,acl_keypos);
+        addACLLogEntry(c,acl_retval,acl_keypos,NULL);
         if (acl_retval == ACL_DENIED_CMD)
             luaPushError(lua, "The user executing the script can't run this "
                               "command or subcommand");
diff --git a/src/server.c b/src/server.c
index 6968f311f..2827188e0 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3380,7 +3380,7 @@ int processCommand(client *c) {
     int acl_keypos;
     int acl_retval = ACLCheckCommandPerm(c,&acl_keypos);
     if (acl_retval != ACL_OK) {
-        addACLLogEntry(c,acl_retval,acl_keypos);
+        addACLLogEntry(c,acl_retval,acl_keypos,NULL);
         flagTransaction(c);
         if (acl_retval == ACL_DENIED_CMD)
             addReplyErrorFormat(c,
diff --git a/src/server.h b/src/server.h
index bd4aed192..637ceec1e 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1820,6 +1820,7 @@ void ACLInit(void);
 #define ACL_OK 0
 #define ACL_DENIED_CMD 1
 #define ACL_DENIED_KEY 2
+#define ACL_DENIED_AUTH 3 /* Only used for ACL LOG entries. */
 int ACLCheckUserCredentials(robj *username, robj *password);
 int ACLAuthenticateUser(client *c, robj *username, robj *password);
 unsigned long ACLGetCommandID(const char *cmdname);
@@ -1836,7 +1837,7 @@ void ACLLoadUsersAtStartup(void);
 void addReplyCommandCategories(client *c, struct redisCommand *cmd);
 user *ACLCreateUnlinkedUser();
 void ACLFreeUserAndKillClients(user *u);
-void addACLLogEntry(client *c, int reason, int keypos);
+void addACLLogEntry(client *c, int reason, int keypos, sds username);
 
 /* Sorted sets data type */
 

From 64a73e9293154482977cd530a2adc05f1fcc92f6 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 4 Feb 2020 12:58:48 +0100
Subject: [PATCH 0070/1098] ACL LOG: test for AUTH reason.

---
 tests/unit/acl.tcl | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl
index bd909d36c..0e6d5c66a 100644
--- a/tests/unit/acl.tcl
+++ b/tests/unit/acl.tcl
@@ -228,4 +228,13 @@ start_server {tags {"acl"}} {
         assert {[llength [r ACL LOG]] > 1}
         assert {[llength [r ACL LOG 2]] == 2}
     }
+
+    test {ACL LOG can log failed auth attempts} {
+        catch {r AUTH antirez wrong-password}
+        set entry [lindex [r ACL LOG] 0]
+        assert {[dict get $entry context] eq {toplevel}}
+        assert {[dict get $entry reason] eq {auth}}
+        assert {[dict get $entry object] eq {AUTH}}
+        assert {[dict get $entry username] eq {antirez}}
+    }
 }

From 90fae58b49cbc0bf0be76fe889952a81f4c3aed1 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 4 Feb 2020 13:19:40 +0100
Subject: [PATCH 0071/1098] ACL LOG: make max log entries configurable.

---
 src/acl.c          |  6 ++++++
 src/config.c       |  1 +
 src/server.h       |  1 +
 tests/unit/acl.tcl | 11 +++++++++++
 4 files changed, 19 insertions(+)

diff --git a/src/acl.c b/src/acl.c
index 97c00d4f8..fa57e210c 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1576,6 +1576,12 @@ void addACLLogEntry(client *c, int reason, int keypos, sds username) {
         /* Add it to our list of entires. We'll have to trim the list
          * to its maximum size. */
         listAddNodeHead(ACLLog, le);
+        while(listLength(ACLLog) > server.acllog_max_len) {
+            listNode *ln = listLast(ACLLog);
+            ACLLogEntry *le = listNodeValue(ln);
+            ACLFreeLogEntry(le);
+            listDelNode(ACLLog,ln);
+        }
     }
 }
 
diff --git a/src/config.c b/src/config.c
index 0526de84d..68a9b0c0d 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2233,6 +2233,7 @@ standardConfig configs[] = {
     /* Unsigned Long configs */
     createULongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, 1000, INTEGER_CONFIG, NULL, NULL), /* Default: keys with more than 1000 fields will be processed separately */
     createULongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, 128, INTEGER_CONFIG, NULL, NULL),
+    createULongConfig("acllog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.acllog_max_len, 128, INTEGER_CONFIG, NULL, NULL),
 
     /* Long Long configs */
     createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, 5000, INTEGER_CONFIG, NULL, NULL),/* milliseconds */
diff --git a/src/server.h b/src/server.h
index 637ceec1e..f2040436c 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1385,6 +1385,7 @@ struct redisServer {
     dict *latency_events;
     /* ACLs */
     char *acl_filename;     /* ACL Users file. NULL if not configured. */
+    unsigned long acllog_max_len; /* Maximum length of the ACL LOG list. */
     /* Assert & bug reporting */
     const char *assert_failed;
     const char *assert_file;
diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl
index 0e6d5c66a..fc1664a75 100644
--- a/tests/unit/acl.tcl
+++ b/tests/unit/acl.tcl
@@ -237,4 +237,15 @@ start_server {tags {"acl"}} {
         assert {[dict get $entry object] eq {AUTH}}
         assert {[dict get $entry username] eq {antirez}}
     }
+
+    test {ACL LOG entries are limited to a maximum amount} {
+        r ACL LOG RESET
+        r CONFIG SET acllog-max-len 5
+        r AUTH antirez foo
+        for {set j 0} {$j < 10} {incr j} {
+            catch {r SET obj:$j 123}
+        }
+        r AUTH default ""
+        assert {[llength [r ACL LOG]] == 5}
+    }
 }

From 1c7a2269d6067de23b0f5d765fdf3a44a3edd4a6 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Tue, 4 Feb 2020 19:28:09 +0530
Subject: [PATCH 0072/1098] Add RM_CreateStringFromDouble

---
 src/module.c      | 12 ++++++++++++
 src/redismodule.h |  2 ++
 2 files changed, 14 insertions(+)

diff --git a/src/module.c b/src/module.c
index d2b267be2..91437c6a6 100644
--- a/src/module.c
+++ b/src/module.c
@@ -1037,6 +1037,17 @@ RedisModuleString *RM_CreateStringFromLongLong(RedisModuleCtx *ctx, long long ll
     return RM_CreateString(ctx,buf,len);
 }
 
+/* Like RedisModule_CreatString(), but creates a string starting from a double
+ * integer instead of taking a buffer and its length.
+ *
+ * The returned string must be released with RedisModule_FreeString() or by
+ * enabling automatic memory management. */
+RedisModuleString *RM_CreateStringFromDouble(RedisModuleCtx *ctx, double d) {
+    char buf[128];
+    size_t len = d2string(buf,sizeof(buf),d);
+    return RM_CreateString(ctx,buf,len);
+}
+
 /* Like RedisModule_CreatString(), but creates a string starting from a long
  * double.
  *
@@ -7656,6 +7667,7 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(CreateStringFromCallReply);
     REGISTER_API(CreateString);
     REGISTER_API(CreateStringFromLongLong);
+    REGISTER_API(CreateStringFromDouble);
     REGISTER_API(CreateStringFromLongDouble);
     REGISTER_API(CreateStringFromString);
     REGISTER_API(CreateStringPrintf);
diff --git a/src/redismodule.h b/src/redismodule.h
index 637078f2b..95dd80e8e 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -467,6 +467,7 @@ size_t REDISMODULE_API_FUNC(RedisModule_CallReplyLength)(RedisModuleCallReply *r
 RedisModuleCallReply *REDISMODULE_API_FUNC(RedisModule_CallReplyArrayElement)(RedisModuleCallReply *reply, size_t idx);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateString)(RedisModuleCtx *ctx, const char *ptr, size_t len);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongLong)(RedisModuleCtx *ctx, long long ll);
+RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromDouble)(RedisModuleCtx *ctx, double d);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongDouble)(RedisModuleCtx *ctx, long double ld, int humanfriendly);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromString)(RedisModuleCtx *ctx, const RedisModuleString *str);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...);
@@ -726,6 +727,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(CreateStringFromCallReply);
     REDISMODULE_GET_API(CreateString);
     REDISMODULE_GET_API(CreateStringFromLongLong);
+    REDISMODULE_GET_API(CreateStringFromDouble);
     REDISMODULE_GET_API(CreateStringFromLongDouble);
     REDISMODULE_GET_API(CreateStringFromString);
     REDISMODULE_GET_API(CreateStringPrintf);

From 9ac6cb9ce4095efc61cc1fd3786f0fbe01cfef37 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 5 Feb 2020 09:42:49 +0200
Subject: [PATCH 0073/1098] memoryGetKeys helper function so that ACL can limit
 access to keys for MEMORY command

---
 src/db.c     | 16 ++++++++++++++++
 src/server.c |  2 +-
 src/server.h |  1 +
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/src/db.c b/src/db.c
index ba7be2725..88342ac4d 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1522,6 +1522,22 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk
     return keys;
 }
 
+/* Helper function to extract keys from memory command.
+ * MEMORY USAGE <key> */
+int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
+    int *keys;
+    UNUSED(cmd);
+
+    if (argc >= 3 && !strcasecmp(argv[1]->ptr,"usage")) {
+        keys = zmalloc(sizeof(int) * 1);
+        keys[0] = 2;
+        *numkeys = 1;
+        return keys;
+    }
+    *numkeys = 0;
+    return NULL;
+}
+
 /* XREAD [BLOCK <milliseconds>] [COUNT <count>] [GROUP <groupname> <ttl>]
  *       STREAMS key_1 key_2 ... key_N ID_1 ID_2 ... ID_N */
 int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
diff --git a/src/server.c b/src/server.c
index 2b226f568..f3b7c37c5 100644
--- a/src/server.c
+++ b/src/server.c
@@ -817,7 +817,7 @@ struct redisCommand redisCommandTable[] = {
 
     {"memory",memoryCommand,-2,
      "random read-only",
-     0,NULL,0,0,0,0,0,0},
+     0,memoryGetKeys,0,0,0,0,0,0},
 
     {"client",clientCommand,-2,
      "admin no-script random @connection",
diff --git a/src/server.h b/src/server.h
index 5d63e9b55..9a403a946 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2074,6 +2074,7 @@ int *sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
 int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
 int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
 int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
+int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
 
 /* Cluster */
 void clusterInit(void);

From ffdd0620d0d6804b5628913b96392e0ea2e47e45 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 5 Feb 2020 11:41:24 +0200
Subject: [PATCH 0074/1098] config.c verbose error replies for CONFIG SET, like
 config file parsing

We noticed that the error replies for the generic mechanism for enums
are very verbose for config file parsing, but not for config set
command.

instead of replicating this code, i did a small refactoring to share
code between CONFIG SET and config file parsing.

and also renamed the enum group functions to be consistent with the
naming of other types.
---
 src/config.c | 128 +++++++++++++--------------------------------------
 1 file changed, 31 insertions(+), 97 deletions(-)

diff --git a/src/config.c b/src/config.c
index 0526de84d..b19e78f74 100644
--- a/src/config.c
+++ b/src/config.c
@@ -190,8 +190,9 @@ typedef struct typeInterface {
     void (*init)(typeData data);
     /* Called on server start, should return 1 on success, 0 on error and should set err */
     int (*load)(typeData data, sds *argc, int argv, char **err);
-    /* Called on CONFIG SET, returns 1 on success, 0 on error */
-    int (*set)(typeData data, sds value, char **err);
+    /* Called on server startup and CONFIG SET, returns 1 on success, 0 on error
+     * and can set a verbose err string, update is true when called from CONFIG SET */
+    int (*set)(typeData data, sds value, int update, char **err);
     /* Called on CONFIG GET, required to add output to the client */
     void (*get)(client *c, typeData data);
     /* Called on CONFIG REWRITE, required to rewrite the config state */
@@ -323,7 +324,11 @@ void loadServerConfigFromString(char *config) {
             if ((!strcasecmp(argv[0],config->name) ||
                 (config->alias && !strcasecmp(argv[0],config->alias))))
             {
-                if (!config->interface.load(config->data, argv, argc, &err)) {
+                if (argc != 2) {
+                    err = "wrong number of arguments";
+                    goto loaderr;
+                }
+                if (!config->interface.set(config->data, argv[1], 0, &err)) {
                     goto loaderr;
                 }
 
@@ -599,7 +604,7 @@ void configSetCommand(client *c) {
         if(config->modifiable && (!strcasecmp(c->argv[2]->ptr,config->name) ||
             (config->alias && !strcasecmp(c->argv[2]->ptr,config->alias))))
         {
-            if (!config->interface.set(config->data,o->ptr, &errstr)) {
+            if (!config->interface.set(config->data,o->ptr,1,&errstr)) {
                 goto badfmt;
             }
             addReply(c,shared.ok);
@@ -1536,9 +1541,8 @@ static char loadbuf[LOADBUF_SIZE];
     .alias = (config_alias), \
     .modifiable = (is_modifiable),
 
-#define embedConfigInterface(initfn, loadfn, setfn, getfn, rewritefn) .interface = { \
+#define embedConfigInterface(initfn, setfn, getfn, rewritefn) .interface = { \
     .init = (initfn), \
-    .load = (loadfn), \
     .set = (setfn), \
     .get = (getfn), \
     .rewrite = (rewritefn) \
@@ -1561,30 +1565,17 @@ static void boolConfigInit(typeData data) {
     *data.yesno.config = data.yesno.default_value;
 }
 
-static int boolConfigLoad(typeData data, sds *argv, int argc, char **err) {
-    int yn;
-    if (argc != 2) {
-        *err = "wrong number of arguments";
-        return 0;
-    }
-    if ((yn = yesnotoi(argv[1])) == -1) {
+static int boolConfigSet(typeData data, sds value, int update, char **err) {
+    int yn = yesnotoi(value);
+    if (yn == -1) {
         *err = "argument must be 'yes' or 'no'";
         return 0;
     }
-    if (data.yesno.is_valid_fn && !data.yesno.is_valid_fn(yn, err))
-        return 0;
-    *data.yesno.config = yn;
-    return 1;
-}
-
-static int boolConfigSet(typeData data, sds value, char **err) {
-    int yn = yesnotoi(value);
-    if (yn == -1) return 0;
     if (data.yesno.is_valid_fn && !data.yesno.is_valid_fn(yn, err))
         return 0;
     int prev = *(data.yesno.config);
     *(data.yesno.config) = yn;
-    if (data.yesno.update_fn && !data.yesno.update_fn(yn, prev, err)) {
+    if (update && data.yesno.update_fn && !data.yesno.update_fn(yn, prev, err)) {
         *(data.yesno.config) = prev;
         return 0;
     }
@@ -1601,7 +1592,7 @@ static void boolConfigRewrite(typeData data, const char *name, struct rewriteCon
 
 #define createBoolConfig(name, alias, modifiable, config_addr, default, is_valid, update) { \
     embedCommonConfig(name, alias, modifiable) \
-    embedConfigInterface(boolConfigInit, boolConfigLoad, boolConfigSet, boolConfigGet, boolConfigRewrite) \
+    embedConfigInterface(boolConfigInit, boolConfigSet, boolConfigGet, boolConfigRewrite) \
     .data.yesno = { \
         .config = &(config_addr), \
         .default_value = (default), \
@@ -1619,23 +1610,7 @@ static void stringConfigInit(typeData data) {
     }
 }
 
-static int stringConfigLoad(typeData data, sds *argv, int argc, char **err) {
-    if (argc != 2) {
-        *err = "wrong number of arguments";
-        return 0;
-    }
-    if (data.string.is_valid_fn && !data.string.is_valid_fn(argv[1], err))
-        return 0;
-    zfree(*data.string.config);
-    if (data.string.convert_empty_to_null) {
-        *data.string.config = argv[1][0] ? zstrdup(argv[1]) : NULL;
-    } else {
-        *data.string.config = zstrdup(argv[1]);
-    }
-    return 1;
-}
-
-static int stringConfigSet(typeData data, sds value, char **err) {
+static int stringConfigSet(typeData data, sds value, int update, char **err) {
     if (data.string.is_valid_fn && !data.string.is_valid_fn(value, err))
         return 0;
     char *prev = *data.string.config;
@@ -1644,7 +1619,7 @@ static int stringConfigSet(typeData data, sds value, char **err) {
     } else {
         *data.string.config = zstrdup(value);
     }
-    if (data.string.update_fn && !data.string.update_fn(*data.string.config, prev, err)) {
+    if (update && data.string.update_fn && !data.string.update_fn(*data.string.config, prev, err)) {
         zfree(*data.string.config);
         *data.string.config = prev;
         return 0;
@@ -1666,7 +1641,7 @@ static void stringConfigRewrite(typeData data, const char *name, struct rewriteC
 
 #define createStringConfig(name, alias, modifiable, empty_to_null, config_addr, default, is_valid, update) { \
     embedCommonConfig(name, alias, modifiable) \
-    embedConfigInterface(stringConfigInit, stringConfigLoad, stringConfigSet, stringConfigGet, stringConfigRewrite) \
+    embedConfigInterface(stringConfigInit, stringConfigSet, stringConfigGet, stringConfigRewrite) \
     .data.string = { \
         .config = &(config_addr), \
         .default_value = (default), \
@@ -1677,17 +1652,12 @@ static void stringConfigRewrite(typeData data, const char *name, struct rewriteC
 }
 
 /* Enum configs */
-static void configEnumInit(typeData data) {
+static void enumConfigInit(typeData data) {
     *data.enumd.config = data.enumd.default_value;
 }
 
-static int configEnumLoad(typeData data, sds *argv, int argc, char **err) {
-    if (argc != 2) {
-        *err = "wrong number of arguments";
-        return 0;
-    }
-
-    int enumval = configEnumGetValue(data.enumd.enum_value, argv[1]);
+static int enumConfigSet(typeData data, sds value, int update, char **err) {
+    int enumval = configEnumGetValue(data.enumd.enum_value, value);
     if (enumval == INT_MIN) {
         sds enumerr = sdsnew("argument must be one of the following: ");
         configEnum *enumNode = data.enumd.enum_value;
@@ -1707,37 +1677,28 @@ static int configEnumLoad(typeData data, sds *argv, int argc, char **err) {
         *err = loadbuf;
         return 0;
     }
-    if (data.enumd.is_valid_fn && !data.enumd.is_valid_fn(enumval, err))
-        return 0;
-    *(data.enumd.config) = enumval;
-    return 1;
-}
-
-static int configEnumSet(typeData data, sds value, char **err) {
-    int enumval = configEnumGetValue(data.enumd.enum_value, value);
-    if (enumval == INT_MIN) return 0;
     if (data.enumd.is_valid_fn && !data.enumd.is_valid_fn(enumval, err))
         return 0;
     int prev = *(data.enumd.config);
     *(data.enumd.config) = enumval;
-    if (data.enumd.update_fn && !data.enumd.update_fn(enumval, prev, err)) {
+    if (update && data.enumd.update_fn && !data.enumd.update_fn(enumval, prev, err)) {
         *(data.enumd.config) = prev;
         return 0;
     }
     return 1;
 }
 
-static void configEnumGet(client *c, typeData data) {
+static void enumConfigGet(client *c, typeData data) {
     addReplyBulkCString(c, configEnumGetNameOrUnknown(data.enumd.enum_value,*data.enumd.config));
 }
 
-static void configEnumRewrite(typeData data, const char *name, struct rewriteConfigState *state) {
+static void enumConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) {
     rewriteConfigEnumOption(state, name,*(data.enumd.config), data.enumd.enum_value, data.enumd.default_value);
 }
 
 #define createEnumConfig(name, alias, modifiable, enum, config_addr, default, is_valid, update) { \
     embedCommonConfig(name, alias, modifiable) \
-    embedConfigInterface(configEnumInit, configEnumLoad, configEnumSet, configEnumGet, configEnumRewrite) \
+    embedConfigInterface(enumConfigInit, enumConfigSet, enumConfigGet, enumConfigRewrite) \
     .data.enumd = { \
         .config = &(config_addr), \
         .default_value = (default), \
@@ -1832,49 +1793,22 @@ static int numericBoundaryCheck(typeData data, long long ll, char **err) {
     return 1;
 }
 
-static int numericConfigLoad(typeData data, sds *argv, int argc, char **err) {
-    long long ll;
-
-    if (argc != 2) {
-        *err = "wrong number of arguments";
-        return 0;
-    }
-
+static int numericConfigSet(typeData data, sds value, int update, char **err) {
+    long long ll, prev = 0;
     if (data.numeric.is_memory) {
         int memerr;
-        ll = memtoll(argv[1], &memerr);
+        ll = memtoll(value, &memerr);
         if (memerr || ll < 0) {
             *err = "argument must be a memory value";
             return 0;
         }
     } else {
-        if (!string2ll(argv[1], sdslen(argv[1]),&ll)) {
+        if (!string2ll(value, sdslen(value),&ll)) {
             *err = "argument couldn't be parsed into an integer" ;
             return 0;
         }
     }
 
-    if (!numericBoundaryCheck(data, ll, err))
-        return 0;
-
-    if (data.numeric.is_valid_fn && !data.numeric.is_valid_fn(ll, err))
-        return 0;
-
-    SET_NUMERIC_TYPE(ll)
-
-    return 1;
-}
-
-static int numericConfigSet(typeData data, sds value, char **err) {
-    long long ll, prev = 0;
-    if (data.numeric.is_memory) {
-        int memerr;
-        ll = memtoll(value, &memerr);
-        if (memerr || ll < 0) return 0;
-    } else {
-        if (!string2ll(value, sdslen(value),&ll)) return 0;
-    }
-
     if (!numericBoundaryCheck(data, ll, err))
         return 0;
 
@@ -1884,7 +1818,7 @@ static int numericConfigSet(typeData data, sds value, char **err) {
     GET_NUMERIC_TYPE(prev)
     SET_NUMERIC_TYPE(ll)
 
-    if (data.numeric.update_fn && !data.numeric.update_fn(ll, prev, err)) {
+    if (update && data.numeric.update_fn && !data.numeric.update_fn(ll, prev, err)) {
         SET_NUMERIC_TYPE(prev)
         return 0;
     }
@@ -1918,7 +1852,7 @@ static void numericConfigRewrite(typeData data, const char *name, struct rewrite
 
 #define embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) { \
     embedCommonConfig(name, alias, modifiable) \
-    embedConfigInterface(numericConfigInit, numericConfigLoad, numericConfigSet, numericConfigGet, numericConfigRewrite) \
+    embedConfigInterface(numericConfigInit, numericConfigSet, numericConfigGet, numericConfigRewrite) \
     .data.numeric = { \
         .lower_bound = (lower), \
         .upper_bound = (upper), \

From 774d8cd721055b768dbffbf5c6b2fa9d6310126e Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 5 Feb 2020 18:06:33 +0200
Subject: [PATCH 0075/1098] Optimize temporary memory allocations for
 getKeysFromCommand mechanism

now that we may use it more often (ACL), these excessive calls to malloc
and free can become an overhead.
---
 src/db.c | 41 +++++++++++++++++++++++++++++++----------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/src/db.c b/src/db.c
index 88342ac4d..caabeaabb 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1292,6 +1292,8 @@ int expireIfNeeded(redisDb *db, robj *key) {
 /* -----------------------------------------------------------------------------
  * API to get key arguments from commands
  * ---------------------------------------------------------------------------*/
+#define MAX_KEYS_BUFFER 65536
+static int getKeysTempBuffer[MAX_KEYS_BUFFER];
 
 /* The base case is to use the keys position as given in the command table
  * (firstkey, lastkey, step). */
@@ -1306,7 +1308,12 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in
 
     last = cmd->lastkey;
     if (last < 0) last = argc+last;
-    keys = zmalloc(sizeof(int)*((last - cmd->firstkey)+1));
+
+    int count = ((last - cmd->firstkey)+1);
+    keys = getKeysTempBuffer;
+    if (count > MAX_KEYS_BUFFER)
+        keys = zmalloc(sizeof(int)*count);
+
     for (j = cmd->firstkey; j <= last; j += cmd->keystep) {
         if (j >= argc) {
             /* Modules commands, and standard commands with a not fixed number
@@ -1316,7 +1323,7 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in
              * return no keys and expect the command implementation to report
              * an arity or syntax error. */
             if (cmd->flags & CMD_MODULE || cmd->arity < 0) {
-                zfree(keys);
+                getKeysFreeResult(keys);
                 *numkeys = 0;
                 return NULL;
             } else {
@@ -1352,7 +1359,8 @@ int *getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, int *nu
 
 /* Free the result of getKeysFromCommand. */
 void getKeysFreeResult(int *result) {
-    zfree(result);
+    if (result != getKeysTempBuffer)
+        zfree(result);
 }
 
 /* Helper function to extract keys from following commands:
@@ -1373,7 +1381,9 @@ int *zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *nu
     /* Keys in z{union,inter}store come from two places:
      * argv[1] = storage key,
      * argv[3...n] = keys to intersect */
-    keys = zmalloc(sizeof(int)*(num+1));
+    keys = getKeysTempBuffer;
+    if (num+1>MAX_KEYS_BUFFER)
+        keys = zmalloc(sizeof(int)*(num+1));
 
     /* Add all key positions for argv[3...n] to keys[] */
     for (i = 0; i < num; i++) keys[i] = 3+i;
@@ -1399,7 +1409,10 @@ int *evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys)
         return NULL;
     }
 
-    keys = zmalloc(sizeof(int)*num);
+    keys = getKeysTempBuffer;
+    if (num>MAX_KEYS_BUFFER)
+        keys = zmalloc(sizeof(int)*num);
+
     *numkeys = num;
 
     /* Add all key positions for argv[3...n] to keys[] */
@@ -1420,7 +1433,7 @@ int *sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys)
     UNUSED(cmd);
 
     num = 0;
-    keys = zmalloc(sizeof(int)*2); /* Alloc 2 places for the worst case. */
+    keys = getKeysTempBuffer; /* Alloc 2 places for the worst case. */
 
     keys[num++] = 1; /* <sort-key> is always present. */
 
@@ -1478,7 +1491,10 @@ int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkey
         }
     }
 
-    keys = zmalloc(sizeof(int)*num);
+    keys = getKeysTempBuffer;
+    if (num>MAX_KEYS_BUFFER)
+        keys = zmalloc(sizeof(int)*num);
+
     for (i = 0; i < num; i++) keys[i] = first+i;
     *numkeys = num;
     return keys;
@@ -1511,7 +1527,9 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk
      * argv[1] = key,
      * argv[5...n] = stored key if present
      */
-    keys = zmalloc(sizeof(int) * num);
+    keys = getKeysTempBuffer;
+    if (num>MAX_KEYS_BUFFER)
+        keys = zmalloc(sizeof(int) * num);
 
     /* Add all key positions to keys[] */
     keys[0] = 1;
@@ -1529,7 +1547,7 @@ int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys
     UNUSED(cmd);
 
     if (argc >= 3 && !strcasecmp(argv[1]->ptr,"usage")) {
-        keys = zmalloc(sizeof(int) * 1);
+        keys = getKeysTempBuffer;
         keys[0] = 2;
         *numkeys = 1;
         return keys;
@@ -1576,7 +1594,10 @@ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys)
     num /= 2; /* We have half the keys as there are arguments because
                  there are also the IDs, one per key. */
 
-    keys = zmalloc(sizeof(int) * num);
+    keys = getKeysTempBuffer;
+    if (num>MAX_KEYS_BUFFER)
+        keys = zmalloc(sizeof(int) * num);
+
     for (i = streams_pos+1; i < argc-num; i++) keys[i-streams_pos-1] = i;
     *numkeys = num;
     return keys;

From 3795aaf42aadf46b415edaa801383cd81085eb91 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 5 Feb 2020 18:15:38 +0200
Subject: [PATCH 0076/1098] update RM_SignalModifiedKey doc comment

---
 src/module.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/module.c b/src/module.c
index 914c50df3..97edb97b3 100644
--- a/src/module.c
+++ b/src/module.c
@@ -890,7 +890,8 @@ void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) {
     ctx->module->options = options;
 }
 
-/* Signals that the key is modified from user's perspective (i.e. invalidate WATCH). */
+/* Signals that the key is modified from user's perspective (i.e. invalidate WATCH
+ * and client side caching). */
 int RM_SignalModifiedKey(RedisModuleCtx *ctx, RedisModuleString *keyname) {
     signalModifiedKey(ctx->client->db,keyname);
     return REDISMODULE_OK;

From 1e02d599dc2a0643fcf82af42047adf07c78fe41 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 5 Feb 2020 18:30:12 +0200
Subject: [PATCH 0077/1098] TLS: Some redis.conf clarifications.

---
 redis.conf | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/redis.conf b/redis.conf
index 07005cffe..3c7336747 100644
--- a/redis.conf
+++ b/redis.conf
@@ -155,23 +155,22 @@ tcp-keepalive 300
 # tls-ca-cert-file ca.crt
 # tls-ca-cert-dir /etc/ssl/certs
 
-# If TLS/SSL clients are required to authenticate using a client side
-# certificate, use this directive.
+# By default, clients (including replica servers) on a TLS port are required
+# to authenticate using valid client side certificates.
 #
-# Note: this applies to all incoming clients, including replicas.
+# It is possible to disable authentication using this directive.
 #
-# tls-auth-clients yes
+# tls-auth-clients no
 
-# If TLS/SSL should be used when connecting as a replica to a master, enable
-# this configuration directive:
+# By default, a Redis replica does not attempt to establish a TLS connection
+# with its master.
+#
+# Use the following directive to enable TLS on replication links.
 #
 # tls-replication yes
 
-# If TLS/SSL should be used for the Redis Cluster bus, enable this configuration
-# directive.
-#
-# NOTE: If TLS/SSL is enabled for Cluster Bus, mutual authentication is always
-# enforced.
+# By default, the Redis Cluster bus uses a plain TCP connection. To enable
+# TLS for the bus protocol, use the following directive:
 #
 # tls-cluster yes
 

From fe7e8dc955ef6c5f3b843f87763f62f6eb84bd4b Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 5 Feb 2020 19:47:09 +0200
Subject: [PATCH 0078/1098] Add handling of short read of module id in rdb

---
 src/rdb.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/rdb.c b/src/rdb.c
index 27e1b3135..61265433d 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1871,7 +1871,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, robj *key) {
         }
     } else if (rdbtype == RDB_TYPE_MODULE || rdbtype == RDB_TYPE_MODULE_2) {
         uint64_t moduleid = rdbLoadLen(rdb,NULL);
-        if (rioGetReadError(rdb)) return NULL;
+        if (rioGetReadError(rdb)) {
+            rdbReportReadError("Short read module id");
+            return NULL;
+        }
         moduleType *mt = moduleTypeLookupModuleByID(moduleid);
         char name[10];
 

From bb3d45a38683fc97c0b9b06ff7725fa1eca5d80c Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 5 Feb 2020 21:13:21 +0200
Subject: [PATCH 0079/1098] TLS: Update documentation.

---
 README.md | 18 ++++++++++++++++++
 TLS.md    | 45 ++++++++++++++-------------------------------
 2 files changed, 32 insertions(+), 31 deletions(-)

diff --git a/README.md b/README.md
index 3442659e6..c08013416 100644
--- a/README.md
+++ b/README.md
@@ -35,6 +35,11 @@ It is as simple as:
 
     % make
 
+To build with TLS support, you'll need OpenSSL development libraries (e.g.
+libssl-dev on Debian/Ubuntu) and run:
+
+    % make BUILD_TLS=yes
+
 You can run a 32 bit Redis binary using:
 
     % make 32bit
@@ -43,6 +48,13 @@ After building Redis, it is a good idea to test it using:
 
     % make test
 
+If TLS is built, running the tests with TLS enabled (you will need `tcl-tls`
+installed):
+
+    % ./utils/gen-test-certs.sh
+    % ./runtest --tls
+
+
 Fixing build problems with dependencies or cached build options
 ---------
 
@@ -125,6 +137,12 @@ as options using the command line. Examples:
 All the options in redis.conf are also supported as options using the command
 line, with exactly the same name.
 
+Running Redis with TLS:
+------------------
+
+Please consult the [TLS.md](TLS.md) file for more information on
+how to use Redis with TLS.
+
 Playing with Redis
 ------------------
 
diff --git a/TLS.md b/TLS.md
index 76fe0be2e..e480c1e9d 100644
--- a/TLS.md
+++ b/TLS.md
@@ -1,8 +1,5 @@
-TLS Support -- Work In Progress
-===============================
-
-This is a brief note to capture current thoughts/ideas and track pending action
-items.
+TLS Support
+===========
 
 Getting Started
 ---------------
@@ -69,37 +66,23 @@ probably not be so hard. For cluster keys migration it might be more difficult,
 but there are probably other good reasons to improve that part anyway.
 
 To-Do List
-==========
+----------
 
-Additional TLS Features
------------------------
+- [ ] Add session caching support. Check if/how it's handled by clients to
+  assess how useful/important it is.
+- [ ] redis-benchmark support. The current implementation is a mix of using
+  hiredis for parsing and basic networking (establishing connections), but
+  directly manipulating sockets for most actions. This will need to be cleaned
+  up for proper TLS support. The best approach is probably to migrate to hiredis
+  async mode.
+- [ ] redis-cli `--slave` and `--rdb` support.
 
-1. Add metrics to INFO?
-2. Add session caching support. Check if/how it's handled by clients to assess
-   how useful/important it is.
-
-redis-benchmark
----------------
-
-The current implementation is a mix of using hiredis for parsing and basic
-networking (establishing connections), but directly manipulating sockets for
-most actions.
-
-This will need to be cleaned up for proper TLS support. The best approach is
-probably to migrate to hiredis async mode.
-
-redis-cli
----------
-
-1. Add support for TLS in --slave and --rdb modes.
-
-Others
-------
+Multi-port
+----------
 
 Consider the implications of allowing TLS to be configured on a separate port,
-making Redis listening on multiple ports.
+making Redis listening on multiple ports:
 
-This impacts many things, like
 1. Startup banner port notification
 2. Proctitle
 3. How slaves announce themselves

From ac2c96f5b1714b36f215fc9caec87e746a804041 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 6 Feb 2020 08:53:23 +0200
Subject: [PATCH 0080/1098] A few non-data commands that should be allowed
 while loading or stale

SELECT, and HELLO are commands that may be executed by the client
as soon as it connects, there's no reason to block them, preventing the
client from doing the rest of his sequence (which might just be INFO or
CONFIG, etc).

MONITOR, DEBUG, SLOWLOG, TIME, LASTSAVE are all non-data accessing
commands, which there's no reason to block.
---
 src/server.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/server.c b/src/server.c
index f3b7c37c5..23e72ce19 100644
--- a/src/server.c
+++ b/src/server.c
@@ -579,7 +579,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"select",selectCommand,2,
-     "ok-loading fast @keyspace",
+     "ok-loading fast ok-stale @keyspace",
      0,NULL,0,0,0,0,0,0},
 
     {"swapdb",swapdbCommand,3,
@@ -660,7 +660,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"lastsave",lastsaveCommand,1,
-     "read-only random fast @admin @dangerous",
+     "read-only random fast ok-loading ok-stale @admin @dangerous",
      0,NULL,0,0,0,0,0,0},
 
     {"type",typeCommand,2,
@@ -708,7 +708,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"monitor",monitorCommand,1,
-     "admin no-script",
+     "admin no-script ok-loading ok-stale",
      0,NULL,0,0,0,0,0,0},
 
     {"ttl",ttlCommand,2,
@@ -740,7 +740,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"debug",debugCommand,-2,
-     "admin no-script",
+     "admin no-script ok-loading ok-stale",
      0,NULL,0,0,0,0,0,0},
 
     {"config",configCommand,-2,
@@ -820,11 +820,11 @@ struct redisCommand redisCommandTable[] = {
      0,memoryGetKeys,0,0,0,0,0,0},
 
     {"client",clientCommand,-2,
-     "admin no-script random @connection",
+     "admin no-script random ok-loading ok-stale @connection",
      0,NULL,0,0,0,0,0,0},
 
     {"hello",helloCommand,-2,
-     "no-auth no-script fast no-monitor no-slowlog @connection",
+     "no-auth no-script fast no-monitor ok-loading ok-stale no-slowlog @connection",
      0,NULL,0,0,0,0,0,0},
 
     /* EVAL can modify the dataset, however it is not flagged as a write
@@ -838,7 +838,7 @@ struct redisCommand redisCommandTable[] = {
      0,evalGetKeys,0,0,0,0,0,0},
 
     {"slowlog",slowlogCommand,-2,
-     "admin random",
+     "admin random ok-loading ok-stale",
      0,NULL,0,0,0,0,0,0},
 
     {"script",scriptCommand,-2,
@@ -846,7 +846,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"time",timeCommand,1,
-     "read-only random fast",
+     "read-only random fast ok-loading ok-stale",
      0,NULL,0,0,0,0,0,0},
 
     {"bitop",bitopCommand,-4,

From 7e1d5954e59ab99c572f2dccccf6843c9aad9127 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 6 Feb 2020 09:15:31 +0200
Subject: [PATCH 0081/1098] Memory leak when bind config is provided twice

---
 src/config.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/config.c b/src/config.c
index b19e78f74..fe26b3884 100644
--- a/src/config.c
+++ b/src/config.c
@@ -349,6 +349,10 @@ void loadServerConfigFromString(char *config) {
             if (addresses > CONFIG_BINDADDR_MAX) {
                 err = "Too many bind addresses specified"; goto loaderr;
             }
+            /* Free old bind addresses */
+            for (j = 0; j < server.bindaddr_count; j++) {
+                zfree(server.bindaddr[j]);
+            }
             for (j = 0; j < addresses; j++)
                 server.bindaddr[j] = zstrdup(argv[j+1]);
             server.bindaddr_count = addresses;

From a17bddf2a14a1bd85a3606d2023fb9bb92004770 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 6 Feb 2020 09:23:22 +0200
Subject: [PATCH 0082/1098] fix maxmemory config warning

the warning condition was if usage > limit (saying it'll cause eviction
or oom), but in fact the eviction and oom depends on used minus slave
buffers.

other than fixing the condition, i add info about the current usage and
limit, which may be useful when looking at the log.
---
 src/config.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/config.c b/src/config.c
index b19e78f74..173a90909 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1995,8 +1995,9 @@ static int updateMaxmemory(long long val, long long prev, char **err) {
     UNUSED(prev);
     UNUSED(err);
     if (val) {
-        if ((unsigned long long)val < zmalloc_used_memory()) {
-            serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy.");
+        size_t used = zmalloc_used_memory()-freeMemoryGetNotCountedMemory();
+        if ((unsigned long long)val < used) {
+            serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET (%llu) is smaller than the current memory usage (%zu). This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy.", server.maxmemory, used);
         }
         freeMemoryIfNeededAndSafe();
     }

From d454d5a4f516991c0f1e21a285c71f38a113e297 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 6 Feb 2020 09:33:20 +0200
Subject: [PATCH 0083/1098] Fix client flags to be int64 in module.c

currently there's no bug since the flags these functions handle are
always lower than 32bit, but still better fix the type to prevent future
bugs.
---
 src/module.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/module.c b/src/module.c
index 914c50df3..a2bc869f9 100644
--- a/src/module.c
+++ b/src/module.c
@@ -714,9 +714,9 @@ void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) {
  * flags into the command flags used by the Redis core.
  *
  * It returns the set of flags, or -1 if unknown flags are found. */
-int commandFlagsFromString(char *s) {
+int64_t commandFlagsFromString(char *s) {
     int count, j;
-    int flags = 0;
+    int64_t flags = 0;
     sds *tokens = sdssplitlen(s,strlen(s)," ",1,&count);
     for (j = 0; j < count; j++) {
         char *t = tokens[j];
@@ -798,7 +798,7 @@ int commandFlagsFromString(char *s) {
  *                     to authenticate a client. 
  */
 int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) {
-    int flags = strflags ? commandFlagsFromString((char*)strflags) : 0;
+    int64_t flags = strflags ? commandFlagsFromString((char*)strflags) : 0;
     if (flags == -1) return REDISMODULE_ERR;
     if ((flags & CMD_MODULE_NO_CLUSTER) && server.cluster_enabled)
         return REDISMODULE_ERR;

From 85cc696f50064327e81a41952ffb2be830048701 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 6 Feb 2020 09:37:04 +0200
Subject: [PATCH 0084/1098] moduleRDBLoadError, add key name, and use panic
 rather than exit

using panic rather than exit means you get s stack trace of the code
path that experianced the error, and possibly other info.
---
 src/module.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/module.c b/src/module.c
index 914c50df3..16b191535 100644
--- a/src/module.c
+++ b/src/module.c
@@ -3649,14 +3649,15 @@ void moduleRDBLoadError(RedisModuleIO *io) {
         io->error = 1;
         return;
     }
-    serverLog(LL_WARNING,
+    serverPanic(
         "Error loading data from RDB (short read or EOF). "
         "Read performed by module '%s' about type '%s' "
-        "after reading '%llu' bytes of a value.",
+        "after reading '%llu' bytes of a value "
+        "for key named: '%s'.",
         io->type->module->name,
         io->type->name,
-        (unsigned long long)io->bytes);
-    exit(1);
+        (unsigned long long)io->bytes,
+        io->key? (char*)io->key->ptr: "(null)");
 }
 
 /* Returns 0 if there's at least one registered data type that did not declare

From 485d5d4a18441feff626e61e31e480c3054fe877 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 6 Feb 2020 09:41:45 +0200
Subject: [PATCH 0085/1098] reduce repeated calls to use_diskless_load

this function possibly iterates on the module list
---
 src/replication.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index b7e77184a..7aebd620a 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1354,7 +1354,7 @@ void disklessLoadRestoreBackups(redisDb *backup, int restore, int empty_db_flags
 void readSyncBulkPayload(connection *conn) {
     char buf[4096];
     ssize_t nread, readlen, nwritten;
-    int use_diskless_load;
+    int use_diskless_load = useDisklessLoad();
     redisDb *diskless_load_backup = NULL;
     int empty_db_flags = server.repl_slave_lazy_flush ? EMPTYDB_ASYNC :
                                                         EMPTYDB_NO_FLAGS;
@@ -1411,19 +1411,18 @@ void readSyncBulkPayload(connection *conn) {
             server.repl_transfer_size = 0;
             serverLog(LL_NOTICE,
                 "MASTER <-> REPLICA sync: receiving streamed RDB from master with EOF %s",
-                useDisklessLoad()? "to parser":"to disk");
+                use_diskless_load? "to parser":"to disk");
         } else {
             usemark = 0;
             server.repl_transfer_size = strtol(buf+1,NULL,10);
             serverLog(LL_NOTICE,
                 "MASTER <-> REPLICA sync: receiving %lld bytes from master %s",
                 (long long) server.repl_transfer_size,
-                useDisklessLoad()? "to parser":"to disk");
+                use_diskless_load? "to parser":"to disk");
         }
         return;
     }
 
-    use_diskless_load = useDisklessLoad();
     if (!use_diskless_load) {
         /* Read the data from the socket, store it to a file and search
          * for the EOF. */

From 86e302f5f302da89790f03000e9f76d322b5f971 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 6 Feb 2020 10:07:17 +0200
Subject: [PATCH 0086/1098] freeClientAsync don't lock mutex if there's just
 one thread

---
 src/networking.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index a2e454d4b..1c9325c43 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1162,9 +1162,14 @@ void freeClientAsync(client *c) {
      * may access the list while Redis uses I/O threads. All the other accesses
      * are in the context of the main thread while the other threads are
      * idle. */
-    static pthread_mutex_t async_free_queue_mutex = PTHREAD_MUTEX_INITIALIZER;
     if (c->flags & CLIENT_CLOSE_ASAP || c->flags & CLIENT_LUA) return;
     c->flags |= CLIENT_CLOSE_ASAP;
+    if (server.io_threads_num == 1) {
+        /* no need to bother with locking if there's just one thread (the main thread) */
+        listAddNodeTail(server.clients_to_close,c);
+        return;
+    }
+    static pthread_mutex_t async_free_queue_mutex = PTHREAD_MUTEX_INITIALIZER;
     pthread_mutex_lock(&async_free_queue_mutex);
     listAddNodeTail(server.clients_to_close,c);
     pthread_mutex_unlock(&async_free_queue_mutex);

From aac6a4cf135aa136b339e84c23db3cfc06b4a776 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 6 Feb 2020 10:14:32 +0200
Subject: [PATCH 0087/1098] move restartAOFAfterSYNC from replicaofCommand to
 replicationUnsetMaster

replicationUnsetMaster can be called from other places, not just
replicaofCOmmand, and all of these need to restart AOF
---
 src/replication.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index b7e77184a..5499ebc57 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2399,6 +2399,10 @@ void replicationUnsetMaster(void) {
     moduleFireServerEvent(REDISMODULE_EVENT_REPLICATION_ROLE_CHANGED,
                           REDISMODULE_EVENT_REPLROLECHANGED_NOW_MASTER,
                           NULL);
+
+    /* Restart the AOF subsystem in case we shut it down during a sync when
+     * we were still a slave. */
+    if (server.aof_enabled && server.aof_state == AOF_OFF) restartAOFAfterSYNC();
 }
 
 /* This function is called when the slave lose the connection with the
@@ -2436,9 +2440,6 @@ void replicaofCommand(client *c) {
             serverLog(LL_NOTICE,"MASTER MODE enabled (user request from '%s')",
                 client);
             sdsfree(client);
-            /* Restart the AOF subsystem in case we shut it down during a sync when
-             * we were still a slave. */
-            if (server.aof_enabled && server.aof_state == AOF_OFF) restartAOFAfterSYNC();
         }
     } else {
         long port;

From 69e8ea7143061eda18c1aa1ca9a345831cf5f884 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 6 Feb 2020 10:17:34 +0200
Subject: [PATCH 0088/1098] stopAppendOnly resets aof_rewrite_scheduled

althouh in theory, users can do BGREWRITEAOF even if aof is disabled, i
suppose it is more common that the scheduled flag is set by either
startAppendOnly, of a failed initial AOFRW fork (AOF_WAIT_REWRITE)
---
 src/aof.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/aof.c b/src/aof.c
index 9eeb3f1e2..3682c4568 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -242,6 +242,7 @@ void stopAppendOnly(void) {
     server.aof_fd = -1;
     server.aof_selected_db = -1;
     server.aof_state = AOF_OFF;
+    server.aof_rewrite_scheduled = 0;
     killAppendOnlyChild();
 }
 

From 6d29c34da7e0caeeb1647f41056f964e3a1ecc68 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 6 Feb 2020 10:31:43 +0200
Subject: [PATCH 0089/1098] add SAVE subcommand to ACL HELP and top comment

---
 src/acl.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/acl.c b/src/acl.c
index 1f395bd3f..aae35226f 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1461,6 +1461,7 @@ void ACLLoadUsersAtStartup(void) {
 /* ACL -- show and modify the configuration of ACL users.
  * ACL HELP
  * ACL LOAD
+ * ACL SAVE
  * ACL LIST
  * ACL USERS
  * ACL CAT [<category>]
@@ -1658,6 +1659,7 @@ void aclCommand(client *c) {
     } else if (!strcasecmp(sub,"help")) {
         const char *help[] = {
 "LOAD                              -- Reload users from the ACL file.",
+"SAVE                              -- Save the current config to the ACL file."
 "LIST                              -- Show user details in config file format.",
 "USERS                             -- List all the registered usernames.",
 "SETUSER <username> [attribs ...]  -- Create or modify a user.",

From e33fffbde1623a83895a9aba80f2fdd25ec84330 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 6 Feb 2020 14:09:45 +0530
Subject: [PATCH 0090/1098] Fix small bugs related to replica and monitor
 ambiguity

1. server.repl_no_slaves_since can be set when a MONITOR client disconnects
2. c->repl_ack_time can be set by a newline from a MONITOR client
3. Improved comments
---
 src/networking.c | 12 +++++++-----
 src/server.c     |  2 +-
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index a2e454d4b..8fca3fc1d 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1121,7 +1121,7 @@ void freeClient(client *c) {
         /* We need to remember the time when we started to have zero
          * attached slaves, as after some time we'll free the replication
          * backlog. */
-        if (c->flags & CLIENT_SLAVE && listLength(server.slaves) == 0)
+        if (getClientType(c) == CLIENT_TYPE_SLAVE && listLength(server.slaves) == 0)
             server.repl_no_slaves_since = server.unixtime;
         refreshGoodSlavesCount();
         /* Fire the replica change modules event. */
@@ -1252,8 +1252,8 @@ int writeToClient(client *c, int handler_installed) {
          * just deliver as much data as it is possible to deliver.
          *
          * Moreover, we also send as much as possible if the client is
-         * a slave (otherwise, on high-speed traffic, the replication
-         * buffer will grow indefinitely) */
+         * a slave or a monitor (otherwise, on high-speed traffic, the
+         * replication/output buffer will grow indefinitely) */
         if (totwritten > NET_MAX_WRITES_PER_EVENT &&
             (server.maxmemory == 0 ||
              zmalloc_used_memory() < server.maxmemory) &&
@@ -1439,7 +1439,7 @@ int processInlineBuffer(client *c) {
     /* Newline from slaves can be used to refresh the last ACK time.
      * This is useful for a slave to ping back while loading a big
      * RDB file. */
-    if (querylen == 0 && c->flags & CLIENT_SLAVE)
+    if (querylen == 0 && getClientType(c) == CLIENT_TYPE_SLAVE)
         c->repl_ack_time = server.unixtime;
 
     /* Move querybuffer position to the next query in the buffer. */
@@ -2433,12 +2433,14 @@ unsigned long getClientOutputBufferMemoryUsage(client *c) {
  *
  * The function will return one of the following:
  * CLIENT_TYPE_NORMAL -> Normal client
- * CLIENT_TYPE_SLAVE  -> Slave or client executing MONITOR command
+ * CLIENT_TYPE_SLAVE  -> Slave
  * CLIENT_TYPE_PUBSUB -> Client subscribed to Pub/Sub channels
  * CLIENT_TYPE_MASTER -> The client representing our replication master.
  */
 int getClientType(client *c) {
     if (c->flags & CLIENT_MASTER) return CLIENT_TYPE_MASTER;
+    /* Even though MONITOR clients are marked as replicas, we
+     * want the expose them as normal clients. */
     if ((c->flags & CLIENT_SLAVE) && !(c->flags & CLIENT_MONITOR))
         return CLIENT_TYPE_SLAVE;
     if (c->flags & CLIENT_PUBSUB) return CLIENT_TYPE_PUBSUB;
diff --git a/src/server.c b/src/server.c
index f3b7c37c5..2b32df0db 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1498,7 +1498,7 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) {
     time_t now = now_ms/1000;
 
     if (server.maxidletime &&
-        !(c->flags & CLIENT_SLAVE) &&    /* no timeout for slaves */
+        !(c->flags & CLIENT_SLAVE) &&    /* no timeout for slaves and monitors */
         !(c->flags & CLIENT_MASTER) &&   /* no timeout for masters */
         !(c->flags & CLIENT_BLOCKED) &&  /* no timeout for BLPOP */
         !(c->flags & CLIENT_PUBSUB) &&   /* no timeout for Pub/Sub clients */

From 31ffbf11332fbc6afd69ca94f77ea66bbfb918e3 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 6 Feb 2020 10:40:29 +0200
Subject: [PATCH 0091/1098] DEBUG HELP - add PROTOCOL

---
 src/debug.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/debug.c b/src/debug.c
index a2d37337d..b910d2d2d 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -355,6 +355,7 @@ void debugCommand(client *c) {
 "CRASH-AND-RECOVER <milliseconds> -- Hard crash and restart after <milliseconds> delay.",
 "DIGEST -- Output a hex signature representing the current DB content.",
 "DIGEST-VALUE <key-1> ... <key-N>-- Output a hex signature of the values of all the specified keys.",
+"DEBUG PROTOCOL [string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false]",
 "ERROR <string> -- Return a Redis protocol error with <string> as message. Useful for clients unit tests to simulate Redis errors.",
 "LOG <message> -- write message to the server log.",
 "HTSTATS <dbid> -- Return hash table statistics of the specified Redis database.",
@@ -586,7 +587,7 @@ NULL
         }
     } else if (!strcasecmp(c->argv[1]->ptr,"protocol") && c->argc == 3) {
         /* DEBUG PROTOCOL [string|integer|double|bignum|null|array|set|map|
-         *                 attrib|push|verbatim|true|false|state|err|bloberr] */
+         *                 attrib|push|verbatim|true|false] */
         char *name = c->argv[2]->ptr;
         if (!strcasecmp(name,"string")) {
             addReplyBulkCString(c,"Hello World");
@@ -634,7 +635,7 @@ NULL
         } else if (!strcasecmp(name,"verbatim")) {
             addReplyVerbatim(c,"This is a verbatim\nstring",25,"txt");
         } else {
-            addReplyError(c,"Wrong protocol type name. Please use one of the following: string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false|state|err|bloberr");
+            addReplyError(c,"Wrong protocol type name. Please use one of the following: string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false");
         }
     } else if (!strcasecmp(c->argv[1]->ptr,"sleep") && c->argc == 3) {
         double dtime = strtod(c->argv[2]->ptr,NULL);

From 91c41b6ddee22eec7757a60fc87c24fd5087306a Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 6 Feb 2020 14:12:08 +0530
Subject: [PATCH 0092/1098] Some refactroing using getClientType instead of
 CLIENT_SLAVE

---
 src/networking.c |  9 +++++----
 src/object.c     | 35 +++++++++++++----------------------
 2 files changed, 18 insertions(+), 26 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 8fca3fc1d..496b0f3dc 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -369,9 +369,10 @@ void addReplyErrorLength(client *c, const char *s, size_t len) {
      * Where the master must propagate the first change even if the second
      * will produce an error. However it is useful to log such events since
      * they are rare and may hint at errors in a script or a bug in Redis. */
-    if (c->flags & (CLIENT_MASTER|CLIENT_SLAVE) && !(c->flags & CLIENT_MONITOR)) {
-        char* to = c->flags & CLIENT_MASTER? "master": "replica";
-        char* from = c->flags & CLIENT_MASTER? "replica": "master";
+    int ctype = getClientType(c);
+    if (ctype == CLIENT_TYPE_MASTER || ctype == CLIENT_TYPE_SLAVE) {
+        char* to = ctype == CLIENT_TYPE_MASTER? "master": "replica";
+        char* from = ctype == CLIENT_TYPE_MASTER? "replica": "master";
         char *cmdname = c->lastcmd ? c->lastcmd->name : "<unknown>";
         serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error "
                              "to its %s: '%s' after processing the command "
@@ -1074,7 +1075,7 @@ void freeClient(client *c) {
     }
 
     /* Log link disconnection with slave */
-    if ((c->flags & CLIENT_SLAVE) && !(c->flags & CLIENT_MONITOR)) {
+    if (getClientType(c) == CLIENT_TYPE_SLAVE) {
         serverLog(LL_WARNING,"Connection with replica %s lost.",
             replicationGetSlaveName(c));
     }
diff --git a/src/object.c b/src/object.c
index 52007b474..cc6b218a0 100644
--- a/src/object.c
+++ b/src/object.c
@@ -974,38 +974,29 @@ struct redisMemOverhead *getMemoryOverheadData(void) {
     mh->repl_backlog = mem;
     mem_total += mem;
 
-    mem = 0;
-    if (listLength(server.slaves)) {
-        listIter li;
-        listNode *ln;
-
-        listRewind(server.slaves,&li);
-        while((ln = listNext(&li))) {
-            client *c = listNodeValue(ln);
-            mem += getClientOutputBufferMemoryUsage(c);
-            mem += sdsAllocSize(c->querybuf);
-            mem += sizeof(client);
-        }
-    }
-    mh->clients_slaves = mem;
-    mem_total+=mem;
-
     mem = 0;
     if (listLength(server.clients)) {
         listIter li;
         listNode *ln;
+        size_t mem_normal = 0, mem_slaves = 0;
 
         listRewind(server.clients,&li);
         while((ln = listNext(&li))) {
+            size_t mem_curr = 0;
             client *c = listNodeValue(ln);
-            if (c->flags & CLIENT_SLAVE && !(c->flags & CLIENT_MONITOR))
-                continue;
-            mem += getClientOutputBufferMemoryUsage(c);
-            mem += sdsAllocSize(c->querybuf);
-            mem += sizeof(client);
+            int type = getClientType(c);
+            mem_curr += getClientOutputBufferMemoryUsage(c);
+            mem_curr += sdsAllocSize(c->querybuf);
+            mem_curr += sizeof(client);
+            if (type == CLIENT_TYPE_SLAVE)
+                mem_slaves += mem_curr;
+            else
+                mem_normal += mem_curr;
         }
+        mh->clients_slaves = mem_slaves;
+        mh->clients_normal = mem_normal;
+        mem = mem_slaves + mem_normal;
     }
-    mh->clients_normal = mem;
     mem_total+=mem;
 
     mem = 0;

From 28ef18a8946815e0d83a1c0a9b6baf9d27022461 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 5 Feb 2020 18:24:14 +0200
Subject: [PATCH 0093/1098] RM_Scan disable dict rehashing

The callback approach we took is very efficient, the module can do any
filtering of keys without building any list and cloning strings, it can
also read data from the key's value. but if the user tries to re-open
the key, or any other key, this can cause dict re-hashing (dictFind does
that), and that's very bad to do from inside dictScan.

this commit protects the dict from doing any rehashing during scan, but
also warns the user not to attempt any writes or command calls from
within the callback, for fear of unexpected side effects and crashes.
---
 src/dict.c   |  7 +++++++
 src/module.c | 20 ++++++++++++++------
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/src/dict.c b/src/dict.c
index 106467ef7..93e6c39a7 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -871,6 +871,10 @@ unsigned long dictScan(dict *d,
 
     if (dictSize(d) == 0) return 0;
 
+    /* Having a safe iterator means no rehashing can happen, see _dictRehashStep.
+     * This is needed in case the scan callback tries to do dictFind or alike. */
+    d->iterators++;
+
     if (!dictIsRehashing(d)) {
         t0 = &(d->ht[0]);
         m0 = t0->sizemask;
@@ -937,6 +941,9 @@ unsigned long dictScan(dict *d,
         } while (v & (m0 ^ m1));
     }
 
+    /* undo the ++ at the top */
+    d->iterators--;
+
     return v;
 }
 
diff --git a/src/module.c b/src/module.c
index 914c50df3..e9bceac47 100644
--- a/src/module.c
+++ b/src/module.c
@@ -6515,9 +6515,13 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) {
  *      }
  *      RedisModule_ScanCursorDestroy(c);
  *
- *  The function will return 1 if there are more elements to scan and 0 otherwise,
- *  possibly setting errno if the call failed.
- *  It is also possible to restart and existing cursor using RM_CursorRestart. */
+ * The function will return 1 if there are more elements to scan and 0 otherwise,
+ * possibly setting errno if the call failed.
+ * It is also possible to restart and existing cursor using RM_CursorRestart.
+ *
+ * NOTE: You must avoid doing any database changes from within the callback, you should avoid any
+ * RedisModule_OpenKey or RedisModule_Call, if you need to do these, you need to keep the key name
+ * and do any work you need to do after the call to Scan returns. */
 int RM_Scan(RedisModuleCtx *ctx, RedisModuleScanCursor *cursor, RedisModuleScanCB fn, void *privdata) {
     if (cursor->done) {
         errno = ENOENT;
@@ -6595,9 +6599,13 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) {
  *      RedisModule_CloseKey(key);
  *      RedisModule_ScanCursorDestroy(c);
  *
- *  The function will return 1 if there are more elements to scan and 0 otherwise,
- *  possibly setting errno if the call failed.
- *  It is also possible to restart and existing cursor using RM_CursorRestart. */
+ * The function will return 1 if there are more elements to scan and 0 otherwise,
+ * possibly setting errno if the call failed.
+ * It is also possible to restart and existing cursor using RM_CursorRestart.
+ *
+ * NOTE: You must avoid doing any database changes from within the callback, you should avoid any
+ * RedisModule_OpenKey or RedisModule_Call, if you need to do these, you need to keep the field name
+ * and do any work you need to do after the call to Scan returns. */
 int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleScanKeyCB fn, void *privdata) {
     if (key == NULL || key->value == NULL) {
         errno = EINVAL;

From 92dc5e1fa41491e0ab0744a2bab55f837db89dc2 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Mon, 3 Feb 2020 17:19:00 +0530
Subject: [PATCH 0094/1098] Diskless-load emptyDb-related fixes

1. Call emptyDb even in case of diskless-load: We want modules
   to get the same FLUSHDB event as disk-based replication.
2. Do not fire any module events when flushing the backups array.
3. Delete redundant call to signalFlushedDb (Called from emptyDb).
---
 src/db.c          | 57 ++++++++++++++++++++++++++++-------------------
 src/replication.c | 14 +++++++-----
 src/server.h      |  1 +
 3 files changed, 44 insertions(+), 28 deletions(-)

diff --git a/src/db.c b/src/db.c
index ba7be2725..c2f974cbb 100644
--- a/src/db.c
+++ b/src/db.c
@@ -347,7 +347,10 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) {
  * DB number if we want to flush only a single Redis database number.
  *
  * Flags are be EMPTYDB_NO_FLAGS if no special flags are specified or
- * EMPTYDB_ASYNC if we want the memory to be freed in a different thread
+ * 1. EMPTYDB_ASYNC if we want the memory to be freed in a different thread.
+ * 2. EMPTYDB_BACKUP if we want to empty the backup dictionaries created by
+ *    disklessLoadMakeBackups. In that case we only free memory and avoid
+ *    firing module events.
  * and the function to return ASAP.
  *
  * On success the fuction returns the number of keys removed from the
@@ -355,6 +358,8 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) {
  * DB number is out of range, and errno is set to EINVAL. */
 long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(void*)) {
     int async = (flags & EMPTYDB_ASYNC);
+    int backup = (flags & EMPTYDB_BACKUP); /* Just free the memory, nothing else */
+    RedisModuleFlushInfoV1 fi = {REDISMODULE_FLUSHINFO_VERSION,!async,dbnum};
     long long removed = 0;
 
     if (dbnum < -1 || dbnum >= server.dbnum) {
@@ -362,16 +367,18 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(
         return -1;
     }
 
-    /* Fire the flushdb modules event. */
-    RedisModuleFlushInfoV1 fi = {REDISMODULE_FLUSHINFO_VERSION,!async,dbnum};
-    moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB,
-                          REDISMODULE_SUBEVENT_FLUSHDB_START,
-                          &fi);
+    /* Pre-flush actions */
+    if (!backup) {
+        /* Fire the flushdb modules event. */
+        moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB,
+                              REDISMODULE_SUBEVENT_FLUSHDB_START,
+                              &fi);
 
-    /* Make sure the WATCHed keys are affected by the FLUSH* commands.
-     * Note that we need to call the function while the keys are still
-     * there. */
-    signalFlushedDb(dbnum);
+        /* Make sure the WATCHed keys are affected by the FLUSH* commands.
+         * Note that we need to call the function while the keys are still
+         * there. */
+        signalFlushedDb(dbnum);
+    }
 
     int startdb, enddb;
     if (dbnum == -1) {
@@ -390,20 +397,24 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(
             dictEmpty(dbarray[j].expires,callback);
         }
     }
-    if (server.cluster_enabled) {
-        if (async) {
-            slotToKeyFlushAsync();
-        } else {
-            slotToKeyFlush();
-        }
-    }
-    if (dbnum == -1) flushSlaveKeysWithExpireList();
 
-    /* Also fire the end event. Note that this event will fire almost
-     * immediately after the start event if the flush is asynchronous. */
-    moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB,
-                          REDISMODULE_SUBEVENT_FLUSHDB_END,
-                          &fi);
+    /* Post-flush actions */
+    if (!backup) {
+        if (server.cluster_enabled) {
+            if (async) {
+                slotToKeyFlushAsync();
+            } else {
+                slotToKeyFlush();
+            }
+        }
+        if (dbnum == -1) flushSlaveKeysWithExpireList();
+
+        /* Also fire the end event. Note that this event will fire almost
+         * immediately after the start event if the flush is asynchronous. */
+        moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB,
+                              REDISMODULE_SUBEVENT_FLUSHDB_END,
+                              &fi);
+    }
 
     return removed;
 }
diff --git a/src/replication.c b/src/replication.c
index b7e77184a..dd2c3dd2f 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1339,8 +1339,8 @@ void disklessLoadRestoreBackups(redisDb *backup, int restore, int empty_db_flags
             server.db[i] = backup[i];
         }
     } else {
-        /* Delete. */
-        emptyDbGeneric(backup,-1,empty_db_flags,replicationEmptyDbCallback);
+        /* Delete (Pass EMPTYDB_BACKUP in order to avoid firing module events) . */
+        emptyDbGeneric(backup,-1,empty_db_flags|EMPTYDB_BACKUP,replicationEmptyDbCallback);
         for (int i=0; i<server.dbnum; i++) {
             dictRelease(backup[i].dict);
             dictRelease(backup[i].expires);
@@ -1525,7 +1525,6 @@ void readSyncBulkPayload(connection *conn) {
     /* We need to stop any AOF rewriting child before flusing and parsing
      * the RDB, otherwise we'll create a copy-on-write disaster. */
     if (server.aof_state != AOF_OFF) stopAppendOnly();
-    signalFlushedDb(-1);
 
     /* When diskless RDB loading is used by replicas, it may be configured
      * in order to save the current DB instead of throwing it away,
@@ -1533,10 +1532,15 @@ void readSyncBulkPayload(connection *conn) {
     if (use_diskless_load &&
         server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB)
     {
+        /* Create a backup of server.db[] and initialize to empty
+         * dictionaries */
         diskless_load_backup = disklessLoadMakeBackups();
-    } else {
-        emptyDb(-1,empty_db_flags,replicationEmptyDbCallback);
     }
+    /* We call to emptyDb even in case of REPL_DISKLESS_LOAD_SWAPDB
+     * (Where disklessLoadMakeBackups left server.db empty) because we
+     * want to execute all the auxiliary logic of emptyDb (Namely,
+     * fire module events) */
+    emptyDb(-1,empty_db_flags,replicationEmptyDbCallback);
 
     /* Before loading the DB into memory we need to delete the readable
      * handler, otherwise it will get called recursively since
diff --git a/src/server.h b/src/server.h
index 8e354c03d..4b7a338da 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2042,6 +2042,7 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o);
 
 #define EMPTYDB_NO_FLAGS 0      /* No flags. */
 #define EMPTYDB_ASYNC (1<<0)    /* Reclaim memory in another thread. */
+#define EMPTYDB_BACKUP (1<<2)   /* DB array is a backup for REPL_DISKLESS_LOAD_SWAPDB. */
 long long emptyDb(int dbnum, int flags, void(callback)(void*));
 long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(void*));
 void flushAllDataAndResetRDB(int flags);

From c8c3281f06acb274e56440e2dec4ec8e2bcd6aa3 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 6 Feb 2020 14:53:33 +0200
Subject: [PATCH 0095/1098] add no_auth to COMMAND INFO

---
 src/server.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/server.c b/src/server.c
index f3b7c37c5..01c048c90 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3764,6 +3764,7 @@ void addReplyCommand(client *c, struct redisCommand *cmd) {
         flagcount += addReplyCommandFlag(c,cmd,CMD_SKIP_SLOWLOG, "skip_slowlog");
         flagcount += addReplyCommandFlag(c,cmd,CMD_ASKING, "asking");
         flagcount += addReplyCommandFlag(c,cmd,CMD_FAST, "fast");
+        flagcount += addReplyCommandFlag(c,cmd,CMD_NO_AUTH, "no_auth");
         if ((cmd->getkeys_proc && !(cmd->flags & CMD_MODULE)) ||
             cmd->flags & CMD_MODULE_GETKEYS)
         {

From 5c73a6e2067f2bad1e622c372793a6d40ab1e2a0 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 6 Feb 2020 18:36:21 +0530
Subject: [PATCH 0096/1098] Fix memory leak in test_ld_conv

---
 tests/modules/misc.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/modules/misc.c b/tests/modules/misc.c
index 41bec06ed..1048d5065 100644
--- a/tests/modules/misc.c
+++ b/tests/modules/misc.c
@@ -74,6 +74,7 @@ int test_ld_conv(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
         RedisModule_ReplyWithError(ctx, err);
         goto final;
     }
+
     /* Make sure we can't convert a string that has \0 in it */
     char buf[4] = "123";
     buf[1] = '\0';
@@ -81,8 +82,11 @@ int test_ld_conv(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     long double ld3;
     if (RedisModule_StringToLongDouble(s3, &ld3) == REDISMODULE_OK) {
         RedisModule_ReplyWithError(ctx, "Invalid string successfully converted to long double");
+        RedisModule_FreeString(ctx, s3);
         goto final;
     }
+    RedisModule_FreeString(ctx, s3);
+
     RedisModule_ReplyWithLongDouble(ctx, ld2);
 final:
     RedisModule_FreeString(ctx, s1);

From 46216b0e838b54099a0ac364b33b2dfb7f1de8c7 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 6 Feb 2020 15:06:33 +0200
Subject: [PATCH 0097/1098] add no-slowlog option to RM_CreateCommand

---
 src/module.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/module.c b/src/module.c
index 914c50df3..84a4527eb 100644
--- a/src/module.c
+++ b/src/module.c
@@ -730,6 +730,7 @@ int commandFlagsFromString(char *s) {
         else if (!strcasecmp(t,"random")) flags |= CMD_RANDOM;
         else if (!strcasecmp(t,"allow-stale")) flags |= CMD_STALE;
         else if (!strcasecmp(t,"no-monitor")) flags |= CMD_SKIP_MONITOR;
+        else if (!strcasecmp(t,"no-slowlog")) flags |= CMD_SKIP_SLOWLOG;
         else if (!strcasecmp(t,"fast")) flags |= CMD_FAST;
         else if (!strcasecmp(t,"no-auth")) flags |= CMD_NO_AUTH;
         else if (!strcasecmp(t,"getkeys-api")) flags |= CMD_MODULE_GETKEYS;
@@ -781,6 +782,8 @@ int commandFlagsFromString(char *s) {
  *                      this means.
  * * **"no-monitor"**: Don't propagate the command on monitor. Use this if
  *                     the command has sensible data among the arguments.
+ * * **"no-slowlog"**: Don't log this command in the slowlog. Use this if
+ *                     the command has sensible data among the arguments.
  * * **"fast"**:      The command time complexity is not greater
  *                    than O(log(N)) where N is the size of the collection or
  *                    anything else representing the normal scalability

From 92357b2d61bff014e3198bb39886d6b4e26163d1 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 7 Feb 2020 14:03:43 +0100
Subject: [PATCH 0098/1098] Tracking: first conversion from hashing to key
 names.

---
 src/server.c   |   4 +-
 src/server.h   |   2 +-
 src/tracking.c | 206 +++++++++++++++++++++----------------------------
 3 files changed, 91 insertions(+), 121 deletions(-)

diff --git a/src/server.c b/src/server.c
index f87dfba46..df6a8c1f4 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4221,7 +4221,7 @@ sds genRedisInfoString(const char *section) {
             "active_defrag_misses:%lld\r\n"
             "active_defrag_key_hits:%lld\r\n"
             "active_defrag_key_misses:%lld\r\n"
-            "tracking_used_slots:%lld\r\n",
+            "tracking_tracked_keys:%lld\r\n",
             server.stat_numconnections,
             server.stat_numcommands,
             getInstantaneousMetric(STATS_METRIC_COMMAND),
@@ -4249,7 +4249,7 @@ sds genRedisInfoString(const char *section) {
             server.stat_active_defrag_misses,
             server.stat_active_defrag_key_hits,
             server.stat_active_defrag_key_misses,
-            trackingGetUsedSlots());
+            (unsigned long long) trackingGetTotalItems());
     }
 
     /* Replication */
diff --git a/src/server.h b/src/server.h
index dfbc15ac3..a30db6b57 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1654,7 +1654,7 @@ void trackingRememberKeys(client *c);
 void trackingInvalidateKey(robj *keyobj);
 void trackingInvalidateKeysOnFlush(int dbid);
 void trackingLimitUsedSlots(void);
-unsigned long long trackingGetUsedSlots(void);
+uint64_t trackingGetTotalItems(void);
 
 /* List data type */
 void listTypeTryConversion(robj *subject, robj *value);
diff --git a/src/tracking.c b/src/tracking.c
index acb97800a..ecb7fbdcc 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -30,37 +30,22 @@
 
 #include "server.h"
 
-/* The tracking table is constituted by 2^24 radix trees (each tree, and the
- * table itself, are allocated in a lazy way only when needed) tracking
- * clients that may have certain keys in their local, client side, cache.
- *
- * Keys are grouped into 2^24 slots, in a way similar to Redis Cluster hash
- * slots, however here the function we use is crc64, taking the least
- * significant 24 bits of the output.
+/* The tracking table is constituted by a radix tree of keys, each pointing
+ * to a radix tree of client IDs, used to track the clients that may have
+ * certain keys in their local, client side, cache.
  *
  * When a client enables tracking with "CLIENT TRACKING on", each key served to
- * the client is hashed to one of such slots, and Redis will remember what
- * client may have keys about such slot. Later, when a key in a given slot is
- * modified, all the clients that may have local copies of keys in that slot
- * will receive an invalidation message. There is no distinction of database
- * number: a single table is used.
+ * the client is remembered in the table mapping the keys to the client IDs.
+ * Later, when a key is modified, all the clients that may have local copy
+ * of such key will receive an invalidation message.
  *
  * Clients will normally take frequently requested objects in memory, removing
- * them when invalidation messages are received. A strategy clients may use is
- * to just cache objects in a dictionary, associating to each cached object
- * some incremental epoch, or just a timestamp. When invalidation messages are
- * received clients may store, in a different table, the timestamp (or epoch)
- * of the invalidation of such given slot: later when accessing objects, the
- * eviction of stale objects may be performed in a lazy way by checking if the
- * cached object timestamp is older than the invalidation timestamp for such
- * objects.
- *
- * The output of the 24 bit hash function is very large (more than 16 million
- * possible slots), so clients that may want to use less resources may only
- * use the most significant bits instead of the full 24 bits. */
-#define TRACKING_TABLE_SIZE (1<<24)
-rax **TrackingTable = NULL;
-unsigned long TrackingTableUsedSlots = 0;
+ * them when invalidation messages are received. */
+rax *TrackingTable = NULL;
+uint64_t TrackingTableTotalItems = 0; /* Total number of IDs stored across
+                                         the whole tracking table. This givesn
+                                         an hint about the total memory we
+                                         are using server side for CSC. */
 robj *TrackingChannelName;
 
 /* Remove the tracking state from the client 'c'. Note that there is not much
@@ -90,7 +75,7 @@ void enableTracking(client *c, uint64_t redirect_to) {
     c->client_tracking_redirection = redirect_to;
     server.tracking_clients++;
     if (TrackingTable == NULL) {
-        TrackingTable = zcalloc(sizeof(rax*) * TRACKING_TABLE_SIZE);
+        TrackingTable = raxNew();
         TrackingChannelName = createStringObject("__redis__:invalidate",20);
     }
 }
@@ -108,19 +93,20 @@ void trackingRememberKeys(client *c) {
     for(int j = 0; j < numkeys; j++) {
         int idx = keys[j];
         sds sdskey = c->argv[idx]->ptr;
-        uint64_t hash = crc64(0,
-            (unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1);
-        if (TrackingTable[hash] == NULL) {
-            TrackingTable[hash] = raxNew();
-            TrackingTableUsedSlots++;
+        rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey));
+        if (ids == raxNotFound) {
+            ids = raxNew();
+            int inserted = raxTryInsert(TrackingTable,(unsigned char*)sdskey,
+                                        sdslen(sdskey),ids, NULL);
+            serverAssert(inserted == 1);
         }
-        raxTryInsert(TrackingTable[hash],
-            (unsigned char*)&c->id,sizeof(c->id),NULL,NULL);
+        if (raxTryInsert(ids,(unsigned char*)&c->id,sizeof(c->id),NULL,NULL))
+            TrackingTableTotalItems++;
     }
     getKeysFreeResult(keys);
 }
 
-void sendTrackingMessage(client *c, long long hash) {
+void sendTrackingMessage(client *c, char *keyname, size_t keylen) {
     int using_redirection = 0;
     if (c->client_tracking_redirection) {
         client *redir = lookupClientByID(c->client_tracking_redirection);
@@ -146,49 +132,44 @@ void sendTrackingMessage(client *c, long long hash) {
     if (c->resp > 2) {
         addReplyPushLen(c,2);
         addReplyBulkCBuffer(c,"invalidate",10);
-        addReplyLongLong(c,hash);
+        addReplyBulkCBuffer(c,keyname,keylen);
     } else if (using_redirection && c->flags & CLIENT_PUBSUB) {
-        robj *msg = createStringObjectFromLongLong(hash);
-        addReplyPubsubMessage(c,TrackingChannelName,msg);
-        decrRefCount(msg);
+        /* We use a static object to speedup things, however we assume
+         * that addReplyPubsubMessage() will not take a reference. */
+        robj keyobj;
+        initStaticStringObject(keyobj,keyname);
+        addReplyPubsubMessage(c,TrackingChannelName,&keyobj);
+        serverAssert(keyobj.refcount == 1);
     }
 }
 
-/* Invalidates a caching slot: this is actually the low level implementation
- * of the API that Redis calls externally, that is trackingInvalidateKey(). */
-void trackingInvalidateSlot(uint64_t slot) {
-    if (TrackingTable == NULL || TrackingTable[slot] == NULL) return;
-
-    raxIterator ri;
-    raxStart(&ri,TrackingTable[slot]);
-    raxSeek(&ri,"^",NULL,0);
-    while(raxNext(&ri)) {
-        uint64_t id;
-        memcpy(&id,ri.key,sizeof(id));
-        client *c = lookupClientByID(id);
-        if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue;
-        sendTrackingMessage(c,slot);
-    }
-    raxStop(&ri);
-
-    /* Free the tracking table: we'll create the radix tree and populate it
-     * again if more keys will be modified in this caching slot. */
-    raxFree(TrackingTable[slot]);
-    TrackingTable[slot] = NULL;
-    TrackingTableUsedSlots--;
-}
-
 /* This function is called from signalModifiedKey() or other places in Redis
  * when a key changes value. In the context of keys tracking, our task here is
  * to send a notification to every client that may have keys about such caching
  * slot. */
 void trackingInvalidateKey(robj *keyobj) {
-    if (TrackingTable == NULL || TrackingTableUsedSlots == 0) return;
-
+    if (TrackingTable == NULL) return;
     sds sdskey = keyobj->ptr;
-    uint64_t hash = crc64(0,
-        (unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1);
-    trackingInvalidateSlot(hash);
+    rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey));
+    if (ids == raxNotFound) return;;
+
+    raxIterator ri;
+    raxStart(&ri,ids);
+    raxSeek(&ri,"^",NULL,0);
+    while(raxNext(&ri)) {
+        uint64_t id;
+        memcpy(&id,ri.key,sizeof(id));
+        client *c = lookupClientByID(id);
+        if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue;
+        sendTrackingMessage(c,sdskey,sdslen(sdskey));
+    }
+    raxStop(&ri);
+
+    /* Free the tracking table: we'll create the radix tree and populate it
+     * again if more keys will be modified in this caching slot. */
+    TrackingTableTotalItems -= raxSize(ids);
+    raxFree(ids);
+    raxRemove(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey),NULL);
 }
 
 /* This function is called when one or all the Redis databases are flushed
@@ -205,6 +186,10 @@ void trackingInvalidateKey(robj *keyobj) {
  * we just send the invalidation message to all the clients, but don't
  * flush the table: it will slowly get garbage collected as more keys
  * are modified in the used caching slots. */
+void freeTrackingRadixTree(void *rt) {
+    raxFree(rt);
+}
+
 void trackingInvalidateKeysOnFlush(int dbid) {
     if (server.tracking_clients) {
         listNode *ln;
@@ -213,84 +198,69 @@ void trackingInvalidateKeysOnFlush(int dbid) {
         while ((ln = listNext(&li)) != NULL) {
             client *c = listNodeValue(ln);
             if (c->flags & CLIENT_TRACKING) {
-                sendTrackingMessage(c,-1);
+                sendTrackingMessage(c,"",1);
             }
         }
     }
 
     /* In case of FLUSHALL, reclaim all the memory used by tracking. */
     if (dbid == -1 && TrackingTable) {
-        for (int j = 0; j < TRACKING_TABLE_SIZE && TrackingTableUsedSlots > 0; j++) {
-            if (TrackingTable[j] != NULL) {
-                raxFree(TrackingTable[j]);
-                TrackingTable[j] = NULL;
-                TrackingTableUsedSlots--;
-            }
-        }
-
-        /* If there are no clients with tracking enabled, we can even
-         * reclaim the memory used by the table itself. The code assumes
-         * the table is allocated only if there is at least one client alive
-         * with tracking enabled. */
-        if (server.tracking_clients == 0) {
-            zfree(TrackingTable);
-            TrackingTable = NULL;
-        }
+        raxFreeWithCallback(TrackingTable,freeTrackingRadixTree);
+        TrackingTableTotalItems = 0;
     }
 }
 
 /* Tracking forces Redis to remember information about which client may have
- * keys about certian caching slots. In workloads where there are a lot of
- * reads, but keys are hardly modified, the amount of information we have
- * to remember server side could be a lot: for each 16 millions of caching
- * slots we may end with a radix tree containing many entries.
+ * certain keys. In workloads where there are a lot of reads, but keys are
+ * hardly modified, the amount of information we have to remember server side
+ * could be a lot, with the number of keys being totally not bound.
  *
- * So Redis allows the user to configure a maximum fill rate for the
+ * So Redis allows the user to configure a maximum number of keys for the
  * invalidation table. This function makes sure that we don't go over the
  * specified fill rate: if we are over, we can just evict informations about
- * random caching slots, and send invalidation messages to clients like if
- * the key was modified. */
+ * a random key, and send invalidation messages to clients like if the key was
+ * modified. */
 void trackingLimitUsedSlots(void) {
     static unsigned int timeout_counter = 0;
-
+    if (TrackingTable == NULL) return;
     if (server.tracking_table_max_fill == 0) return; /* No limits set. */
-    unsigned int max_slots =
-        (TRACKING_TABLE_SIZE/100) * server.tracking_table_max_fill;
-    if (TrackingTableUsedSlots <= max_slots) {
+    size_t max_keys = server.tracking_table_max_fill;
+    if (raxSize(TrackingTable) <= max_keys) {
         timeout_counter = 0;
         return; /* Limit not reached. */
     }
 
-    /* We have to invalidate a few slots to reach the limit again. The effort
+    /* We have to invalidate a few keys to reach the limit again. The effort
      * we do here is proportional to the number of times we entered this
      * function and found that we are still over the limit. */
     int effort = 100 * (timeout_counter+1);
 
-    /* Let's start at a random position, and perform linear probing, in order
-     * to improve cache locality. However once we are able to find an used
-     * slot, jump again randomly, in order to avoid creating big holes in the
-     * table (that will make this funciton use more resourced later). */
+    /* We just remove one key after another by using a random walk. */
+    raxIterator ri;
+    raxStart(&ri,TrackingTable);
     while(effort > 0) {
-        unsigned int idx = rand() % TRACKING_TABLE_SIZE;
-        do {
-            effort--;
-            idx = (idx+1) % TRACKING_TABLE_SIZE;
-            if (TrackingTable[idx] != NULL) {
-                trackingInvalidateSlot(idx);
-                if (TrackingTableUsedSlots <= max_slots) {
-                    timeout_counter = 0;
-                    return; /* Return ASAP: we are again under the limit. */
-                } else {
-                    break; /* Jump to next random position. */
-                }
-            }
-        } while(effort > 0);
+        effort--;
+        raxSeek(&ri,"^",NULL,0);
+        raxRandomWalk(&ri,0);
+        rax *ids = ri.data;
+        TrackingTableTotalItems -= raxSize(ids);
+        raxFree(ids);
+        raxRemove(TrackingTable,ri.key,ri.key_len,NULL);
+        if (raxSize(TrackingTable) <= max_keys) {
+            timeout_counter = 0;
+            raxStop(&ri);
+            return; /* Return ASAP: we are again under the limit. */
+        }
     }
+
+    /* If we reach this point, we were not able to go under the configured
+     * limit using the maximum effort we had for this run. */
+    raxStop(&ri);
     timeout_counter++;
 }
 
 /* This is just used in order to access the amount of used slots in the
  * tracking table. */
-unsigned long long trackingGetUsedSlots(void) {
-    return TrackingTableUsedSlots;
+uint64_t trackingGetTotalItems(void) {
+    return TrackingTableTotalItems;
 }

From d933d6f2a42b9f0add28df212b5cfbc7756e3cf2 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 7 Feb 2020 17:19:11 +0100
Subject: [PATCH 0099/1098] Tracking: rename INFO field with total items.

---
 src/server.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index df6a8c1f4..2cf8dbf19 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4221,7 +4221,7 @@ sds genRedisInfoString(const char *section) {
             "active_defrag_misses:%lld\r\n"
             "active_defrag_key_hits:%lld\r\n"
             "active_defrag_key_misses:%lld\r\n"
-            "tracking_tracked_keys:%lld\r\n",
+            "tracking_total_items:%lld\r\n",
             server.stat_numconnections,
             server.stat_numcommands,
             getInstantaneousMetric(STATS_METRIC_COMMAND),

From 1ea66724300f1cd7c97b842a7c9190e72ea91976 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 7 Feb 2020 18:12:10 +0100
Subject: [PATCH 0100/1098] Rax.c: populate data field after random walk.

---
 src/rax.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/rax.c b/src/rax.c
index 29b74ae90..a560dde02 100644
--- a/src/rax.c
+++ b/src/rax.c
@@ -1766,6 +1766,7 @@ int raxRandomWalk(raxIterator *it, size_t steps) {
         if (n->iskey) steps--;
     }
     it->node = n;
+    it->data = raxGetData(it->node);
     return 1;
 }
 

From 85e4777d5c4411b499a845565729910bb09d64ac Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 7 Feb 2020 18:12:45 +0100
Subject: [PATCH 0101/1098] Tracking: minor change of names and new INFO field.

---
 src/config.c   | 2 +-
 src/server.c   | 2 ++
 src/server.h   | 3 ++-
 src/tracking.c | 8 ++++++--
 4 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/config.c b/src/config.c
index a6b374817..b2e5fc12e 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2160,7 +2160,7 @@ standardConfig configs[] = {
     createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, 0, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, 0, INTEGER_CONFIG, NULL, NULL),
-    createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_fill, 10, INTEGER_CONFIG, NULL, NULL), /* Default: 10% tracking table max fill. */
+    createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_keys, 1000000, INTEGER_CONFIG, NULL, NULL), /* Default: 10% tracking table max number of keys tracked. */
     createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, 1, INTEGER_CONFIG, NULL, NULL), /* From 1 to 10. */
     createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG, NULL, updateHZ),
     createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves),
diff --git a/src/server.c b/src/server.c
index 2cf8dbf19..910cf5410 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4221,6 +4221,7 @@ sds genRedisInfoString(const char *section) {
             "active_defrag_misses:%lld\r\n"
             "active_defrag_key_hits:%lld\r\n"
             "active_defrag_key_misses:%lld\r\n"
+            "tracking_total_keys:%lld\r\n"
             "tracking_total_items:%lld\r\n",
             server.stat_numconnections,
             server.stat_numcommands,
@@ -4249,6 +4250,7 @@ sds genRedisInfoString(const char *section) {
             server.stat_active_defrag_misses,
             server.stat_active_defrag_key_hits,
             server.stat_active_defrag_key_misses,
+            (unsigned long long) trackingGetTotalKeys(),
             (unsigned long long) trackingGetTotalItems());
     }
 
diff --git a/src/server.h b/src/server.h
index a30db6b57..3e055a7db 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1306,7 +1306,7 @@ struct redisServer {
     list *ready_keys;        /* List of readyList structures for BLPOP & co */
     /* Client side caching. */
     unsigned int tracking_clients;  /* # of clients with tracking enabled.*/
-    int tracking_table_max_fill;    /* Max fill percentage. */
+    int tracking_table_max_keys;    /* Max number of keys in tracking table. */
     /* Sort parameters - qsort_r() is only available under BSD so we
      * have to take this state global, in order to pass it to sortCompare() */
     int sort_desc;
@@ -1655,6 +1655,7 @@ void trackingInvalidateKey(robj *keyobj);
 void trackingInvalidateKeysOnFlush(int dbid);
 void trackingLimitUsedSlots(void);
 uint64_t trackingGetTotalItems(void);
+uint64_t trackingGetTotalKeys(void);
 
 /* List data type */
 void listTypeTryConversion(robj *subject, robj *value);
diff --git a/src/tracking.c b/src/tracking.c
index ecb7fbdcc..9c1c9620c 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -223,8 +223,8 @@ void trackingInvalidateKeysOnFlush(int dbid) {
 void trackingLimitUsedSlots(void) {
     static unsigned int timeout_counter = 0;
     if (TrackingTable == NULL) return;
-    if (server.tracking_table_max_fill == 0) return; /* No limits set. */
-    size_t max_keys = server.tracking_table_max_fill;
+    if (server.tracking_table_max_keys == 0) return; /* No limits set. */
+    size_t max_keys = server.tracking_table_max_keys;
     if (raxSize(TrackingTable) <= max_keys) {
         timeout_counter = 0;
         return; /* Limit not reached. */
@@ -264,3 +264,7 @@ void trackingLimitUsedSlots(void) {
 uint64_t trackingGetTotalItems(void) {
     return TrackingTableTotalItems;
 }
+
+uint64_t trackingGetTotalKeys(void) {
+    return raxSize(TrackingTable);
+}

From ef764dde1cca2f25d00686673d1bc89448819571 Mon Sep 17 00:00:00 2001
From: Seunghoon Woo <toad58@nate.com>
Date: Mon, 10 Feb 2020 16:32:46 +0900
Subject: [PATCH 0102/1098] [FIX] revisit CVE-2015-8080 vulnerability

---
 deps/lua/src/lua_struct.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/deps/lua/src/lua_struct.c b/deps/lua/src/lua_struct.c
index 4d5f027b8..c58c8e72b 100644
--- a/deps/lua/src/lua_struct.c
+++ b/deps/lua/src/lua_struct.c
@@ -89,12 +89,14 @@ typedef struct Header {
 } Header;
 
 
-static int getnum (const char **fmt, int df) {
+static int getnum (lua_State *L, const char **fmt, int df) {
   if (!isdigit(**fmt))  /* no number? */
     return df;  /* return default value */
   else {
     int a = 0;
     do {
+      if (a > (INT_MAX / 10) || a * 10 > (INT_MAX - (**fmt - '0')))
+        luaL_error(L, "integral size overflow");
       a = a*10 + *((*fmt)++) - '0';
     } while (isdigit(**fmt));
     return a;
@@ -115,9 +117,9 @@ static size_t optsize (lua_State *L, char opt, const char **fmt) {
     case 'f':  return sizeof(float);
     case 'd':  return sizeof(double);
     case 'x': return 1;
-    case 'c': return getnum(fmt, 1);
+    case 'c': return getnum(L, fmt, 1);
     case 'i': case 'I': {
-      int sz = getnum(fmt, sizeof(int));
+      int sz = getnum(L, fmt, sizeof(int));
       if (sz > MAXINTSIZE)
         luaL_error(L, "integral size %d is larger than limit of %d",
                        sz, MAXINTSIZE);
@@ -150,7 +152,7 @@ static void controloptions (lua_State *L, int opt, const char **fmt,
     case '>': h->endian = BIG; return;
     case '<': h->endian = LITTLE; return;
     case '!': {
-      int a = getnum(fmt, MAXALIGN);
+      int a = getnum(L, fmt, MAXALIGN);
       if (!isp2(a))
         luaL_error(L, "alignment %d is not a power of 2", a);
       h->align = a;

From 67dbc5f365ccaff65d996486a86c77e5bff390a8 Mon Sep 17 00:00:00 2001
From: "meir@redislabs.com" <meir@redislabs.com>
Date: Mon, 10 Feb 2020 12:10:32 +0200
Subject: [PATCH 0103/1098] Changed log level for module fork api from 'notice'
 to 'verbos'.

---
 src/module.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/module.c b/src/module.c
index ed79ee226..d15aa6c93 100644
--- a/src/module.c
+++ b/src/module.c
@@ -6706,7 +6706,7 @@ int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) {
         server.module_child_pid = childpid;
         moduleForkInfo.done_handler = cb;
         moduleForkInfo.done_handler_user_data = user_data;
-        serverLog(LL_NOTICE, "Module fork started pid: %d ", childpid);
+        serverLog(LL_VERBOSE, "Module fork started pid: %d ", childpid);
     }
     return childpid;
 }
@@ -6729,7 +6729,7 @@ int TerminateModuleForkChild(int child_pid, int wait) {
         server.module_child_pid != child_pid) return C_ERR;
 
     int statloc;
-    serverLog(LL_NOTICE,"Killing running module fork child: %ld",
+    serverLog(LL_VERBOSE,"Killing running module fork child: %ld",
         (long) server.module_child_pid);
     if (kill(server.module_child_pid,SIGUSR1) != -1 && wait) {
         while(wait4(server.module_child_pid,&statloc,0,NULL) !=

From f53cc00c09a4e7c612b3781021246cbbeb533d7b Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 10 Feb 2020 13:42:18 +0100
Subject: [PATCH 0104/1098] Tracking: always reply with an array of keys.

---
 src/pubsub.c   | 8 ++++++--
 src/tracking.c | 5 ++++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/pubsub.c b/src/pubsub.c
index 994dd9734..5cb4298e0 100644
--- a/src/pubsub.c
+++ b/src/pubsub.c
@@ -35,7 +35,11 @@ int clientSubscriptionsCount(client *c);
  * Pubsub client replies API
  *----------------------------------------------------------------------------*/
 
-/* Send a pubsub message of type "message" to the client. */
+/* Send a pubsub message of type "message" to the client.
+ * Normally 'msg' is a Redis object containing the string to send as
+ * message. However if the caller sets 'msg' as NULL, it will be able
+ * to send a special message (for instance an Array type) by using the
+ * addReply*() API family. */
 void addReplyPubsubMessage(client *c, robj *channel, robj *msg) {
     if (c->resp == 2)
         addReply(c,shared.mbulkhdr[3]);
@@ -43,7 +47,7 @@ void addReplyPubsubMessage(client *c, robj *channel, robj *msg) {
         addReplyPushLen(c,3);
     addReply(c,shared.messagebulk);
     addReplyBulk(c,channel);
-    addReplyBulk(c,msg);
+    if (msg) addReplyBulk(c,msg);
 }
 
 /* Send a pubsub message of type "pmessage" to the client. The difference
diff --git a/src/tracking.c b/src/tracking.c
index 9c1c9620c..3122563ac 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -132,13 +132,16 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen) {
     if (c->resp > 2) {
         addReplyPushLen(c,2);
         addReplyBulkCBuffer(c,"invalidate",10);
+        addReplyArrayLen(c,1);
         addReplyBulkCBuffer(c,keyname,keylen);
     } else if (using_redirection && c->flags & CLIENT_PUBSUB) {
         /* We use a static object to speedup things, however we assume
          * that addReplyPubsubMessage() will not take a reference. */
         robj keyobj;
         initStaticStringObject(keyobj,keyname);
-        addReplyPubsubMessage(c,TrackingChannelName,&keyobj);
+        addReplyPubsubMessage(c,TrackingChannelName,NULL);
+        addReplyArrayLen(c,1);
+        addReplyBulk(c,&keyobj);
         serverAssert(keyobj.refcount == 1);
     }
 }

From dfe126f3e92c9770bef1915b6e64add2c41edcfa Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 10 Feb 2020 17:18:11 +0100
Subject: [PATCH 0105/1098] Tracking: BCAST: parsing of the options + skeleton.

---
 src/networking.c | 63 +++++++++++++++++++++++++++++++++++++-----------
 src/server.c     |  5 +++-
 src/server.h     |  8 +++++-
 src/tracking.c   | 16 +++++++++---
 4 files changed, 73 insertions(+), 19 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 2b0f7464a..690a134f1 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -154,6 +154,7 @@ client *createClient(connection *conn) {
     c->peerid = NULL;
     c->client_list_node = NULL;
     c->client_tracking_redirection = 0;
+    c->client_tracking_prefix_nodes = NULL;
     c->auth_callback = NULL;
     c->auth_callback_privdata = NULL;
     c->auth_module = NULL;
@@ -2219,38 +2220,72 @@ NULL
                 UNIT_MILLISECONDS) != C_OK) return;
         pauseClients(duration);
         addReply(c,shared.ok);
-    } else if (!strcasecmp(c->argv[1]->ptr,"tracking") &&
-               (c->argc == 3 || c->argc == 5))
-    {
-        /* CLIENT TRACKING (on|off) [REDIRECT <id>] */
+    } else if (!strcasecmp(c->argv[1]->ptr,"tracking") && c->argc >= 3) {
+        /* CLIENT TRACKING (on|off) [REDIRECT <id>] [BCAST] [PREFIX first]
+         *                          [PREFIX second] ... */
         long long redir = 0;
+        int bcast = 0;
+        robj **prefix;
+        size_t numprefix = 0;
 
-        /* Parse the redirection option: we'll require the client with
-         * the specified ID to exist right now, even if it is possible
-         * it will get disconnected later. */
-        if (c->argc == 5) {
-            if (strcasecmp(c->argv[3]->ptr,"redirect") != 0) {
-                addReply(c,shared.syntaxerr);
-                return;
-            } else {
-                if (getLongLongFromObjectOrReply(c,c->argv[4],&redir,NULL) !=
+        /* Parse the options. */
+        if (for int j = 3; j < argc; j++) {
+            int moreargs = (c->argc-1) - j;
+
+            if (!strcasecmp(c->argv[j]->ptr,"redirect") && moreargs) {
+                j++;
+                if (getLongLongFromObjectOrReply(c,c->argv[j],&redir,NULL) !=
                     C_OK) return;
+                /* We will require the client with the specified ID to exist
+                 * right now, even if it is possible that it gets disconnected
+                 * later. Still a valid sanity check. */
                 if (lookupClientByID(redir) == NULL) {
                     addReplyError(c,"The client ID you want redirect to "
                                     "does not exist");
                     return;
                 }
+            } else if (!strcasecmp(c->argv[j]->ptr,"bcast")) {
+                bcast++;
+            } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && morearg) {
+                j++;
+                prefix = zrealloc(sizeof(robj*)*(numprefix+1));
+                prefix[numprefix++] = argv[j];
+            } else {
+                addReply(c,shared.syntaxerr);
+                return;
             }
         }
 
+        /* Make sure options are compatible among each other and with the
+         * current state of the client. */
+        if (!bcast && numprefix) {
+            addReplyError("PREFIX option requires BCAST mode to be enabled");
+            zfree(prefix);
+            return;
+        }
+
+        if (client->flags & CLIENT_TRACKING) {
+            int oldbcast = !!client->flags & CLIENT_TRACKING_BCAST;
+            if (oldbcast != bcast) {
+            }
+            addReplyError(
+                "You can't switch BCAST mode on/off before disabling "
+                "tracking for this client, and then re-enabling it with "
+                "a different mode.");
+            zfree(prefix);
+            return;
+        }
+
+        /* Options are ok: enable or disable the tracking for this client. */
         if (!strcasecmp(c->argv[2]->ptr,"on")) {
-            enableTracking(c,redir);
+            enableTracking(c,redir,bcast,prefix,numprefix);
         } else if (!strcasecmp(c->argv[2]->ptr,"off")) {
             disableTracking(c);
         } else {
             addReply(c,shared.syntaxerr);
             return;
         }
+        zfree(prefix);
         addReply(c,shared.ok);
     } else if (!strcasecmp(c->argv[1]->ptr,"getredir") && c->argc == 2) {
         /* CLIENT GETREDIR */
diff --git a/src/server.c b/src/server.c
index 910cf5410..1001fa4f7 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3310,8 +3310,11 @@ void call(client *c, int flags) {
     if (c->cmd->flags & CMD_READONLY) {
         client *caller = (c->flags & CLIENT_LUA && server.lua_caller) ?
                             server.lua_caller : c;
-        if (caller->flags & CLIENT_TRACKING)
+        if (caller->flags & CLIENT_TRACKING &&
+            !(caller->flags & CLIENT_TRACKING_BCAST))
+        {
             trackingRememberKeys(caller);
+        }
     }
 
     server.fixed_time_expire--;
diff --git a/src/server.h b/src/server.h
index 3e055a7db..d3ca0d01b 100644
--- a/src/server.h
+++ b/src/server.h
@@ -247,6 +247,7 @@ typedef long long ustime_t; /* microsecond time type. */
 #define CLIENT_TRACKING (1ULL<<31) /* Client enabled keys tracking in order to
                                    perform client side caching. */
 #define CLIENT_TRACKING_BROKEN_REDIR (1ULL<<32) /* Target client is invalid. */
+#define CLIENT_TRACKING_BCAST (1ULL<<33) /* Tracking in BCAST mode. */
 
 /* Client block type (btype field in client structure)
  * if CLIENT_BLOCKED flag is set. */
@@ -822,6 +823,11 @@ typedef struct client {
      * invalidation messages for keys fetched by this client will be send to
      * the specified client ID. */
     uint64_t client_tracking_redirection;
+    list *client_tracking_prefix_nodes; /* This list contains listNode pointers
+                                           to the nodes we have in every list
+                                           of clients in the tracking bcast
+                                           table. This way we can remove our
+                                           client in O(1) for each list. */
 
     /* Response buffer */
     int bufpos;
@@ -1648,7 +1654,7 @@ void addReplyStatusFormat(client *c, const char *fmt, ...);
 #endif
 
 /* Client side caching (tracking mode) */
-void enableTracking(client *c, uint64_t redirect_to);
+void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, size_t numprefix);
 void disableTracking(client *c);
 void trackingRememberKeys(client *c);
 void trackingInvalidateKey(robj *keyobj);
diff --git a/src/tracking.c b/src/tracking.c
index 3122563ac..413b21328 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -42,6 +42,7 @@
  * Clients will normally take frequently requested objects in memory, removing
  * them when invalidation messages are received. */
 rax *TrackingTable = NULL;
+rax *PrefixTable = NULL;
 uint64_t TrackingTableTotalItems = 0; /* Total number of IDs stored across
                                          the whole tracking table. This givesn
                                          an hint about the total memory we
@@ -68,16 +69,25 @@ void disableTracking(client *c) {
  * eventually get freed, we'll send a message to the original client to
  * inform it of the condition. Multiple clients can redirect the invalidation
  * messages to the same client ID. */
-void enableTracking(client *c, uint64_t redirect_to) {
-    if (c->flags & CLIENT_TRACKING) return;
+void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, size_t numprefix) {
     c->flags |= CLIENT_TRACKING;
     c->flags &= ~CLIENT_TRACKING_BROKEN_REDIR;
     c->client_tracking_redirection = redirect_to;
-    server.tracking_clients++;
+    if (!(c->flags & CLIENT_TRACKING)) server.tracking_clients++;
     if (TrackingTable == NULL) {
         TrackingTable = raxNew();
+        PrefixTable = raxNew();
         TrackingChannelName = createStringObject("__redis__:invalidate",20);
     }
+
+    if (bcast) {
+        c->flags |= CLIENT_TRACKING_BCAST;
+        if (numprefix == 0) enableBcastTrackingForPrefix(c,"",0);
+        for (int j = 0; j < numprefix; j++) {
+            sds sdsprefix = prefix[j]->ptr;
+            enableBcastTrackingForPrefix(c,sdsprefix,sdslen(prefix));
+        }
+    }
 }
 
 /* This function is called after the excution of a readonly command in the

From 3f7ba86255b9d6acd73dd39cc8f05d3d3f8741a9 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 11 Feb 2020 17:26:27 +0100
Subject: [PATCH 0106/1098] Tracking: BCAST: registration in the prefix table.

---
 src/networking.c | 21 +++++++++---------
 src/server.h     |  9 +++-----
 src/tracking.c   | 57 +++++++++++++++++++++++++++++++++++++++++++++---
 3 files changed, 67 insertions(+), 20 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 690a134f1..344b76260 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -154,7 +154,7 @@ client *createClient(connection *conn) {
     c->peerid = NULL;
     c->client_list_node = NULL;
     c->client_tracking_redirection = 0;
-    c->client_tracking_prefix_nodes = NULL;
+    c->client_tracking_prefixes = NULL;
     c->auth_callback = NULL;
     c->auth_callback_privdata = NULL;
     c->auth_module = NULL;
@@ -2028,7 +2028,6 @@ int clientSetNameOrReply(client *c, robj *name) {
 void clientCommand(client *c) {
     listNode *ln;
     listIter li;
-    client *client;
 
     if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
         const char *help[] = {
@@ -2142,7 +2141,7 @@ NULL
         /* Iterate clients killing all the matching clients. */
         listRewind(server.clients,&li);
         while ((ln = listNext(&li)) != NULL) {
-            client = listNodeValue(ln);
+            client *client = listNodeValue(ln);
             if (addr && strcmp(getClientPeerId(client),addr) != 0) continue;
             if (type != -1 && getClientType(client) != type) continue;
             if (id != 0 && client->id != id) continue;
@@ -2229,7 +2228,7 @@ NULL
         size_t numprefix = 0;
 
         /* Parse the options. */
-        if (for int j = 3; j < argc; j++) {
+        for (int j = 3; j < c->argc; j++) {
             int moreargs = (c->argc-1) - j;
 
             if (!strcasecmp(c->argv[j]->ptr,"redirect") && moreargs) {
@@ -2246,10 +2245,10 @@ NULL
                 }
             } else if (!strcasecmp(c->argv[j]->ptr,"bcast")) {
                 bcast++;
-            } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && morearg) {
+            } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && moreargs) {
                 j++;
-                prefix = zrealloc(sizeof(robj*)*(numprefix+1));
-                prefix[numprefix++] = argv[j];
+                prefix = zrealloc(prefix,sizeof(robj*)*(numprefix+1));
+                prefix[numprefix++] = c->argv[j];
             } else {
                 addReply(c,shared.syntaxerr);
                 return;
@@ -2259,16 +2258,16 @@ NULL
         /* Make sure options are compatible among each other and with the
          * current state of the client. */
         if (!bcast && numprefix) {
-            addReplyError("PREFIX option requires BCAST mode to be enabled");
+            addReplyError(c,"PREFIX option requires BCAST mode to be enabled");
             zfree(prefix);
             return;
         }
 
-        if (client->flags & CLIENT_TRACKING) {
-            int oldbcast = !!client->flags & CLIENT_TRACKING_BCAST;
+        if (c->flags & CLIENT_TRACKING) {
+            int oldbcast = !!c->flags & CLIENT_TRACKING_BCAST;
             if (oldbcast != bcast) {
             }
-            addReplyError(
+            addReplyError(c,
                 "You can't switch BCAST mode on/off before disabling "
                 "tracking for this client, and then re-enabling it with "
                 "a different mode.");
diff --git a/src/server.h b/src/server.h
index d3ca0d01b..725c3cbc8 100644
--- a/src/server.h
+++ b/src/server.h
@@ -823,12 +823,9 @@ typedef struct client {
      * invalidation messages for keys fetched by this client will be send to
      * the specified client ID. */
     uint64_t client_tracking_redirection;
-    list *client_tracking_prefix_nodes; /* This list contains listNode pointers
-                                           to the nodes we have in every list
-                                           of clients in the tracking bcast
-                                           table. This way we can remove our
-                                           client in O(1) for each list. */
-
+    rax *client_tracking_prefixes; /* A dictionary of prefixes we are already
+                                      subscribed to in BCAST mode, in the
+                                      context of client side caching. */
     /* Response buffer */
     int bufpos;
     char buf[PROTO_REPLY_CHUNK_BYTES];
diff --git a/src/tracking.c b/src/tracking.c
index 413b21328..9f46275a4 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -49,6 +49,15 @@ uint64_t TrackingTableTotalItems = 0; /* Total number of IDs stored across
                                          are using server side for CSC. */
 robj *TrackingChannelName;
 
+/* This is the structure that we have as value of the PrefixTable, and
+ * represents the list of keys modified, and the list of clients that need
+ * to be notified, for a given prefix. */
+typedef struct bcastState {
+    rax *keys;      /* Keys modified in the current event loop cycle. */
+    rax *clients;   /* Clients subscribed to the notification events for this
+                       prefix. */
+} bcastState;
+
 /* Remove the tracking state from the client 'c'. Note that there is not much
  * to do for us here, if not to decrement the counter of the clients in
  * tracking mode, because we just store the ID of the client in the tracking
@@ -56,9 +65,51 @@ robj *TrackingChannelName;
  * client with many entries in the table is removed, it would cost a lot of
  * time to do the cleanup. */
 void disableTracking(client *c) {
+    /* If this client is in broadcasting mode, we need to unsubscribe it
+     * from all the prefixes it is registered to. */
+    if (c->flags & CLIENT_TRACKING_BCAST) {
+        raxIterator ri;
+        raxStart(&ri,c->client_tracking_prefixes);
+        raxSeek(&ri,"^",NULL,0);
+        while(raxNext(&ri)) {
+            bcastState *bs = raxFind(PrefixTable,ri.key,ri.key_len);
+            serverAssert(bs != raxNotFound);
+            raxRemove(bs->clients,(unsigned char*)&c,sizeof(c),NULL);
+            /* Was it the last client? Remove the prefix from the
+             * table. */
+            if (raxSize(bs->clients) == 0) {
+                raxFree(bs->clients);
+                raxFree(bs->keys);
+                zfree(bs);
+                raxRemove(PrefixTable,ri.key,ri.key_len,NULL);
+            }
+        }
+        raxStop(&ri);
+    }
+
+    /* Clear flags and adjust the count. */
     if (c->flags & CLIENT_TRACKING) {
         server.tracking_clients--;
-        c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR);
+        c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR|
+                      CLIENT_TRACKING_BCAST);
+    }
+}
+
+/* Set the client 'c' to track the prefix 'prefix'. If the client 'c' is
+ * already registered for the specified prefix, no operation is performed. */
+void enableBcastTrackingForPrefix(client *c, char *prefix, size_t plen) {
+    bcastState *bs = raxFind(PrefixTable,(unsigned char*)prefix,sdslen(prefix));
+    /* If this is the first client subscribing to such prefix, create
+     * the prefix in the table. */
+    if (bs == raxNotFound) {
+        bs = zmalloc(sizeof(*bs));
+        bs->keys = raxNew();
+        bs->clients = raxNew();
+        raxInsert(PrefixTable,(unsigned char*)prefix,plen,bs,NULL);
+    }
+    if (raxTryInsert(bs->clients,(unsigned char*)&c,sizeof(c),NULL,NULL)) {
+        raxInsert(c->client_tracking_prefixes,
+                  (unsigned char*)prefix,plen,NULL,NULL);
     }
 }
 
@@ -83,9 +134,9 @@ void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, s
     if (bcast) {
         c->flags |= CLIENT_TRACKING_BCAST;
         if (numprefix == 0) enableBcastTrackingForPrefix(c,"",0);
-        for (int j = 0; j < numprefix; j++) {
+        for (size_t j = 0; j < numprefix; j++) {
             sds sdsprefix = prefix[j]->ptr;
-            enableBcastTrackingForPrefix(c,sdsprefix,sdslen(prefix));
+            enableBcastTrackingForPrefix(c,sdsprefix,sdslen(sdsprefix));
         }
     }
 }

From 71f3f3f1afe4fbb6f8634970258b5dec2d389c68 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 11 Feb 2020 18:11:59 +0100
Subject: [PATCH 0107/1098] Tracking: BCAST: broadcasting of keys in prefixes
 implemented.

---
 src/server.h   |   1 +
 src/tracking.c | 104 ++++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 95 insertions(+), 10 deletions(-)

diff --git a/src/server.h b/src/server.h
index 725c3cbc8..439bbc393 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1659,6 +1659,7 @@ void trackingInvalidateKeysOnFlush(int dbid);
 void trackingLimitUsedSlots(void);
 uint64_t trackingGetTotalItems(void);
 uint64_t trackingGetTotalKeys(void);
+void trackingBroadcastInvalidationMessages(void);
 
 /* List data type */
 void listTypeTryConversion(robj *subject, robj *value);
diff --git a/src/tracking.c b/src/tracking.c
index 9f46275a4..345c5f1ad 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -167,7 +167,17 @@ void trackingRememberKeys(client *c) {
     getKeysFreeResult(keys);
 }
 
-void sendTrackingMessage(client *c, char *keyname, size_t keylen) {
+/* Given a key name, this function sends an invalidation message in the
+ * proper channel (depending on RESP version: PubSub or Push message) and
+ * to the proper client (in case fo redirection), in the context of the
+ * client 'c' with tracking enabled.
+ *
+ * In case the 'proto' argument is non zero, the function will assume that
+ * 'keyname' points to a buffer of 'keylen' bytes already expressed in the
+ * form of Redis RESP protocol, representing an array of keys to send
+ * to the client as value of the invalidation. This is used in BCAST mode
+ * in order to optimized the implementation to use less CPU time. */
+void sendTrackingMessage(client *c, char *keyname, size_t keylen, int proto) {
     int using_redirection = 0;
     if (c->client_tracking_redirection) {
         client *redir = lookupClientByID(c->client_tracking_redirection);
@@ -193,18 +203,38 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen) {
     if (c->resp > 2) {
         addReplyPushLen(c,2);
         addReplyBulkCBuffer(c,"invalidate",10);
-        addReplyArrayLen(c,1);
-        addReplyBulkCBuffer(c,keyname,keylen);
     } else if (using_redirection && c->flags & CLIENT_PUBSUB) {
         /* We use a static object to speedup things, however we assume
          * that addReplyPubsubMessage() will not take a reference. */
-        robj keyobj;
-        initStaticStringObject(keyobj,keyname);
         addReplyPubsubMessage(c,TrackingChannelName,NULL);
-        addReplyArrayLen(c,1);
-        addReplyBulk(c,&keyobj);
-        serverAssert(keyobj.refcount == 1);
     }
+
+    /* Send the "value" part, which is the array of keys. */
+    if (proto) {
+        addReplyProto(c,keyname,keylen);
+    } else {
+        addReplyArrayLen(c,1);
+        addReplyBulkCBuffer(c,keyname,keylen);
+    }
+}
+
+/* This function is called when a key is modified in Redis and in the case
+ * we have at least one client with the BCAST mode enabled.
+ * Its goal is to set the key in the right broadcast state if the key
+ * matches one or more prefixes in the prefix table. Later when we
+ * return to the event loop, we'll send invalidation messages to the
+ * clients subscribed to each prefix. */
+void trackingRememberKeyToBroadcast(char *keyname, size_t keylen) {
+    raxIterator ri;
+    raxStart(&ri,PrefixTable);
+    raxSeek(&ri,"^",NULL,0);
+    while(raxNext(&ri)) {
+         if (keylen > ri.key_len) continue;
+         if (memcmp(ri.key,keyname,ri.key_len) != 0) continue;
+         bcastState *bs = ri.data;
+         raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,NULL,NULL);
+    }
+    raxStop(&ri);
 }
 
 /* This function is called from signalModifiedKey() or other places in Redis
@@ -214,6 +244,10 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen) {
 void trackingInvalidateKey(robj *keyobj) {
     if (TrackingTable == NULL) return;
     sds sdskey = keyobj->ptr;
+
+    if (raxSize(PrefixTable) > 0)
+        trackingRememberKeyToBroadcast(sdskey,sdslen(sdskey));
+
     rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey));
     if (ids == raxNotFound) return;;
 
@@ -225,7 +259,7 @@ void trackingInvalidateKey(robj *keyobj) {
         memcpy(&id,ri.key,sizeof(id));
         client *c = lookupClientByID(id);
         if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue;
-        sendTrackingMessage(c,sdskey,sdslen(sdskey));
+        sendTrackingMessage(c,sdskey,sdslen(sdskey),0);
     }
     raxStop(&ri);
 
@@ -262,7 +296,7 @@ void trackingInvalidateKeysOnFlush(int dbid) {
         while ((ln = listNext(&li)) != NULL) {
             client *c = listNodeValue(ln);
             if (c->flags & CLIENT_TRACKING) {
-                sendTrackingMessage(c,"",1);
+                sendTrackingMessage(c,"",1,0);
             }
         }
     }
@@ -323,6 +357,56 @@ void trackingLimitUsedSlots(void) {
     timeout_counter++;
 }
 
+/* This function will run the prefixes of clients in BCAST mode and
+ * keys that were modified about each prefix, and will send the
+ * notifications to each client in each prefix. */
+void trackingBroadcastInvalidationMessages(void) {
+    raxIterator ri, ri2;
+    raxStart(&ri,PrefixTable);
+    raxSeek(&ri,"^",NULL,0);
+    while(raxNext(&ri)) {
+        bcastState *bs = ri.data;
+        /* Create the array reply with the list of keys once, then send
+        * it to all the clients subscribed to this prefix. */
+        char buf[32];
+        size_t len = ll2string(buf,sizeof(buf),raxSize(bs->keys));
+        sds proto = sdsempty();
+        proto = sdsMakeRoomFor(proto,raxSize(bs->keys)*15);
+        proto = sdscatlen(proto,"*",1);
+        proto = sdscatlen(proto,buf,len);
+        proto = sdscatlen(proto,"\r\n",2);
+        raxStart(&ri2,bs->keys);
+        raxSeek(&ri2,"^",NULL,0);
+        while(raxNext(&ri2)) {
+            len = ll2string(buf,sizeof(buf),ri2.key_len);
+            sds proto = sdsnewlen("$",1);
+            proto = sdscatlen(proto,ri2.key,ri2.key_len);
+            proto = sdscatlen(proto,"\r\n",2);
+        }
+        raxStop(&ri2);
+
+        /* Send this array of keys to every client in the list. */
+        raxStart(&ri2,bs->clients);
+        raxSeek(&ri2,"^",NULL,0);
+        while(raxNext(&ri2)) {
+            client *c;
+            memcpy(&c,ri2.key,sizeof(c));
+            sendTrackingMessage(c,proto,sdslen(proto),1);
+        }
+        raxStop(&ri2);
+
+        /* Clean up: we can remove everything from this state, because we
+         * want to only track the new keys that will be accumulated starting
+         * from now. */
+        sdsfree(proto);
+        raxFree(bs->clients);
+        raxFree(bs->keys);
+        bs->clients = raxNew();
+        bs->keys = raxNew();
+    }
+    raxStop(&ri);
+}
+
 /* This is just used in order to access the amount of used slots in the
  * tracking table. */
 uint64_t trackingGetTotalItems(void) {

From dc8f947d7c8a26881c3a5789a2dbf8d3a5ef946d Mon Sep 17 00:00:00 2001
From: lifubang <lifubang@acmcoder.com>
Date: Wed, 12 Feb 2020 16:34:22 +0800
Subject: [PATCH 0108/1098] correct help info for --user and --pass

Signed-off-by: lifubang <lifubang@acmcoder.com>
---
 src/redis-cli.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index c1bda0a00..1d79e0db0 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -1688,8 +1688,8 @@ static void usage(void) {
 "                     You can also use the " REDIS_CLI_AUTH_ENV " environment\n"
 "                     variable to pass this password more safely\n"
 "                     (if both are used, this argument takes predecence).\n"
-"  -user <username>   Used to send ACL style 'AUTH username pass'. Needs -a.\n"
-"  -pass <password>   Alias of -a for consistency with the new --user option.\n"
+"  --user <username>  Used to send ACL style 'AUTH username pass'. Needs -a.\n"
+"  --pass <password>  Alias of -a for consistency with the new --user option.\n"
 "  -u <uri>           Server URI.\n"
 "  -r <repeat>        Execute specified command N times.\n"
 "  -i <interval>      When -r is used, waits <interval> seconds per command.\n"

From 40194a2a6809520b5f01da4a7b41afe2a2441f64 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 12 Feb 2020 19:22:04 +0100
Subject: [PATCH 0109/1098] Tracking: BCAST: basic feature now works.

---
 src/networking.c |  2 +-
 src/server.c     |  4 +++
 src/tracking.c   | 89 +++++++++++++++++++++++++++---------------------
 3 files changed, 55 insertions(+), 40 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 344b76260..46534253e 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2224,7 +2224,7 @@ NULL
          *                          [PREFIX second] ... */
         long long redir = 0;
         int bcast = 0;
-        robj **prefix;
+        robj **prefix = NULL;
         size_t numprefix = 0;
 
         /* Parse the options. */
diff --git a/src/server.c b/src/server.c
index 1001fa4f7..22c81070c 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2124,6 +2124,10 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     if (listLength(server.unblocked_clients))
         processUnblockedClients();
 
+    /* Send the invalidation messages to clients participating to the
+     * client side caching protocol in broadcasting (BCAST) mode. */
+    trackingBroadcastInvalidationMessages();
+
     /* Write the AOF buffer on disk */
     flushAppendOnlyFile(0);
 
diff --git a/src/tracking.c b/src/tracking.c
index 345c5f1ad..672b886a3 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -85,6 +85,8 @@ void disableTracking(client *c) {
             }
         }
         raxStop(&ri);
+        raxFree(c->client_tracking_prefixes);
+        c->client_tracking_prefixes = NULL;
     }
 
     /* Clear flags and adjust the count. */
@@ -108,6 +110,8 @@ void enableBcastTrackingForPrefix(client *c, char *prefix, size_t plen) {
         raxInsert(PrefixTable,(unsigned char*)prefix,plen,bs,NULL);
     }
     if (raxTryInsert(bs->clients,(unsigned char*)&c,sizeof(c),NULL,NULL)) {
+        if (c->client_tracking_prefixes == NULL)
+            c->client_tracking_prefixes = raxNew();
         raxInsert(c->client_tracking_prefixes,
                   (unsigned char*)prefix,plen,NULL,NULL);
     }
@@ -121,10 +125,10 @@ void enableBcastTrackingForPrefix(client *c, char *prefix, size_t plen) {
  * inform it of the condition. Multiple clients can redirect the invalidation
  * messages to the same client ID. */
 void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, size_t numprefix) {
-    c->flags |= CLIENT_TRACKING;
-    c->flags &= ~CLIENT_TRACKING_BROKEN_REDIR;
-    c->client_tracking_redirection = redirect_to;
     if (!(c->flags & CLIENT_TRACKING)) server.tracking_clients++;
+    c->flags |= CLIENT_TRACKING;
+    c->flags &= ~(CLIENT_TRACKING_BROKEN_REDIR|CLIENT_TRACKING_BCAST);
+    c->client_tracking_redirection = redirect_to;
     if (TrackingTable == NULL) {
         TrackingTable = raxNew();
         PrefixTable = raxNew();
@@ -229,10 +233,11 @@ void trackingRememberKeyToBroadcast(char *keyname, size_t keylen) {
     raxStart(&ri,PrefixTable);
     raxSeek(&ri,"^",NULL,0);
     while(raxNext(&ri)) {
-         if (keylen > ri.key_len) continue;
-         if (memcmp(ri.key,keyname,ri.key_len) != 0) continue;
-         bcastState *bs = ri.data;
-         raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,NULL,NULL);
+        if (ri.key_len > keylen) continue;
+        if (ri.key_len != 0 && memcmp(ri.key,keyname,ri.key_len) != 0)
+            continue;
+        bcastState *bs = ri.data;
+        raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,NULL,NULL);
     }
     raxStop(&ri);
 }
@@ -362,46 +367,52 @@ void trackingLimitUsedSlots(void) {
  * notifications to each client in each prefix. */
 void trackingBroadcastInvalidationMessages(void) {
     raxIterator ri, ri2;
+
+    /* Return ASAP if there is nothing to do here. */
+    if (TrackingTable == NULL || !server.tracking_clients) return;
+
     raxStart(&ri,PrefixTable);
     raxSeek(&ri,"^",NULL,0);
     while(raxNext(&ri)) {
         bcastState *bs = ri.data;
-        /* Create the array reply with the list of keys once, then send
-        * it to all the clients subscribed to this prefix. */
-        char buf[32];
-        size_t len = ll2string(buf,sizeof(buf),raxSize(bs->keys));
-        sds proto = sdsempty();
-        proto = sdsMakeRoomFor(proto,raxSize(bs->keys)*15);
-        proto = sdscatlen(proto,"*",1);
-        proto = sdscatlen(proto,buf,len);
-        proto = sdscatlen(proto,"\r\n",2);
-        raxStart(&ri2,bs->keys);
-        raxSeek(&ri2,"^",NULL,0);
-        while(raxNext(&ri2)) {
-            len = ll2string(buf,sizeof(buf),ri2.key_len);
-            sds proto = sdsnewlen("$",1);
-            proto = sdscatlen(proto,ri2.key,ri2.key_len);
+        if (raxSize(bs->keys)) {
+            /* Create the array reply with the list of keys once, then send
+            * it to all the clients subscribed to this prefix. */
+            char buf[32];
+            size_t len = ll2string(buf,sizeof(buf),raxSize(bs->keys));
+            sds proto = sdsempty();
+            proto = sdsMakeRoomFor(proto,raxSize(bs->keys)*15);
+            proto = sdscatlen(proto,"*",1);
+            proto = sdscatlen(proto,buf,len);
             proto = sdscatlen(proto,"\r\n",2);
-        }
-        raxStop(&ri2);
+            raxStart(&ri2,bs->keys);
+            raxSeek(&ri2,"^",NULL,0);
+            while(raxNext(&ri2)) {
+                len = ll2string(buf,sizeof(buf),ri2.key_len);
+                proto = sdscatlen(proto,"$",1);
+                proto = sdscatlen(proto,buf,len);
+                proto = sdscatlen(proto,"\r\n",2);
+                proto = sdscatlen(proto,ri2.key,ri2.key_len);
+                proto = sdscatlen(proto,"\r\n",2);
+            }
+            raxStop(&ri2);
 
-        /* Send this array of keys to every client in the list. */
-        raxStart(&ri2,bs->clients);
-        raxSeek(&ri2,"^",NULL,0);
-        while(raxNext(&ri2)) {
-            client *c;
-            memcpy(&c,ri2.key,sizeof(c));
-            sendTrackingMessage(c,proto,sdslen(proto),1);
-        }
-        raxStop(&ri2);
+            /* Send this array of keys to every client in the list. */
+            raxStart(&ri2,bs->clients);
+            raxSeek(&ri2,"^",NULL,0);
+            while(raxNext(&ri2)) {
+                client *c;
+                memcpy(&c,ri2.key,sizeof(c));
+                sendTrackingMessage(c,proto,sdslen(proto),1);
+            }
+            raxStop(&ri2);
 
-        /* Clean up: we can remove everything from this state, because we
-         * want to only track the new keys that will be accumulated starting
-         * from now. */
-        sdsfree(proto);
-        raxFree(bs->clients);
+            /* Clean up: we can remove everything from this state, because we
+             * want to only track the new keys that will be accumulated starting
+             * from now. */
+            sdsfree(proto);
+        }
         raxFree(bs->keys);
-        bs->clients = raxNew();
         bs->keys = raxNew();
     }
     raxStop(&ri);

From 6922ccc0b98156e787b3d2f35daf0299e7844250 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 13 Feb 2020 16:58:07 +0100
Subject: [PATCH 0110/1098] Tracking: fix sending messages bug + tracking off
 bug.

---
 src/networking.c | 42 ++++++++++++++++++++++--------------------
 src/tracking.c   |  6 ++++++
 2 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 46534253e..1b4b19645 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2250,37 +2250,39 @@ NULL
                 prefix = zrealloc(prefix,sizeof(robj*)*(numprefix+1));
                 prefix[numprefix++] = c->argv[j];
             } else {
+                zfree(prefix);
                 addReply(c,shared.syntaxerr);
                 return;
             }
         }
 
-        /* Make sure options are compatible among each other and with the
-         * current state of the client. */
-        if (!bcast && numprefix) {
-            addReplyError(c,"PREFIX option requires BCAST mode to be enabled");
-            zfree(prefix);
-            return;
-        }
-
-        if (c->flags & CLIENT_TRACKING) {
-            int oldbcast = !!c->flags & CLIENT_TRACKING_BCAST;
-            if (oldbcast != bcast) {
-            }
-            addReplyError(c,
-                "You can't switch BCAST mode on/off before disabling "
-                "tracking for this client, and then re-enabling it with "
-                "a different mode.");
-            zfree(prefix);
-            return;
-        }
-
         /* Options are ok: enable or disable the tracking for this client. */
         if (!strcasecmp(c->argv[2]->ptr,"on")) {
+            /* Before enabling tracking, make sure options are compatible
+             * among each other and with the current state of the client. */
+            if (!bcast && numprefix) {
+                addReplyError(c,
+                    "PREFIX option requires BCAST mode to be enabled");
+                zfree(prefix);
+                return;
+            }
+
+            if (c->flags & CLIENT_TRACKING) {
+                int oldbcast = !!c->flags & CLIENT_TRACKING_BCAST;
+                if (oldbcast != bcast) {
+                    addReplyError(c,
+                    "You can't switch BCAST mode on/off before disabling "
+                    "tracking for this client, and then re-enabling it with "
+                    "a different mode.");
+                    zfree(prefix);
+                    return;
+                }
+            }
             enableTracking(c,redir,bcast,prefix,numprefix);
         } else if (!strcasecmp(c->argv[2]->ptr,"off")) {
             disableTracking(c);
         } else {
+            zfree(prefix);
             addReply(c,shared.syntaxerr);
             return;
         }
diff --git a/src/tracking.c b/src/tracking.c
index 672b886a3..ef5840863 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -211,6 +211,12 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen, int proto) {
         /* We use a static object to speedup things, however we assume
          * that addReplyPubsubMessage() will not take a reference. */
         addReplyPubsubMessage(c,TrackingChannelName,NULL);
+    } else {
+        /* If are here, the client is not using RESP3, nor is
+         * redirecting to another client. We can't send anything to
+         * it since RESP2 does not support push messages in the same
+         * connection. */
+        return;
     }
 
     /* Send the "value" part, which is the array of keys. */

From f6e32a832f4aaa92721f4ea1eadc1d3897ba32c2 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 14 Feb 2020 14:17:10 +0100
Subject: [PATCH 0111/1098] Tracking: fix behavior when switchinig from normal
 to BCAST.

---
 src/tracking.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/tracking.c b/src/tracking.c
index ef5840863..3333472a4 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -269,7 +269,17 @@ void trackingInvalidateKey(robj *keyobj) {
         uint64_t id;
         memcpy(&id,ri.key,sizeof(id));
         client *c = lookupClientByID(id);
-        if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue;
+        /* Note that if the client is in BCAST mode, we don't want to
+         * send invalidation messages that were pending in the case
+         * previously the client was not in BCAST mode. This can happen if
+         * TRACKING is enabled normally, and then the client switches to
+         * BCAST mode. */
+        if (c == NULL ||
+            !(c->flags & CLIENT_TRACKING)||
+            c->flags & CLIENT_TRACKING_BCAST)
+        {
+            continue;
+        }
         sendTrackingMessage(c,sdskey,sdslen(sdskey),0);
     }
     raxStop(&ri);

From 47177c9edc9d6f738f1aacb33bd4e1d6c2c5a697 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 14 Feb 2020 14:27:22 +0100
Subject: [PATCH 0112/1098] Tracking: fix operators precedence error in bcast
 check.

---
 src/networking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index 1b4b19645..69350eed1 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2268,7 +2268,7 @@ NULL
             }
 
             if (c->flags & CLIENT_TRACKING) {
-                int oldbcast = !!c->flags & CLIENT_TRACKING_BCAST;
+                int oldbcast = !!(c->flags & CLIENT_TRACKING_BCAST);
                 if (oldbcast != bcast) {
                     addReplyError(c,
                     "You can't switch BCAST mode on/off before disabling "

From 8ea7a3ee686a8bddf0b07585922917adcfda91dc Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 14 Feb 2020 14:29:00 +0100
Subject: [PATCH 0113/1098] Tracking: first set of tests for the feature.

---
 tests/unit/tracking.tcl | 66 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 tests/unit/tracking.tcl

diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
new file mode 100644
index 000000000..2058319f7
--- /dev/null
+++ b/tests/unit/tracking.tcl
@@ -0,0 +1,66 @@
+start_server {tags {"tracking"}} {
+    # Create a deferred client we'll use to redirect invalidation
+    # messages to.
+    set rd1 [redis_deferring_client]
+    $rd1 client id
+    set redir [$rd1 read]
+    $rd1 subscribe __redis__:invalidate
+    $rd1 read ; # Consume the SUBSCRIBE reply.
+
+    test {Clients are able to enable tracking and redirect it} {
+        r CLIENT TRACKING on REDIRECT $redir
+    } {*OK}
+
+    test {The other connection is able to get invalidations} {
+        r SET a 1
+        r GET a
+        r INCR a
+        r INCR b ; # This key should not be notified, since it wasn't fetched.
+        set keys [lindex [$rd1 read] 2]
+        assert {[llength $keys] == 1}
+        assert {[lindex $keys 0] eq {a}}
+    }
+
+    test {The client is now able to disable tracking} {
+        # Make sure to add a few more keys in the tracking list
+        # so that we can check for leaks, as a side effect.
+        r MGET a b c d e f g
+        r CLIENT TRACKING off
+    }
+
+    test {Clients can enable the BCAST mode with the empty prefix} {
+        r CLIENT TRACKING on BCAST REDIRECT $redir
+    } {*OK*}
+
+    test {The connection gets invalidation messages about all the keys} {
+        r MSET a 1 b 2 c 3
+        set keys [lsort [lindex [$rd1 read] 2]]
+        assert {$keys eq {a b c}}
+    }
+
+    test {Clients can enable the BCAST mode with prefixes} {
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on BCAST REDIRECT $redir PREFIX a: PREFIX b:
+        r MULTI
+        r INCR a:1
+        r INCR a:2
+        r INCR b:1
+        r INCR b:2
+        r EXEC
+        # Because of the internals, we know we are going to receive
+        # two separated notifications for the two different prefixes.
+        set keys1 [lsort [lindex [$rd1 read] 2]]
+        set keys2 [lsort [lindex [$rd1 read] 2]]
+        set keys [lsort [list {*}$keys1 {*}$keys2]]
+        assert {$keys eq {a:1 a:2 b:1 b:2}}
+    }
+    
+    test {Adding prefixes to BCAST mode works} {
+        r CLIENT TRACKING on BCAST REDIRECT $redir PREFIX c:
+        r INCR c:1234
+        set keys [lsort [lindex [$rd1 read] 2]]
+        assert {$keys eq {c:1234}}
+    }
+
+    $rd1 close
+}

From 8a44b2cc7ee15fedbacceced5d4a64d1b2a917c5 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Fri, 14 Feb 2020 17:13:58 +0200
Subject: [PATCH 0114/1098] Fixes segfault on calling trackingGetTotalKeys

... with CSC disabled
---
 src/tracking.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/tracking.c b/src/tracking.c
index 3333472a4..7179a54f8 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -441,5 +441,6 @@ uint64_t trackingGetTotalItems(void) {
 }
 
 uint64_t trackingGetTotalKeys(void) {
+    if (TrackingTable == NULL) return 0;
     return raxSize(TrackingTable);
 }

From 20eeddfb8a8840aff8ac9e54a69650890d9c1e64 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 14 Feb 2020 18:22:25 +0100
Subject: [PATCH 0115/1098] Signal key as modified when expired on-access.

This fixes WATCH and client side caching with keys expiring because of
a synchronous access and not because of background expiring.
---
 src/db.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/db.c b/src/db.c
index c80524e94..8a0242d9e 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1296,8 +1296,10 @@ int expireIfNeeded(redisDb *db, robj *key) {
     propagateExpire(db,key,server.lazyfree_lazy_expire);
     notifyKeyspaceEvent(NOTIFY_EXPIRED,
         "expired",key,db->id);
-    return server.lazyfree_lazy_expire ? dbAsyncDelete(db,key) :
-                                         dbSyncDelete(db,key);
+    int retval = server.lazyfree_lazy_expire ? dbAsyncDelete(db,key) :
+                                               dbSyncDelete(db,key);
+    if (retval) signalModifiedKey(db,key);
+    return retval;
 }
 
 /* -----------------------------------------------------------------------------

From d63a43536277a8514593aa95b9c343c1b6d98cfa Mon Sep 17 00:00:00 2001
From: Madelyn Olson <matolson@amazon.com>
Date: Sun, 16 Feb 2020 05:16:51 -0800
Subject: [PATCH 0116/1098] Minor CSC fixes and fixed documentation

---
 src/networking.c | 11 ++++++++---
 src/tracking.c   | 22 ++++++++--------------
 2 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 69350eed1..dad61904d 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2044,7 +2044,7 @@ void clientCommand(client *c) {
 "REPLY (on|off|skip)    -- Control the replies sent to the current connection.",
 "SETNAME <name>         -- Assign the name <name> to the current connection.",
 "UNBLOCK <clientid> [TIMEOUT|ERROR] -- Unblock the specified blocked client.",
-"TRACKING (on|off) [REDIRECT <id>] -- Enable client keys tracking for client side caching.",
+"TRACKING (on|off) [REDIRECT <id>] [BCAST] [PREFIX first] [PREFIX second] ... -- Enable client keys tracking for client side caching.",
 "GETREDIR               -- Return the client ID we are redirecting to when tracking is enabled.",
 NULL
         };
@@ -2234,17 +2234,22 @@ NULL
             if (!strcasecmp(c->argv[j]->ptr,"redirect") && moreargs) {
                 j++;
                 if (getLongLongFromObjectOrReply(c,c->argv[j],&redir,NULL) !=
-                    C_OK) return;
+                    C_OK)
+                {
+                    zfree(prefix);
+                    return;
+                }
                 /* We will require the client with the specified ID to exist
                  * right now, even if it is possible that it gets disconnected
                  * later. Still a valid sanity check. */
                 if (lookupClientByID(redir) == NULL) {
                     addReplyError(c,"The client ID you want redirect to "
                                     "does not exist");
+                    zfree(prefix);
                     return;
                 }
             } else if (!strcasecmp(c->argv[j]->ptr,"bcast")) {
-                bcast++;
+                bcast = 1;
             } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && moreargs) {
                 j++;
                 prefix = zrealloc(prefix,sizeof(robj*)*(numprefix+1));
diff --git a/src/tracking.c b/src/tracking.c
index 7179a54f8..619148f2f 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -44,7 +44,7 @@
 rax *TrackingTable = NULL;
 rax *PrefixTable = NULL;
 uint64_t TrackingTableTotalItems = 0; /* Total number of IDs stored across
-                                         the whole tracking table. This givesn
+                                         the whole tracking table. This gives
                                          an hint about the total memory we
                                          are using server side for CSC. */
 robj *TrackingChannelName;
@@ -145,9 +145,9 @@ void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, s
     }
 }
 
-/* This function is called after the excution of a readonly command in the
+/* This function is called after the execution of a readonly command in the
  * case the client 'c' has keys tracking enabled. It will populate the
- * tracking ivalidation table according to the keys the user fetched, so that
+ * tracking invalidation table according to the keys the user fetched, so that
  * Redis will know what are the clients that should receive an invalidation
  * message with certain groups of keys are modified. */
 void trackingRememberKeys(client *c) {
@@ -292,19 +292,12 @@ void trackingInvalidateKey(robj *keyobj) {
 }
 
 /* This function is called when one or all the Redis databases are flushed
- * (dbid == -1 in case of FLUSHALL). Caching slots are not specific for
- * each DB but are global: currently what we do is sending a special
+ * (dbid == -1 in case of FLUSHALL). Caching keys are not specific for
+ * each DB but are global: currently what we do is send a special
  * notification to clients with tracking enabled, invalidating the caching
- * slot "-1", which means, "all the keys", in order to avoid flooding clients
+ * key "", which means, "all the keys", in order to avoid flooding clients
  * with many invalidation messages for all the keys they may hold.
- *
- * However trying to flush the tracking table here is very costly:
- * we need scanning 16 million caching slots in the table to check
- * if they are used, this introduces a big delay. So what we do is to really
- * flush the table in the case of FLUSHALL. When a FLUSHDB is called instead
- * we just send the invalidation message to all the clients, but don't
- * flush the table: it will slowly get garbage collected as more keys
- * are modified in the used caching slots. */
+ */
 void freeTrackingRadixTree(void *rt) {
     raxFree(rt);
 }
@@ -325,6 +318,7 @@ void trackingInvalidateKeysOnFlush(int dbid) {
     /* In case of FLUSHALL, reclaim all the memory used by tracking. */
     if (dbid == -1 && TrackingTable) {
         raxFreeWithCallback(TrackingTable,freeTrackingRadixTree);
+        TrackingTable = raxNew();
         TrackingTableTotalItems = 0;
     }
 }

From c1b5220ee74d6671ae4bbf9c0aa6e44aab83a319 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <matolson@amazon.com>
Date: Sun, 16 Feb 2020 05:41:39 -0800
Subject: [PATCH 0117/1098] Give an error message if you specify redirect twice

---
 src/networking.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index dad61904d..5b1229fde 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2233,6 +2233,13 @@ NULL
 
             if (!strcasecmp(c->argv[j]->ptr,"redirect") && moreargs) {
                 j++;
+                if (redir != 0) {
+                    addReplyError(c,"A client can only redirect to a single "
+                                    "other client");
+                    zfree(prefix);
+                    return;
+                }
+
                 if (getLongLongFromObjectOrReply(c,c->argv[j],&redir,NULL) !=
                     C_OK)
                 {

From 6fff2cf9b65effe2846d8383afa32dc0b04dd149 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 16 Feb 2020 15:43:19 +0200
Subject: [PATCH 0118/1098] module api docs for aux_save and aux_load

---
 src/module.c | 6 ++++++
 src/rdb.c    | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/module.c b/src/module.c
index aae15d74b..69101161d 100644
--- a/src/module.c
+++ b/src/module.c
@@ -3529,6 +3529,8 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) {
  *          // Optional fields
  *          .digest = myType_DigestCallBack,
  *          .mem_usage = myType_MemUsageCallBack,
+ *          .aux_load = myType_AuxRDBLoadCallBack,
+ *          .aux_save = myType_AuxRDBSaveCallBack,
  *      }
  *
  * * **rdb_load**: A callback function pointer that loads data from RDB files.
@@ -3536,6 +3538,10 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) {
  * * **aof_rewrite**: A callback function pointer that rewrites data as commands.
  * * **digest**: A callback function pointer that is used for `DEBUG DIGEST`.
  * * **free**: A callback function pointer that can free a type value.
+ * * **aux_save**: A callback function pointer that saves out of keyspace data to RDB files.
+ *   'when' argument is either REDISMODULE_AUX_BEFORE_RDB or REDISMODULE_AUX_AFTER_RDB.
+ * * **aux_load**: A callback function pointer that loads out of keyspace data from RDB files.
+ *   Similar to aux_save, returns REDISMODULE_OK on success, and ERR otherwise.
  *
  * The **digest* and **mem_usage** methods should currently be omitted since
  * they are not yet implemented inside the Redis modules core.
diff --git a/src/rdb.c b/src/rdb.c
index 61265433d..cbcea96c6 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2195,7 +2195,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
                 io.ver = 2;
                 /* Call the rdb_load method of the module providing the 10 bit
                  * encoding version in the lower 10 bits of the module ID. */
-                if (mt->aux_load(&io,moduleid&1023, when) || io.error) {
+                if (mt->aux_load(&io,moduleid&1023, when) != REDISMODULE_OK || io.error) {
                     moduleTypeNameByID(name,moduleid);
                     serverLog(LL_WARNING,"The RDB file contains module AUX data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name);
                     exit(1);

From 7d4ebe114fbf42e4474f0cb8db71f4b09d2d2c44 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Mon, 17 Feb 2020 23:40:24 -0500
Subject: [PATCH 0119/1098] add missing subcommand description for debug oom

---
 src/debug.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/debug.c b/src/debug.c
index b910d2d2d..dd96ad416 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -363,6 +363,7 @@ void debugCommand(client *c) {
 "LOADAOF -- Flush the AOF buffers on disk and reload the AOF in memory.",
 "LUA-ALWAYS-REPLICATE-COMMANDS <0|1> -- Setting it to 1 makes Lua replication defaulting to replicating single commands, without the script having to enable effects replication.",
 "OBJECT <key> -- Show low level info about key and associated value.",
+"OOM -- Crash the server simulating an out-of-memory error.",
 "PANIC -- Crash the server simulating a panic.",
 "POPULATE <count> [prefix] [size] -- Create <count> string keys named key:<num>. If a prefix is specified is used instead of the 'key' prefix.",
 "RELOAD -- Save the RDB on disk and reload it back in memory.",

From 485425cec76e2bfeb3e11503c4519f3d526bc09e Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 18 Feb 2020 16:19:52 +0200
Subject: [PATCH 0120/1098] Defrag big lists in portions to avoid latency and
 freeze

When active defrag kicks in and finds a big list, it will create a bookmark to
a node so that it is able to resume iteration from that node later.

The quicklist manages that bookmark, and updates it in case that node is deleted.

This will increase memory usage only on lists of over 1000 (see
active-defrag-max-scan-fields) quicklist nodes (1000 ziplists, not 1000 items)
by 16 bytes.

In 32 bit build, this change reduces the maximum effective config of
list-compress-depth and list-max-ziplist-size (from 32767 to 8191)
---
 src/defrag.c                 |  96 +++++++++++++++-------
 src/quicklist.c              | 150 ++++++++++++++++++++++++++++++++++-
 src/quicklist.h              |  46 ++++++++++-
 tests/unit/memefficiency.tcl |  92 +++++++++++++++++++++
 4 files changed, 350 insertions(+), 34 deletions(-)

diff --git a/src/defrag.c b/src/defrag.c
index 04e57955b..e729297a5 100644
--- a/src/defrag.c
+++ b/src/defrag.c
@@ -5,8 +5,8 @@
  * We do that by scanning the keyspace and for each pointer we have, we can try to
  * ask the allocator if moving it to a new address will help reduce fragmentation.
  *
- * Copyright (c) 2017, Oran Agra
- * Copyright (c) 2017, Redis Labs, Inc
+ * Copyright (c) 2020, Oran Agra
+ * Copyright (c) 2020, Redis Labs, Inc
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -408,25 +408,32 @@ dictEntry* replaceSateliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sd
     return NULL;
 }
 
-long activeDefragQuickListNodes(quicklist *ql) {
-    quicklistNode *node = ql->head, *newnode;
+long activeDefragQuickListNode(quicklist *ql, quicklistNode **node_ref) {
+    quicklistNode *newnode, *node = *node_ref;
     long defragged = 0;
     unsigned char *newzl;
+    if ((newnode = activeDefragAlloc(node))) {
+        if (newnode->prev)
+            newnode->prev->next = newnode;
+        else
+            ql->head = newnode;
+        if (newnode->next)
+            newnode->next->prev = newnode;
+        else
+            ql->tail = newnode;
+        *node_ref = node = newnode;
+        defragged++;
+    }
+    if ((newzl = activeDefragAlloc(node->zl)))
+        defragged++, node->zl = newzl;
+    return defragged;
+}
+
+long activeDefragQuickListNodes(quicklist *ql) {
+    quicklistNode *node = ql->head;
+    long defragged = 0;
     while (node) {
-        if ((newnode = activeDefragAlloc(node))) {
-            if (newnode->prev)
-                newnode->prev->next = newnode;
-            else
-                ql->head = newnode;
-            if (newnode->next)
-                newnode->next->prev = newnode;
-            else
-                ql->tail = newnode;
-            node = newnode;
-            defragged++;
-        }
-        if ((newzl = activeDefragAlloc(node->zl)))
-            defragged++, node->zl = newzl;
+        defragged += activeDefragQuickListNode(ql, &node);
         node = node->next;
     }
     return defragged;
@@ -440,12 +447,48 @@ void defragLater(redisDb *db, dictEntry *kde) {
     listAddNodeTail(db->defrag_later, key);
 }
 
-long scanLaterList(robj *ob) {
+/* returns 0 if no more work needs to be been done, and 1 if time is up and more work is needed. */
+long scanLaterList(robj *ob, unsigned long *cursor, long long endtime, long long *defragged) {
     quicklist *ql = ob->ptr;
+    quicklistNode *node;
+    long iterations = 0;
+    int bookmark_failed = 0;
     if (ob->type != OBJ_LIST || ob->encoding != OBJ_ENCODING_QUICKLIST)
         return 0;
-    server.stat_active_defrag_scanned+=ql->len;
-    return activeDefragQuickListNodes(ql);
+
+    if (*cursor == 0) {
+        /* if cursor is 0, we start new iteration */
+        node = ql->head;
+    } else {
+        node = quicklistBookmarkFind(ql, "_AD");
+        if (!node) {
+            /* if the bookmark was deleted, it means we reached the end. */
+            *cursor = 0;
+            return 0;
+        }
+        node = node->next;
+    }
+
+    (*cursor)++;
+    while (node) {
+        (*defragged) += activeDefragQuickListNode(ql, &node);
+        server.stat_active_defrag_scanned++;
+        if (++iterations > 128 && !bookmark_failed) {
+            if (ustime() > endtime) {
+                if (!quicklistBookmarkCreate(&ql, "_AD", node)) {
+                    bookmark_failed = 1;
+                } else {
+                    ob->ptr = ql; /* bookmark creation may have re-allocated the quicklist */
+                    return 1;
+                }
+            }
+            iterations = 0;
+        }
+        node = node->next;
+    }
+    quicklistBookmarkDelete(ql, "_AD");
+    *cursor = 0;
+    return bookmark_failed? 1: 0;
 }
 
 typedef struct {
@@ -638,7 +681,8 @@ int scanLaterStraemListpacks(robj *ob, unsigned long *cursor, long long endtime,
         void *newdata = activeDefragAlloc(ri.data);
         if (newdata)
             raxSetData(ri.node, ri.data=newdata), (*defragged)++;
-        if (++iterations > 16) {
+        server.stat_active_defrag_scanned++;
+        if (++iterations > 128) {
             if (ustime() > endtime) {
                 serverAssert(ri.key_len==sizeof(last));
                 memcpy(last,ri.key,ri.key_len);
@@ -900,8 +944,7 @@ int defragLaterItem(dictEntry *de, unsigned long *cursor, long long endtime) {
     if (de) {
         robj *ob = dictGetVal(de);
         if (ob->type == OBJ_LIST) {
-            server.stat_active_defrag_hits += scanLaterList(ob);
-            *cursor = 0; /* list has no scan, we must finish it in one go */
+            return scanLaterList(ob, cursor, endtime, &server.stat_active_defrag_hits);
         } else if (ob->type == OBJ_SET) {
             server.stat_active_defrag_hits += scanLaterSet(ob, cursor);
         } else if (ob->type == OBJ_ZSET) {
@@ -961,11 +1004,6 @@ int defragLaterStep(redisDb *db, long long endtime) {
             if (defragLaterItem(de, &defrag_later_cursor, endtime))
                 quit = 1; /* time is up, we didn't finish all the work */
 
-            /* Don't start a new BIG key in this loop, this is because the
-             * next key can be a list, and scanLaterList must be done in once cycle */
-            if (!defrag_later_cursor)
-                quit = 1;
-
             /* Once in 16 scan iterations, 512 pointer reallocations, or 64 fields
              * (if we have a lot of pointers in one hash bucket, or rehashing),
              * check if we reached the time limit. */
diff --git a/src/quicklist.c b/src/quicklist.c
index 7b5484116..ae183ffd8 100644
--- a/src/quicklist.c
+++ b/src/quicklist.c
@@ -70,6 +70,12 @@ static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536};
     } while (0);
 #endif
 
+/* Bookmarks forward declarations */
+#define QL_MAX_BM ((1 << QL_BM_BITS)-1)
+quicklistBookmark *_quicklistBookmarkFindByName(quicklist *ql, const char *name);
+quicklistBookmark *_quicklistBookmarkFindByNode(quicklist *ql, quicklistNode *node);
+void _quicklistBookmarkDelete(quicklist *ql, quicklistBookmark *bm);
+
 /* Simple way to give quicklistEntry structs default values with one call. */
 #define initEntry(e)                                                           \
     do {                                                                       \
@@ -100,10 +106,11 @@ quicklist *quicklistCreate(void) {
     quicklist->count = 0;
     quicklist->compress = 0;
     quicklist->fill = -2;
+    quicklist->bookmark_count = 0;
     return quicklist;
 }
 
-#define COMPRESS_MAX (1 << 16)
+#define COMPRESS_MAX (1 << QL_COMP_BITS)
 void quicklistSetCompressDepth(quicklist *quicklist, int compress) {
     if (compress > COMPRESS_MAX) {
         compress = COMPRESS_MAX;
@@ -113,7 +120,7 @@ void quicklistSetCompressDepth(quicklist *quicklist, int compress) {
     quicklist->compress = compress;
 }
 
-#define FILL_MAX (1 << 15)
+#define FILL_MAX (1 << (QL_FILL_BITS-1))
 void quicklistSetFill(quicklist *quicklist, int fill) {
     if (fill > FILL_MAX) {
         fill = FILL_MAX;
@@ -169,6 +176,7 @@ void quicklistRelease(quicklist *quicklist) {
         quicklist->len--;
         current = next;
     }
+    quicklistBookmarksClear(quicklist);
     zfree(quicklist);
 }
 
@@ -578,6 +586,15 @@ quicklist *quicklistCreateFromZiplist(int fill, int compress,
 
 REDIS_STATIC void __quicklistDelNode(quicklist *quicklist,
                                      quicklistNode *node) {
+    /* Update the bookmark if any */
+    quicklistBookmark *bm = _quicklistBookmarkFindByNode(quicklist, node);
+    if (bm) {
+        bm->node = node->next;
+        /* if the bookmark was to the last node, delete it. */
+        if (!bm->node)
+            _quicklistBookmarkDelete(quicklist, bm);
+    }
+
     if (node->next)
         node->next->prev = node->prev;
     if (node->prev)
@@ -1410,6 +1427,87 @@ void quicklistPush(quicklist *quicklist, void *value, const size_t sz,
     }
 }
 
+/* Create or update a bookmark in the list which will be updated to the next node
+ * automatically when the one referenced gets deleted.
+ * Returns 1 on success (creation of new bookmark or override of an existing one).
+ * Returns 0 on failure (reached the maximum supported number of bookmarks).
+ * NOTE: use short simple names, so that string compare on find is quick.
+ * NOTE: bookmakrk creation may re-allocate the quicklist, so the input pointer
+         may change and it's the caller responsibilty to update the reference.
+ */
+int quicklistBookmarkCreate(quicklist **ql_ref, const char *name, quicklistNode *node) {
+    quicklist *ql = *ql_ref;
+    if (ql->bookmark_count >= QL_MAX_BM)
+        return 0;
+    quicklistBookmark *bm = _quicklistBookmarkFindByName(ql, name);
+    if (bm) {
+        bm->node = node;
+        return 1;
+    }
+    ql = zrealloc(ql, sizeof(quicklist) + (ql->bookmark_count+1) * sizeof(quicklistBookmark));
+    *ql_ref = ql;
+    ql->bookmarks[ql->bookmark_count].node = node;
+    ql->bookmarks[ql->bookmark_count].name = zstrdup(name);
+    ql->bookmark_count++;
+    return 1;
+}
+
+/* Find the quicklist node referenced by a named bookmark.
+ * When the bookmarked node is deleted the bookmark is updated to the next node,
+ * and if that's the last node, the bookmark is deleted (so find returns NULL). */
+quicklistNode *quicklistBookmarkFind(quicklist *ql, const char *name) {
+    quicklistBookmark *bm = _quicklistBookmarkFindByName(ql, name);
+    if (!bm) return NULL;
+    return bm->node;
+}
+
+/* Delete a named bookmark.
+ * returns 0 if bookmark was not found, and 1 if deleted.
+ * Note that the bookmark memory is not freed yet, and is kept for future use. */
+int quicklistBookmarkDelete(quicklist *ql, const char *name) {
+    quicklistBookmark *bm = _quicklistBookmarkFindByName(ql, name);
+    if (!bm)
+        return 0;
+    _quicklistBookmarkDelete(ql, bm);
+    return 1;
+}
+
+quicklistBookmark *_quicklistBookmarkFindByName(quicklist *ql, const char *name) {
+    unsigned i;
+    for (i=0; i<ql->bookmark_count; i++) {
+        if (!strcmp(ql->bookmarks[i].name, name)) {
+            return &ql->bookmarks[i];
+        }
+    }
+    return NULL;
+}
+
+quicklistBookmark *_quicklistBookmarkFindByNode(quicklist *ql, quicklistNode *node) {
+    unsigned i;
+    for (i=0; i<ql->bookmark_count; i++) {
+        if (ql->bookmarks[i].node == node) {
+            return &ql->bookmarks[i];
+        }
+    }
+    return NULL;
+}
+
+void _quicklistBookmarkDelete(quicklist *ql, quicklistBookmark *bm) {
+    int index = bm - ql->bookmarks;
+    zfree(bm->name);
+    ql->bookmark_count--;
+    memmove(bm, bm+1, (ql->bookmark_count - index)* sizeof(*bm));
+    /* NOTE: We do not shrink (realloc) the quicklist yet (to avoid resonance,
+     * it may be re-used later (a call to realloc may NOP). */
+}
+
+void quicklistBookmarksClear(quicklist *ql) {
+    while (ql->bookmark_count)
+        zfree(ql->bookmarks[--ql->bookmark_count].name);
+    /* NOTE: We do not shrink (realloc) the quick list. main use case for this
+     * function is just before releasing the allocation. */
+}
+
 /* The rest of this file is test cases and test helpers. */
 #ifdef REDIS_TEST
 #include <stdint.h>
@@ -2641,6 +2739,54 @@ int quicklistTest(int argc, char *argv[]) {
     printf("Compressions: %0.2f seconds.\n", (float)(stop - start) / 1000);
     printf("\n");
 
+    TEST("bookmark get updated to next item") {
+        quicklist *ql = quicklistNew(1, 0);
+        quicklistPushTail(ql, "1", 1);
+        quicklistPushTail(ql, "2", 1);
+        quicklistPushTail(ql, "3", 1);
+        quicklistPushTail(ql, "4", 1);
+        quicklistPushTail(ql, "5", 1);
+        assert(ql->len==5);
+        /* add two bookmarks, one pointing to the node before the last. */
+        assert(quicklistBookmarkCreate(&ql, "_dummy", ql->head->next));
+        assert(quicklistBookmarkCreate(&ql, "_test", ql->tail->prev));
+        /* test that the bookmark returns the right node, delete it and see that the bookmark points to the last node */
+        assert(quicklistBookmarkFind(ql, "_test") == ql->tail->prev);
+        assert(quicklistDelRange(ql, -2, 1));
+        assert(quicklistBookmarkFind(ql, "_test") == ql->tail);
+        /* delete the last node, and see that the bookmark was deleted. */
+        assert(quicklistDelRange(ql, -1, 1));
+        assert(quicklistBookmarkFind(ql, "_test") == NULL);
+        /* test that other bookmarks aren't affected */
+        assert(quicklistBookmarkFind(ql, "_dummy") == ql->head->next);
+        assert(quicklistBookmarkFind(ql, "_missing") == NULL);
+        assert(ql->len==3);
+        quicklistBookmarksClear(ql); /* for coverage */
+        assert(quicklistBookmarkFind(ql, "_dummy") == NULL);
+        quicklistRelease(ql);
+    }
+
+    TEST("bookmark limit") {
+        int i;
+        quicklist *ql = quicklistNew(1, 0);
+        quicklistPushHead(ql, "1", 1);
+        for (i=0; i<QL_MAX_BM; i++)
+            assert(quicklistBookmarkCreate(&ql, genstr("",i), ql->head));
+        /* when all bookmarks are used, creation fails */
+        assert(!quicklistBookmarkCreate(&ql, "_test", ql->head));
+        /* delete one and see that we can now create another */
+        assert(quicklistBookmarkDelete(ql, "0"));
+        assert(quicklistBookmarkCreate(&ql, "_test", ql->head));
+        /* delete one and see that the rest survive */
+        assert(quicklistBookmarkDelete(ql, "_test"));
+        for (i=1; i<QL_MAX_BM; i++)
+            assert(quicklistBookmarkFind(ql, genstr("",i)) == ql->head);
+        /* make sure the deleted ones are indeed gone */
+        assert(!quicklistBookmarkFind(ql, "0"));
+        assert(!quicklistBookmarkFind(ql, "_test"));
+        quicklistRelease(ql);
+    }
+
     if (!err)
         printf("ALL TESTS PASSED!\n");
     else
diff --git a/src/quicklist.h b/src/quicklist.h
index a7e27a2dd..8b553c119 100644
--- a/src/quicklist.h
+++ b/src/quicklist.h
@@ -28,6 +28,8 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <stdint.h> // for UINTPTR_MAX
+
 #ifndef __QUICKLIST_H__
 #define __QUICKLIST_H__
 
@@ -64,19 +66,51 @@ typedef struct quicklistLZF {
     char compressed[];
 } quicklistLZF;
 
+/* Bookmarks are padded with realloc at the end of of the quicklist struct.
+ * They should only be used for very big lists if thousands of nodes were the
+ * excess memory usage is negligible, and there's a real need to iterate on them
+ * in portions.
+ * When not used, they don't add any memory overhead, but when used and then
+ * deleted, some overhead remains (to avoid resonance).
+ * The number of bookmarks used should be kept to minimum since it also adds
+ * overhead on node deletion (searching for a bookmark to update). */
+typedef struct quicklistBookmark {
+    quicklistNode *node;
+    char *name;
+} quicklistBookmark;
+
+#if UINTPTR_MAX == 0xffffffff
+/* 32-bit */
+#   define QL_FILL_BITS 14
+#   define QL_COMP_BITS 14
+#   define QL_BM_BITS 4
+#elif UINTPTR_MAX == 0xffffffffffffffff
+/* 64-bit */
+#   define QL_FILL_BITS 16
+#   define QL_COMP_BITS 16
+#   define QL_BM_BITS 4 /* we can encode more, but we rather limit the user
+                           since they cause performance degradation. */
+#else
+#   error unknown arch bits count
+#endif
+
 /* quicklist is a 40 byte struct (on 64-bit systems) describing a quicklist.
  * 'count' is the number of total entries.
  * 'len' is the number of quicklist nodes.
  * 'compress' is: -1 if compression disabled, otherwise it's the number
  *                of quicklistNodes to leave uncompressed at ends of quicklist.
- * 'fill' is the user-requested (or default) fill factor. */
+ * 'fill' is the user-requested (or default) fill factor.
+ * 'bookmakrs are an optional feature that is used by realloc this struct,
+ *      so that they don't consume memory when not used. */
 typedef struct quicklist {
     quicklistNode *head;
     quicklistNode *tail;
     unsigned long count;        /* total count of all entries in all ziplists */
     unsigned long len;          /* number of quicklistNodes */
-    int fill : 16;              /* fill factor for individual nodes */
-    unsigned int compress : 16; /* depth of end nodes not to compress;0=off */
+    int fill : QL_FILL_BITS;              /* fill factor for individual nodes */
+    unsigned int compress : QL_COMP_BITS; /* depth of end nodes not to compress;0=off */
+    unsigned int bookmark_count: QL_BM_BITS;
+    quicklistBookmark bookmarks[];
 } quicklist;
 
 typedef struct quicklistIter {
@@ -158,6 +192,12 @@ unsigned long quicklistCount(const quicklist *ql);
 int quicklistCompare(unsigned char *p1, unsigned char *p2, int p2_len);
 size_t quicklistGetLzf(const quicklistNode *node, void **data);
 
+/* bookmarks */
+int quicklistBookmarkCreate(quicklist **ql_ref, const char *name, quicklistNode *node);
+int quicklistBookmarkDelete(quicklist *ql, const char *name);
+quicklistNode *quicklistBookmarkFind(quicklist *ql, const char *name);
+void quicklistBookmarksClear(quicklist *ql);
+
 #ifdef REDIS_TEST
 int quicklistTest(int argc, char *argv[]);
 #endif
diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl
index d152e212c..ec80c7384 100644
--- a/tests/unit/memefficiency.tcl
+++ b/tests/unit/memefficiency.tcl
@@ -209,5 +209,97 @@ start_server {tags {"defrag"}} {
             assert {$digest eq $newdigest}
             r save ;# saving an rdb iterates over all the data / pointers
         } {OK}
+
+        test "Active defrag big list" {
+            r flushdb
+            r config resetstat
+            r config set save "" ;# prevent bgsave from interfereing with save below
+            r config set hz 100
+            r config set activedefrag no
+            r config set active-defrag-max-scan-fields 1000
+            r config set active-defrag-threshold-lower 5
+            r config set active-defrag-cycle-min 65
+            r config set active-defrag-cycle-max 75
+            r config set active-defrag-ignore-bytes 2mb
+            r config set maxmemory 0
+            r config set list-max-ziplist-size 5 ;# list of 500k items will have 100k quicklist nodes
+
+            # create big keys with 10k items
+            set rd [redis_deferring_client]
+
+            set expected_frag 1.7
+            # add a mass of list nodes to two lists (allocations are interlaced)
+            set val [string repeat A 100] ;# 5 items of 100 bytes puts us in the 640 bytes bin, which has 32 regs, so high potential for fragmentation
+            for {set j 0} {$j < 500000} {incr j} {
+                $rd lpush biglist1 $val
+                $rd lpush biglist2 $val
+            }
+            for {set j 0} {$j < 500000} {incr j} {
+                $rd read ; # Discard replies
+                $rd read ; # Discard replies
+            }
+
+            # create some fragmentation
+            r del biglist2
+
+            # start defrag
+            after 120 ;# serverCron only updates the info once in 100ms
+            set frag [s allocator_frag_ratio]
+            if {$::verbose} {
+                puts "frag $frag"
+            }
+
+            assert {$frag >= $expected_frag}
+            r config set latency-monitor-threshold 5
+            r latency reset
+
+            set digest [r debug digest]
+            catch {r config set activedefrag yes} e
+            if {![string match {DISABLED*} $e]} {
+                # wait for the active defrag to start working (decision once a second)
+                wait_for_condition 50 100 {
+                    [s active_defrag_running] ne 0
+                } else {
+                    fail "defrag not started."
+                }
+
+                # wait for the active defrag to stop working
+                wait_for_condition 500 100 {
+                    [s active_defrag_running] eq 0
+                } else {
+                    after 120 ;# serverCron only updates the info once in 100ms
+                    puts [r info memory]
+                    puts [r info stats]
+                    puts [r memory malloc-stats]
+                    fail "defrag didn't stop."
+                }
+
+                # test the the fragmentation is lower
+                after 120 ;# serverCron only updates the info once in 100ms
+                set frag [s allocator_frag_ratio]
+                set max_latency 0
+                foreach event [r latency latest] {
+                    lassign $event eventname time latency max
+                    if {$eventname == "active-defrag-cycle"} {
+                        set max_latency $max
+                    }
+                }
+                if {$::verbose} {
+                    puts "frag $frag"
+                    puts "max latency $max_latency"
+                    puts [r latency latest]
+                    puts [r latency history active-defrag-cycle]
+                }
+                assert {$frag < 1.1}
+                # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75,
+                # we expect max latency to be not much higher than 7.5ms
+                assert {$max_latency <= 12}
+            }
+            # verify the data isn't corrupted or changed
+            set newdigest [r debug digest]
+            assert {$digest eq $newdigest}
+            r save ;# saving an rdb iterates over all the data / pointers
+            r del biglist1 ;# coverage for quicklistBookmarksClear
+        } {1}
     }
 }

From 770cb0ba97974623808f04de8549b509779bb645 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Wed, 19 Feb 2020 08:24:20 +0530
Subject: [PATCH 0121/1098] XGROUP DESTROY should unblock XREADGROUP with
 -NOGROUP

---
 src/t_stream.c                     | 2 ++
 tests/unit/type/stream-cgroups.tcl | 9 +++++++++
 2 files changed, 11 insertions(+)

diff --git a/src/t_stream.c b/src/t_stream.c
index 0f0f97a1d..900fa3a17 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1850,6 +1850,8 @@ NULL
             server.dirty++;
             notifyKeyspaceEvent(NOTIFY_STREAM,"xgroup-destroy",
                                 c->argv[2],c->db->id);
+            /* We want to unblock any XREADGROUP consumers with -NOGROUP. */
+            signalKeyAsReady(c->db,c->argv[2]);
         } else {
             addReply(c,shared.czero);
         }
diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl
index a59e168ef..072ed14d6 100644
--- a/tests/unit/type/stream-cgroups.tcl
+++ b/tests/unit/type/stream-cgroups.tcl
@@ -161,6 +161,15 @@ start_server {
         assert {[$rd read] == {}} ;# before the fix, client didn't even block, but was served synchronously with {mystream {}}
     }
 
+    test {XGROUP DESTROY should unblock XREADGROUP with -NOGROUP} {
+        r del mystream
+        r XGROUP CREATE mystream mygroup $ MKSTREAM
+        set rd [redis_deferring_client]
+        $rd XREADGROUP GROUP mygroup Alice BLOCK 100 STREAMS mystream ">"
+        r XGROUP DESTROY mystream mygroup
+        assert_error "*NOGROUP*" {$rd read}
+    }
+
     test {XCLAIM can claim PEL items from another consumer} {
         # Add 3 items into the stream, and create a consumer group
         r del mystream

From c8c78bd4b9644483c8828f6dbf4cb67155ba526e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 19 Feb 2020 19:00:29 +0100
Subject: [PATCH 0122/1098] Tracking: fix max-keys configuration directive.

---
 src/config.c | 2 +-
 src/server.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/config.c b/src/config.c
index b2e5fc12e..d55d1f8b5 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2160,7 +2160,6 @@ standardConfig configs[] = {
     createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, 0, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, 0, INTEGER_CONFIG, NULL, NULL),
-    createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_keys, 1000000, INTEGER_CONFIG, NULL, NULL), /* Default: 10% tracking table max number of keys tracked. */
     createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, 1, INTEGER_CONFIG, NULL, NULL), /* From 1 to 10. */
     createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG, NULL, updateHZ),
     createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves),
@@ -2195,6 +2194,7 @@ standardConfig configs[] = {
     createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, 4096, MEMORY_CONFIG, NULL, NULL),
     createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL),
     createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, 3000, MEMORY_CONFIG, NULL, NULL),
+    createSizeTConfig("tracking-table-max-keys", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.tracking_table_max_keys, 1000000, INTEGER_CONFIG, NULL, NULL), /* Default: 1 million keys max. */
 
     /* Other configs */
     createTimeTConfig("repl-backlog-ttl", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.repl_backlog_time_limit, 60*60, INTEGER_CONFIG, NULL, NULL), /* Default: 1 hour */
diff --git a/src/server.h b/src/server.h
index 439bbc393..bb40ca4e1 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1309,7 +1309,7 @@ struct redisServer {
     list *ready_keys;        /* List of readyList structures for BLPOP & co */
     /* Client side caching. */
     unsigned int tracking_clients;  /* # of clients with tracking enabled.*/
-    int tracking_table_max_keys;    /* Max number of keys in tracking table. */
+    size_t tracking_table_max_keys; /* Max number of keys in tracking table. */
     /* Sort parameters - qsort_r() is only available under BSD so we
      * have to take this state global, in order to pass it to sortCompare() */
     int sort_desc;

From a8d70ac568537f89497a778fd39e3ba313803df0 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 21 Feb 2020 13:48:43 +0100
Subject: [PATCH 0123/1098] Test is more complex now, increase default timeout.

---
 tests/test_helper.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index cb7e4e328..b266bc56d 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -87,7 +87,7 @@ set ::file ""; # If set, runs only the tests in this comma separated list
 set ::curfile ""; # Hold the filename of the current suite
 set ::accurate 0; # If true runs fuzz tests with more iterations
 set ::force_failure 0
-set ::timeout 600; # 10 minutes without progresses will quit the test.
+set ::timeout 1200; # 20 minutes without progresses will quit the test.
 set ::last_progress [clock seconds]
 set ::active_servers {} ; # Pids of active Redis instances.
 set ::dont_clean 0

From c6954de3ea90a2a72e83933cf5e96dfe848bbb1a Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 21 Feb 2020 17:08:45 +0100
Subject: [PATCH 0124/1098] Test engine: better tracking of what workers are
 doing.

---
 tests/support/server.tcl |  3 +++
 tests/test_helper.tcl    | 13 +++++++++----
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index b20f1ad36..174b05852 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -53,6 +53,7 @@ proc kill_server config {
     }
 
     # kill server and wait for the process to be totally exited
+    send_data_packet $::test_server_fd server-killing $pid
     catch {exec kill $pid}
     if {$::valgrind} {
         set max_wait 60000
@@ -231,6 +232,8 @@ proc start_server {options {code undefined}} {
     set stdout [format "%s/%s" [dict get $config "dir"] "stdout"]
     set stderr [format "%s/%s" [dict get $config "dir"] "stderr"]
 
+    send_data_packet $::test_server_fd "server-spawning" "port $::port"
+
     if {$::valgrind} {
         set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &]
     } elseif ($::stack_logging) {
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index b266bc56d..fa5579669 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -289,7 +289,7 @@ proc read_from_test_client fd {
         puts "\[$completed_tests_count/$all_tests_count [colorstr yellow $status]\]: $data ($elapsed seconds)"
         lappend ::clients_time_history $elapsed $data
         signal_idle_client $fd
-        set ::active_clients_task($fd) DONE
+        set ::active_clients_task($fd) "(DONE) $data"
     } elseif {$status eq {ok}} {
         if {!$::quiet} {
             puts "\[[colorstr green $status]\]: $data"
@@ -320,10 +320,16 @@ proc read_from_test_client fd {
         exit 1
     } elseif {$status eq {testing}} {
         set ::active_clients_task($fd) "(IN PROGRESS) $data"
+    } elseif {$status eq {server-spawning}} {
+        set ::active_clients_task($fd) "(SPAWNING SERVER) $data"
     } elseif {$status eq {server-spawned}} {
         lappend ::active_servers $data
+        set ::active_clients_task($fd) "(SPAWNED SERVER) pid:$data"
+    } elseif {$status eq {server-killing}} {
+        set ::active_clients_task($fd) "(KILLING SERVER) pid:$data"
     } elseif {$status eq {server-killed}} {
         set ::active_servers [lsearch -all -inline -not -exact $::active_servers $data]
+        set ::active_clients_task($fd) "(KILLED SERVER) pid:$data"
     } else {
         if {!$::quiet} {
             puts "\[$status\]: $data"
@@ -333,7 +339,7 @@ proc read_from_test_client fd {
 
 proc show_clients_state {} {
     # The following loop is only useful for debugging tests that may
-    # enter an infinite loop. Commented out normally.
+    # enter an infinite loop.
     foreach x $::active_clients {
         if {[info exist ::active_clients_task($x)]} {
             puts "$x => $::active_clients_task($x)"
@@ -363,8 +369,6 @@ proc signal_idle_client fd {
     set ::active_clients \
         [lsearch -all -inline -not -exact $::active_clients $fd]
 
-    if 0 {show_clients_state}
-
     # New unit to process?
     if {$::next_test != [llength $::all_tests]} {
         if {!$::quiet} {
@@ -380,6 +384,7 @@ proc signal_idle_client fd {
         }
     } else {
         lappend ::idle_clients $fd
+        set ::active_clients_task($fd) "SLEEPING, no more units to assign"
         if {[llength $::active_clients] == 0} {
             the_end
         }

From e78c4e813cecfa97505efe29c073f8088542f69e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 21 Feb 2020 18:55:56 +0100
Subject: [PATCH 0125/1098] Test engine: detect timeout when checking for Redis
 startup.

---
 tests/support/server.tcl | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index 174b05852..eec43e485 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -271,9 +271,19 @@ proc start_server {options {code undefined}} {
     }
 
     # Wait for actual startup
+    set checkperiod 100; # Milliseconds
+    set maxiter [expr {120*1000/100}] ; # Wait up to 2 minutes.
     while {![info exists _pid]} {
         regexp {PID:\s(\d+)} [exec cat $stdout] _ _pid
-        after 100
+        after $checkperiod
+        incr maxiter -1
+        if {$maxiter == 0} {
+            start_server_error $config_file "No PID detected in log $stdout"
+            puts "--- LOG CONTENT ---"
+            puts [exec cat $stdout]
+            puts "-------------------"
+            break
+        }
     }
 
     # setup properties to be able to initialize a client object

From 9947956d7bb7e6637139b6d4dcdfe2e1f3e2dd47 Mon Sep 17 00:00:00 2001
From: chendianqiang <chendianqiang@meituan.com>
Date: Sat, 22 Feb 2020 15:03:01 +0800
Subject: [PATCH 0126/1098] use correct list for moduleUnregisterUsedAPI

---
 src/module.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/module.c b/src/module.c
index aae15d74b..3268f77b2 100644
--- a/src/module.c
+++ b/src/module.c
@@ -6202,7 +6202,7 @@ int moduleUnregisterUsedAPI(RedisModule *module) {
         RedisModule *used = ln->value;
         listNode *ln = listSearchKey(used->usedby,module);
         if (ln) {
-            listDelNode(module->using,ln);
+            listDelNode(used->usedby,ln);
             count++;
         }
     }

From 743cfc0ad604c48bfa8c408ca36c6a5b90996aad Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Sat, 22 Feb 2020 11:38:51 -0500
Subject: [PATCH 0127/1098] add missing file marco

---
 src/sdsalloc.h | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/sdsalloc.h b/src/sdsalloc.h
index 531d41929..c04ff2a0a 100644
--- a/src/sdsalloc.h
+++ b/src/sdsalloc.h
@@ -36,7 +36,12 @@
  * the include of your alternate allocator if needed (not needed in order
  * to use the default libc allocator). */
 
+#ifndef __SDS_ALLOC_H__
+#define __SDS_ALLOC_H__
+
 #include "zmalloc.h"
 #define s_malloc zmalloc
 #define s_realloc zrealloc
 #define s_free zfree
+
+#endif

From 28bde891723b2bd815663d3228b28376b1e0dcb9 Mon Sep 17 00:00:00 2001
From: Ariel <ashtul@gmail.com>
Date: Sat, 22 Feb 2020 23:49:23 +0200
Subject: [PATCH 0128/1098] fix ThreadSafeContext lock/unlock function names

---
 src/module.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/module.c b/src/module.c
index 705ffec2c..3856fe50e 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4698,9 +4698,9 @@ int RM_BlockedClientDisconnected(RedisModuleCtx *ctx) {
  *
  * To call non-reply APIs, the thread safe context must be prepared with:
  *
- *     RedisModule_ThreadSafeCallStart(ctx);
+ *     RedisModule_ThreadSafeContextLock(ctx);
  *     ... make your call here ...
- *     RedisModule_ThreadSafeCallStop(ctx);
+ *     RedisModule_ThreadSafeContextUnlock(ctx);
  *
  * This is not needed when using `RedisModule_Reply*` functions, assuming
  * that a blocked client was used when the context was created, otherwise

From 62adabd0e0935cbb1cdedb3cde012c50ad1356a1 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 23 Feb 2020 12:46:14 +0200
Subject: [PATCH 0129/1098] Fix latency sensitivity of new defrag test

I saw that the new defag test for list was failing in CI recently, so i
reduce it's threshold from 12 to 60.

besides that, i add / improve the latency test for that other two defrag
tests (add a sensitive latency and digest / save checks)

and fix bad usage of debug populate (can't overrides existing keys).
this was the original intention, which creates higher fragmentation.
---
 tests/unit/memefficiency.tcl | 40 ++++++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 8 deletions(-)

diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl
index ec80c7384..468825a47 100644
--- a/tests/unit/memefficiency.tcl
+++ b/tests/unit/memefficiency.tcl
@@ -39,6 +39,8 @@ start_server {tags {"memefficiency"}} {
 start_server {tags {"defrag"}} {
     if {[string match {*jemalloc*} [s mem_allocator]]} {
         test "Active defrag" {
+            r config set save "" ;# prevent bgsave from interfereing with save below
+            r config set hz 100
             r config set activedefrag no
             r config set active-defrag-threshold-lower 5
             r config set active-defrag-cycle-min 65
@@ -46,8 +48,8 @@ start_server {tags {"defrag"}} {
             r config set active-defrag-ignore-bytes 2mb
             r config set maxmemory 100mb
             r config set maxmemory-policy allkeys-lru
-            r debug populate 700000 asdf 150
-            r debug populate 170000 asdf 300
+            r debug populate 700000 asdf1 150
+            r debug populate 170000 asdf2 300
             r ping ;# trigger eviction following the previous population
             after 120 ;# serverCron only updates the info once in 100ms
             set frag [s allocator_frag_ratio]
@@ -55,6 +57,10 @@ start_server {tags {"defrag"}} {
                 puts "frag $frag"
             }
             assert {$frag >= 1.4}
+
+            r config set latency-monitor-threshold 5
+            r latency reset
+            set digest [r debug digest]
             catch {r config set activedefrag yes} e
             if {![string match {DISABLED*} $e]} {
                 # Wait for the active defrag to start working (decision once a
@@ -78,19 +84,37 @@ start_server {tags {"defrag"}} {
                 # Test the the fragmentation is lower.
                 after 120 ;# serverCron only updates the info once in 100ms
                 set frag [s allocator_frag_ratio]
+                set max_latency 0
+                foreach event [r latency latest] {
+                    lassign $event eventname time latency max
+                    if {$eventname == "active-defrag-cycle"} {
+                        set max_latency $max
+                    }
+                }
                 if {$::verbose} {
                     puts "frag $frag"
+                    puts "max latency $max_latency"
+                    puts [r latency latest]
+                    puts [r latency history active-defrag-cycle]
                 }
                 assert {$frag < 1.1}
+                # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75,
+                # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher
+                assert {$max_latency <= 60}
             } else {
                 set _ ""
             }
-        } {}
+            # verify the data isn't corrupted or changed
+            set newdigest [r debug digest]
+            assert {$digest eq $newdigest}
+            r save ;# saving an rdb iterates over all the data / pointers
+        } {OK}
 
         test "Active defrag big keys" {
             r flushdb
             r config resetstat
             r config set save "" ;# prevent bgsave from interfereing with save below
+            r config set hz 100
             r config set activedefrag no
             r config set active-defrag-max-scan-fields 1000
             r config set active-defrag-threshold-lower 5
@@ -200,9 +224,9 @@ start_server {tags {"defrag"}} {
                     puts [r latency history active-defrag-cycle]
                 }
                 assert {$frag < 1.1}
-                # due to high fragmentation, 10hz, and active-defrag-cycle-max set to 75,
-                # we expect max latency to be not much higher than 75ms
-                assert {$max_latency <= 120}
+                # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75,
+                # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher
+                assert {$max_latency <= 60}
             }
             # verify the data isn't corrupted or changed
             set newdigest [r debug digest]
@@ -292,8 +316,8 @@ start_server {tags {"defrag"}} {
                 }
                 assert {$frag < 1.1}
                 # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75,
-                # we expect max latency to be not much higher than 7.5ms
-                assert {$max_latency <= 12}
+                # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher
+                assert {$max_latency <= 60}
             }
             # verify the data isn't corrupted or changed
             set newdigest [r debug digest]

From a6c7e61e61a253e5cddd4cf919e63da321d4afe8 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Sun, 23 Feb 2020 19:13:09 +0530
Subject: [PATCH 0130/1098] XREADGROUP should propagate XCALIM/SETID in
 MULTI/EXEC

Use built-in alsoPropagate mechanism that wraps commands
in MULTI/EXEC before sending them to replica/AOF
---
 src/t_stream.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 0f0f97a1d..fcfc5ca71 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -848,7 +848,7 @@ void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupnam
     argv[11] = createStringObject("JUSTID",6);
     argv[12] = createStringObject("LASTID",6);
     argv[13] = createObjectFromStreamID(&group->last_id);
-    propagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL);
+    alsoPropagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL);
     decrRefCount(argv[0]);
     decrRefCount(argv[3]);
     decrRefCount(argv[4]);
@@ -875,7 +875,7 @@ void streamPropagateGroupID(client *c, robj *key, streamCG *group, robj *groupna
     argv[2] = key;
     argv[3] = groupname;
     argv[4] = createObjectFromStreamID(&group->last_id);
-    propagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL);
+    alsoPropagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL);
     decrRefCount(argv[0]);
     decrRefCount(argv[1]);
     decrRefCount(argv[4]);

From 376a806bfbe545b6aff0b8eb5e0ddf7502c8606b Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Wed, 19 Feb 2020 13:24:50 +0530
Subject: [PATCH 0131/1098] Modules: Do not auto-unblock clients if not blocked
 on keys

---
 src/module.c | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/src/module.c b/src/module.c
index aae15d74b..f37d987e0 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4290,12 +4290,15 @@ void unblockClientFromModule(client *c) {
      * We must call moduleUnblockClient in order to free privdata and
      * RedisModuleBlockedClient.
      *
-     * Note that clients implementing threads and working with private data,
-     * should make sure to stop the threads or protect the private data
-     * in some other way in the disconnection and timeout callback, because
-     * here we are going to free the private data associated with the
-     * blocked client. */
-    if (!bc->unblocked)
+     * Note that we only do that for clients that are blocked on keys, for which
+     * the contract is that the module should not call RM_UnblockClient under
+     * normal circumstances.
+     * Clients implementing threads and working with private data should be
+     * aware that calling RM_UnblockClient for every blocked client is their
+     * responsibility, and if they fail to do so memory may leak. Ideally they
+     * should implement the disconnect and timeout callbacks and call
+     * RM_UnblockClient, but any other way is also acceptable. */
+    if (bc->blocked_on_keys && !bc->unblocked)
         moduleUnblockClient(c);
 
     bc->client = NULL;
@@ -4409,6 +4412,10 @@ int moduleTryServeClientBlockedOnKey(client *c, robj *key) {
  *
  *     free_privdata:   called in order to free the private data that is passed
  *                      by RedisModule_UnblockClient() call.
+ *
+ * Note: RedisModule_UnblockClient should be called for every blocked client,
+ *       even if client was killed, timed-out or disconnected. Failing to do so
+ *       will result in memory leaks.
  */
 RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms) {
     return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, NULL,0,NULL);
@@ -4463,7 +4470,15 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc
  * freed using the free_privdata callback provided by the user.
  *
  * However the reply callback will be able to access the argument vector of
- * the command, so the private data is often not needed. */
+ * the command, so the private data is often not needed.
+ *
+ * Note: Under normal circumstances RedisModule_UnblockClient should not be
+ *       called for clients that are blocked on keys (Either the key will
+ *       become ready or a timeout will occur). If for some reason you do want
+ *       to call RedisModule_UnblockClient it is possible: Client will be
+ *       handled as if it were timed-out (You must implement the timeout
+ *       callback in that case).
+ */
 RedisModuleBlockedClient *RM_BlockClientOnKeys(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata) {
     return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, keys,numkeys,privdata);
 }

From 0a643efa0c86e3053fd9a7bf7a74dfa4d0610f22 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 20 Feb 2020 17:56:52 +0200
Subject: [PATCH 0132/1098] fix race in module api test for fork

in some cases we were trying to kill the fork before it got created
---
 tests/modules/fork.c          | 2 +-
 tests/unit/moduleapi/fork.tcl | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/modules/fork.c b/tests/modules/fork.c
index 1a139ef1b..0443d9ef0 100644
--- a/tests/modules/fork.c
+++ b/tests/modules/fork.c
@@ -42,7 +42,7 @@ int fork_create(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
 
     /* child */
     RedisModule_Log(ctx, "notice", "fork child started");
-    usleep(200000);
+    usleep(500000);
     RedisModule_Log(ctx, "notice", "fork child exiting");
     RedisModule_ExitFromChild(code_to_exit_with);
     /* unreachable */
diff --git a/tests/unit/moduleapi/fork.tcl b/tests/unit/moduleapi/fork.tcl
index f7d7e47d5..8535a3382 100644
--- a/tests/unit/moduleapi/fork.tcl
+++ b/tests/unit/moduleapi/fork.tcl
@@ -20,9 +20,8 @@ start_server {tags {"modules"}} {
 
     test {Module fork kill} {
         r fork.create 3
-        after 20
+        after 250
         r fork.kill
-        after 100
 
         assert {[count_log_message "fork child started"] eq "2"}
         assert {[count_log_message "Received SIGUSR1 in child"] eq "1"}

From 73305861f5dc11af6ef6dd268fedbe290b00e396 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 24 Feb 2020 10:46:23 +0100
Subject: [PATCH 0133/1098] Test engine: experimental change to avoid busy port
 problems.

---
 tests/support/server.tcl | 149 ++++++++++++++++++++++++---------------
 1 file changed, 92 insertions(+), 57 deletions(-)

diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index eec43e485..d086366dc 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -141,6 +141,18 @@ proc tags {tags code} {
     uplevel 1 $code
     set ::tags [lrange $::tags 0 end-[llength $tags]]
 }
+
+# Write the configuration in the dictionary 'config' in the specified
+# file name.
+proc create_server_config_file {filename config} {
+    set fp [open $filename w+]
+    foreach directive [dict keys $config] {
+        puts -nonewline $fp "$directive "
+        puts $fp [dict get $config $directive]
+    }
+    close $fp
+}
+
 proc start_server {options {code undefined}} {
     # If we are running against an external server, we just push the
     # host/port pair in the stack the first time
@@ -222,68 +234,91 @@ proc start_server {options {code undefined}} {
 
     # write new configuration to temporary file
     set config_file [tmpfile redis.conf]
-    set fp [open $config_file w+]
-    foreach directive [dict keys $config] {
-        puts -nonewline $fp "$directive "
-        puts $fp [dict get $config $directive]
-    }
-    close $fp
+    create_server_config_file $config_file $config
 
     set stdout [format "%s/%s" [dict get $config "dir"] "stdout"]
     set stderr [format "%s/%s" [dict get $config "dir"] "stderr"]
 
-    send_data_packet $::test_server_fd "server-spawning" "port $::port"
-
-    if {$::valgrind} {
-        set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &]
-    } elseif ($::stack_logging) {
-        set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/redis-server $config_file > $stdout 2> $stderr &]
-    } else {
-        set pid [exec src/redis-server $config_file > $stdout 2> $stderr &]
-    }
-
-    # Tell the test server about this new instance.
-    send_data_packet $::test_server_fd server-spawned $pid
-
-    # check that the server actually started
-    # ugly but tries to be as fast as possible...
-    if {$::valgrind} {set retrynum 1000} else {set retrynum 100}
-
-    if {$::verbose} {
-        puts -nonewline "=== ($tags) Starting server ${::host}:${::port} "
-    }
-
-    if {$code ne "undefined"} {
-        set serverisup [server_is_up $::host $::port $retrynum]
-    } else {
-        set serverisup 1
-    }
-
-    if {$::verbose} {
-        puts ""
-    }
-
-    if {!$serverisup} {
-        set err {}
-        append err [exec cat $stdout] "\n" [exec cat $stderr]
-        start_server_error $config_file $err
-        return
-    }
-
-    # Wait for actual startup
-    set checkperiod 100; # Milliseconds
-    set maxiter [expr {120*1000/100}] ; # Wait up to 2 minutes.
-    while {![info exists _pid]} {
-        regexp {PID:\s(\d+)} [exec cat $stdout] _ _pid
-        after $checkperiod
-        incr maxiter -1
-        if {$maxiter == 0} {
-            start_server_error $config_file "No PID detected in log $stdout"
-            puts "--- LOG CONTENT ---"
-            puts [exec cat $stdout]
-            puts "-------------------"
-            break
+    # We need a loop here to retry with different ports.
+    set server_started 0
+    while {$server_started == 0} {
+        if {$::verbose} {
+            puts -nonewline "=== ($tags) Starting server ${::host}:${::port} "
         }
+
+        send_data_packet $::test_server_fd "server-spawning" "port $::port"
+
+        if {$::valgrind} {
+            set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &]
+        } elseif ($::stack_logging) {
+            set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/redis-server $config_file > $stdout 2> $stderr &]
+        } else {
+            set pid [exec src/redis-server $config_file > $stdout 2> $stderr &]
+        }
+
+        # Tell the test server about this new instance.
+        send_data_packet $::test_server_fd server-spawned $pid
+
+        # check that the server actually started
+        # ugly but tries to be as fast as possible...
+        if {$::valgrind} {set retrynum 1000} else {set retrynum 100}
+
+        # Wait for actual startup
+        set checkperiod 100; # Milliseconds
+        set maxiter [expr {120*1000/100}] ; # Wait up to 2 minutes.
+        set port_busy 0
+        while {![info exists _pid]} {
+            regexp {PID:\s(\d+)} [exec cat $stdout] _ _pid
+            after $checkperiod
+            incr maxiter -1
+            if {$maxiter == 0} {
+                start_server_error $config_file "No PID detected in log $stdout"
+                puts "--- LOG CONTENT ---"
+                puts [exec cat $stdout]
+                puts "-------------------"
+                break
+            }
+
+            # Check if the port is actually busy and the server failed
+            # for this reason.
+            if {[regexp {Could not create server TCP} [exec cat $stdout]]} {
+                set port_busy 1
+                break
+            }
+        }
+
+        # Sometimes we have to try a different port, even if we checked
+        # for availability. Other test clients may grab the port before we
+        # are able to do it for example.
+        if {$port_busy} {
+            puts "Port $::port was already busy, trying another port..."
+            set ::port [find_available_port [expr {$::port+1}]]
+            if {$::tls} {
+                dict set config "tls-port" $::port
+            } else {
+                dict set config port $::port
+            }
+            create_server_config_file $config_file $config
+            continue; # Try again
+        }
+
+        if {$code ne "undefined"} {
+            set serverisup [server_is_up $::host $::port $retrynum]
+        } else {
+            set serverisup 1
+        }
+
+        if {$::verbose} {
+            puts ""
+        }
+
+        if {!$serverisup} {
+            set err {}
+            append err [exec cat $stdout] "\n" [exec cat $stderr]
+            start_server_error $config_file $err
+            return
+        }
+        set server_started 1
     }
 
     # setup properties to be able to initialize a client object

From b6378edcd684fb8194861fbc7506614f987afc78 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 21 Feb 2020 16:39:42 +0100
Subject: [PATCH 0134/1098] Tracking: optin/out implemented.

---
 src/networking.c | 65 ++++++++++++++++++++++++++++++++++++++++++------
 src/server.h     |  6 ++++-
 src/tracking.c   | 27 ++++++++++++++------
 3 files changed, 82 insertions(+), 16 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 5b1229fde..4c394af70 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1365,6 +1365,12 @@ void resetClient(client *c) {
     if (!(c->flags & CLIENT_MULTI) && prevcmd != askingCommand)
         c->flags &= ~CLIENT_ASKING;
 
+    /* We do the same for the CACHING command as well. It also affects
+     * the next command or transaction executed, in a way very similar
+     * to ASKING. */
+    if (!(c->flags & CLIENT_MULTI) && prevcmd != clientCommand)
+        c->flags &= ~CLIENT_TRACKING_CACHING;
+
     /* Remove the CLIENT_REPLY_SKIP flag if any so that the reply
      * to the next command will be sent, but set the flag if the command
      * we just processed was "CLIENT REPLY SKIP". */
@@ -2044,7 +2050,7 @@ void clientCommand(client *c) {
 "REPLY (on|off|skip)    -- Control the replies sent to the current connection.",
 "SETNAME <name>         -- Assign the name <name> to the current connection.",
 "UNBLOCK <clientid> [TIMEOUT|ERROR] -- Unblock the specified blocked client.",
-"TRACKING (on|off) [REDIRECT <id>] [BCAST] [PREFIX first] [PREFIX second] ... -- Enable client keys tracking for client side caching.",
+"TRACKING (on|off) [REDIRECT <id>] [BCAST] [PREFIX first] [PREFIX second] [OPTIN] [OPTOUT]... -- Enable client keys tracking for client side caching.",
 "GETREDIR               -- Return the client ID we are redirecting to when tracking is enabled.",
 NULL
         };
@@ -2221,9 +2227,9 @@ NULL
         addReply(c,shared.ok);
     } else if (!strcasecmp(c->argv[1]->ptr,"tracking") && c->argc >= 3) {
         /* CLIENT TRACKING (on|off) [REDIRECT <id>] [BCAST] [PREFIX first]
-         *                          [PREFIX second] ... */
+         *                          [PREFIX second] [OPTIN] [OPTOUT] ... */
         long long redir = 0;
-        int bcast = 0;
+        uint64_t options = 0;
         robj **prefix = NULL;
         size_t numprefix = 0;
 
@@ -2256,7 +2262,11 @@ NULL
                     return;
                 }
             } else if (!strcasecmp(c->argv[j]->ptr,"bcast")) {
-                bcast = 1;
+                options |= CLIENT_TRACKING_BCAST;
+            } else if (!strcasecmp(c->argv[j]->ptr,"optin")) {
+                options |= CLIENT_TRACKING_OPTIN;
+            } else if (!strcasecmp(c->argv[j]->ptr,"optout")) {
+                options |= CLIENT_TRACKING_OPTOUT;
             } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && moreargs) {
                 j++;
                 prefix = zrealloc(prefix,sizeof(robj*)*(numprefix+1));
@@ -2272,7 +2282,7 @@ NULL
         if (!strcasecmp(c->argv[2]->ptr,"on")) {
             /* Before enabling tracking, make sure options are compatible
              * among each other and with the current state of the client. */
-            if (!bcast && numprefix) {
+            if (!(options & CLIENT_TRACKING_BCAST) && numprefix) {
                 addReplyError(c,
                     "PREFIX option requires BCAST mode to be enabled");
                 zfree(prefix);
@@ -2281,7 +2291,8 @@ NULL
 
             if (c->flags & CLIENT_TRACKING) {
                 int oldbcast = !!(c->flags & CLIENT_TRACKING_BCAST);
-                if (oldbcast != bcast) {
+                int newbcast = !!(options & CLIENT_TRACKING_BCAST);
+                if (oldbcast != newbcast) {
                     addReplyError(c,
                     "You can't switch BCAST mode on/off before disabling "
                     "tracking for this client, and then re-enabling it with "
@@ -2290,7 +2301,17 @@ NULL
                     return;
                 }
             }
-            enableTracking(c,redir,bcast,prefix,numprefix);
+
+            if (options & CLIENT_TRACKING_BCAST &&
+                options & (CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT))
+            {
+                addReplyError(c,
+                "OPTIN and OPTOUT are not compatible with BCAST");
+                zfree(prefix);
+                return;
+            }
+
+            enableTracking(c,redir,options,prefix,numprefix);
         } else if (!strcasecmp(c->argv[2]->ptr,"off")) {
             disableTracking(c);
         } else {
@@ -2300,6 +2321,36 @@ NULL
         }
         zfree(prefix);
         addReply(c,shared.ok);
+    } else if (!strcasecmp(c->argv[1]->ptr,"caching") && c->argc >= 3) {
+        if (!(c->flags & CLIENT_TRACKING)) {
+            addReplyError(c,"CLIENT CACHING can be called only when the "
+                            "client is in tracking mode with OPTIN or "
+                            "OPTOUT mode enabled");
+            return;
+        }
+
+        char *opt = c->argv[2]->ptr;
+        if (!strcasecmp(opt,"yes")) {
+            if (c->flags & CLIENT_TRACKING_OPTIN) {
+                c->flags |= CLIENT_TRACKING_CACHING;
+            } else {
+                addReplyError(c,"CLIENT CACHING YES is only valid when tracking is enabled in OPTIN mode.");
+                return;
+            }
+        } else if (!strcasecmp(opt,"no")) {
+            if (c->flags & CLIENT_TRACKING_OPTOUT) {
+                c->flags |= CLIENT_TRACKING_CACHING;
+            } else {
+                addReplyError(c,"CLIENT CACHING NO is only valid when tracking is enabled in OPTOUT mode.");
+                return;
+            }
+        } else {
+            addReply(c,shared.syntaxerr);
+            return;
+        }
+
+        /* Common reply for when we succeeded. */
+        addReply(c,shared.ok);
     } else if (!strcasecmp(c->argv[1]->ptr,"getredir") && c->argc == 2) {
         /* CLIENT GETREDIR */
         if (c->flags & CLIENT_TRACKING) {
diff --git a/src/server.h b/src/server.h
index bb40ca4e1..87c293c26 100644
--- a/src/server.h
+++ b/src/server.h
@@ -248,6 +248,10 @@ typedef long long ustime_t; /* microsecond time type. */
                                    perform client side caching. */
 #define CLIENT_TRACKING_BROKEN_REDIR (1ULL<<32) /* Target client is invalid. */
 #define CLIENT_TRACKING_BCAST (1ULL<<33) /* Tracking in BCAST mode. */
+#define CLIENT_TRACKING_OPTIN (1ULL<<34)  /* Tracking in opt-in mode. */
+#define CLIENT_TRACKING_OPTOUT (1ULL<<35) /* Tracking in opt-out mode. */
+#define CLIENT_TRACKING_CACHING (1ULL<<36) /* CACHING yes/no was given,
+                                              depending on optin/optout mode. */
 
 /* Client block type (btype field in client structure)
  * if CLIENT_BLOCKED flag is set. */
@@ -1651,7 +1655,7 @@ void addReplyStatusFormat(client *c, const char *fmt, ...);
 #endif
 
 /* Client side caching (tracking mode) */
-void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, size_t numprefix);
+void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **prefix, size_t numprefix);
 void disableTracking(client *c);
 void trackingRememberKeys(client *c);
 void trackingInvalidateKey(robj *keyobj);
diff --git a/src/tracking.c b/src/tracking.c
index 619148f2f..45f83103a 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -93,7 +93,8 @@ void disableTracking(client *c) {
     if (c->flags & CLIENT_TRACKING) {
         server.tracking_clients--;
         c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR|
-                      CLIENT_TRACKING_BCAST);
+                      CLIENT_TRACKING_BCAST|CLIENT_TRACKING_OPTIN|
+                      CLIENT_TRACKING_OPTOUT);
     }
 }
 
@@ -124,10 +125,11 @@ void enableBcastTrackingForPrefix(client *c, char *prefix, size_t plen) {
  * eventually get freed, we'll send a message to the original client to
  * inform it of the condition. Multiple clients can redirect the invalidation
  * messages to the same client ID. */
-void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, size_t numprefix) {
+void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **prefix, size_t numprefix) {
     if (!(c->flags & CLIENT_TRACKING)) server.tracking_clients++;
     c->flags |= CLIENT_TRACKING;
-    c->flags &= ~(CLIENT_TRACKING_BROKEN_REDIR|CLIENT_TRACKING_BCAST);
+    c->flags &= ~(CLIENT_TRACKING_BROKEN_REDIR|CLIENT_TRACKING_BCAST|
+                  CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT);
     c->client_tracking_redirection = redirect_to;
     if (TrackingTable == NULL) {
         TrackingTable = raxNew();
@@ -135,7 +137,7 @@ void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, s
         TrackingChannelName = createStringObject("__redis__:invalidate",20);
     }
 
-    if (bcast) {
+    if (options & CLIENT_TRACKING_BCAST) {
         c->flags |= CLIENT_TRACKING_BCAST;
         if (numprefix == 0) enableBcastTrackingForPrefix(c,"",0);
         for (size_t j = 0; j < numprefix; j++) {
@@ -143,14 +145,23 @@ void enableTracking(client *c, uint64_t redirect_to, int bcast, robj **prefix, s
             enableBcastTrackingForPrefix(c,sdsprefix,sdslen(sdsprefix));
         }
     }
+    c->flags |= options & (CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT);
 }
 
 /* This function is called after the execution of a readonly command in the
- * case the client 'c' has keys tracking enabled. It will populate the
- * tracking invalidation table according to the keys the user fetched, so that
- * Redis will know what are the clients that should receive an invalidation
- * message with certain groups of keys are modified. */
+ * case the client 'c' has keys tracking enabled and the tracking is not
+ * in BCAST mode. It will populate the tracking invalidation table according
+ * to the keys the user fetched, so that Redis will know what are the clients
+ * that should receive an invalidation message with certain groups of keys
+ * are modified. */
 void trackingRememberKeys(client *c) {
+    /* Return if we are in optin/out mode and the right CACHING command
+     * was/wasn't given in order to modify the default behavior. */
+    uint64_t optin = c->flags & CLIENT_TRACKING_OPTIN;
+    uint64_t optout = c->flags & CLIENT_TRACKING_OPTOUT;
+    uint64_t caching_given = c->flags & CLIENT_TRACKING_CACHING;
+    if ((optin && !caching_given) || (optout && caching_given)) return;
+
     int numkeys;
     int *keys = getKeysFromCommand(c->cmd,c->argv,c->argc,&numkeys);
     if (keys == NULL) return;

From 1ab550832477319195cd3455e657e35ab33ae985 Mon Sep 17 00:00:00 2001
From: Hengjian Tang <hengjian_tang@TSG.AISE.com>
Date: Tue, 25 Feb 2020 15:55:28 +0800
Subject: [PATCH 0135/1098] modify the read buf size according to the write buf
 size PROTO_IOBUF_LEN defined before

---
 src/replication.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/replication.c b/src/replication.c
index 4843f97d5..c497051c8 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1352,7 +1352,7 @@ void disklessLoadRestoreBackups(redisDb *backup, int restore, int empty_db_flags
 /* Asynchronously read the SYNC payload we receive from a master */
 #define REPL_MAX_WRITTEN_BEFORE_FSYNC (1024*1024*8) /* 8 MB */
 void readSyncBulkPayload(connection *conn) {
-    char buf[4096];
+    char buf[PROTO_IOBUF_LEN];
     ssize_t nread, readlen, nwritten;
     int use_diskless_load = useDisklessLoad();
     redisDb *diskless_load_backup = NULL;

From 537893420bc5a51ece060b53e5492b101bcfaba3 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 25 Feb 2020 13:01:52 +0200
Subject: [PATCH 0136/1098] fix github actions failing latency test for active
 defrag

seems that github actions are slow, using just one client to reduce
false positives.

also adding verbose, testing only on latest ubuntu, and building on
older one.

when doing that, i can reduce the test threshold back to something saner
---
 .github/workflows/ci.yml     | 23 ++++++++++++-----------
 tests/unit/memefficiency.tcl |  6 +++---
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 847abcf02..559ae61d8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3,11 +3,8 @@ name: CI
 on: [push, pull_request]
 
 jobs:
-  build-ubuntu:
-    strategy:
-      matrix:
-        platform: [ubuntu-latest, ubuntu-16.04]
-    runs-on: ${{ matrix.platform }}
+  test-ubuntu-latest:
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1
     - name: make
@@ -15,13 +12,17 @@ jobs:
     - name: test
       run: |
         sudo apt-get install tcl8.5
-        make test
+        ./runtest --clients 1 --verbose
 
-  build-macos-latest:
-    strategy:
-      matrix:
-        platform: [macos-latest, macOS-10.14]
-    runs-on: ${{ matrix.platform }}
+  test-ubuntu-old:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/checkout@v1
+    - name: make
+      run: make
+
+  build-macos-latest:
+    runs-on: macos-latest
     steps:
     - uses: actions/checkout@v1
     - name: make
diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl
index 468825a47..c899103fd 100644
--- a/tests/unit/memefficiency.tcl
+++ b/tests/unit/memefficiency.tcl
@@ -100,7 +100,7 @@ start_server {tags {"defrag"}} {
                 assert {$frag < 1.1}
                 # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75,
                 # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher
-                assert {$max_latency <= 60}
+                assert {$max_latency <= 30}
             } else {
                 set _ ""
             }
@@ -226,7 +226,7 @@ start_server {tags {"defrag"}} {
                 assert {$frag < 1.1}
                 # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75,
                 # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher
-                assert {$max_latency <= 60}
+                assert {$max_latency <= 30}
             }
             # verify the data isn't corrupted or changed
             set newdigest [r debug digest]
@@ -317,7 +317,7 @@ start_server {tags {"defrag"}} {
                 assert {$frag < 1.1}
                 # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75,
                 # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher
-                assert {$max_latency <= 60}
+                assert {$max_latency <= 30}
             }
             # verify the data isn't corrupted or changed
             set newdigest [r debug digest]

From 2f1a1c3835efefec8de97d3222538a45ccdd257e Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 26 Feb 2020 08:12:07 +0200
Subject: [PATCH 0137/1098] fix github actions failing latency test for active
 defrag - part 2

it seems that running two clients at a time is ok too, resuces action
time from 20 minutes to 10. we'll use this for now, and if one day it
won't be enough we'll have to run just the sensitive tests one by one
separately from the others.

this commit also fixes an issue with the defrag test that appears to be
very rare.
---
 .github/workflows/ci.yml     | 4 ++--
 tests/unit/memefficiency.tcl | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 559ae61d8..cc4991606 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,9 +12,9 @@ jobs:
     - name: test
       run: |
         sudo apt-get install tcl8.5
-        ./runtest --clients 1 --verbose
+        ./runtest --clients 2 --verbose
 
-  test-ubuntu-old:
+  build-ubuntu-old:
     runs-on: ubuntu-16.04
     steps:
     - uses: actions/checkout@v1
diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl
index c899103fd..06b0e07d7 100644
--- a/tests/unit/memefficiency.tcl
+++ b/tests/unit/memefficiency.tcl
@@ -60,6 +60,7 @@ start_server {tags {"defrag"}} {
 
             r config set latency-monitor-threshold 5
             r latency reset
+            r config set maxmemory 110mb ;# prevent further eviction (not to fail the digest test)
             set digest [r debug digest]
             catch {r config set activedefrag yes} e
             if {![string match {DISABLED*} $e]} {
@@ -166,7 +167,7 @@ start_server {tags {"defrag"}} {
             for {set j 0} {$j < 500000} {incr j} {
                 $rd read ; # Discard replies
             }
-            assert {[r dbsize] == 500010}
+            assert_equal [r dbsize] 500010
 
             # create some fragmentation
             for {set j 0} {$j < 500000} {incr j 2} {
@@ -175,7 +176,7 @@ start_server {tags {"defrag"}} {
             for {set j 0} {$j < 500000} {incr j 2} {
                 $rd read ; # Discard replies
             }
-            assert {[r dbsize] == 250010}
+            assert_equal [r dbsize] 250010
 
             # start defrag
             after 120 ;# serverCron only updates the info once in 100ms

From a477b68109e944634dc2979f1da2304c4186e278 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 23 Feb 2020 16:51:27 +0200
Subject: [PATCH 0138/1098] change CI to build and run the module api tests

---
 .github/workflows/ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cc4991606..3a81d1a08 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -13,6 +13,8 @@ jobs:
       run: |
         sudo apt-get install tcl8.5
         ./runtest --clients 2 --verbose
+    - name: module api test
+      run: ./runtest-moduleapi --clients 2 --verbose
 
   build-ubuntu-old:
     runs-on: ubuntu-16.04

From d8ab7e8012771ebcc4dbdc988ce84f7df57688c7 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 27 Feb 2020 17:41:48 +0100
Subject: [PATCH 0139/1098] Improve aeDeleteEventLoop() top comment grammar.

---
 src/ae.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/ae.c b/src/ae.c
index 248096e1f..d2248fe5c 100644
--- a/src/ae.c
+++ b/src/ae.c
@@ -135,7 +135,8 @@ void aeDeleteEventLoop(aeEventLoop *eventLoop) {
     aeApiFree(eventLoop);
     zfree(eventLoop->events);
     zfree(eventLoop->fired);
-    /* Free time event. */
+
+    /* Free the time events list. */
     aeTimeEvent *next_te, *te = eventLoop->timeEventHead;
     while (te) {
         next_te = te->next;

From 43f39f2e37b3b46683747a8207637f04747c9e94 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 27 Feb 2020 17:45:48 +0100
Subject: [PATCH 0140/1098] Remove useless comment from enumConfigSet().

---
 src/config.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/config.c b/src/config.c
index 5841ae7a0..8d069f8db 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1674,7 +1674,6 @@ static int enumConfigSet(typeData data, sds value, int update, char **err) {
         enumerr[sdslen(enumerr) - 2] = '\0';
 
         strncpy(loadbuf, enumerr, LOADBUF_SIZE);
-        /* strncpy does not if null terminate if source string length is >= destination buffer. */
         loadbuf[LOADBUF_SIZE - 1] = '\0';
 
         sdsfree(enumerr);

From 9abdc089f84c7b7e16b62b41a7a30eb9814d5b83 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 27 Feb 2020 17:47:50 +0100
Subject: [PATCH 0141/1098] Fix SDS misuse in enumConfigSet(). Related to
 #6778.

---
 src/config.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/config.c b/src/config.c
index 8d069f8db..aeb2fea7e 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1666,12 +1666,12 @@ static int enumConfigSet(typeData data, sds value, int update, char **err) {
         sds enumerr = sdsnew("argument must be one of the following: ");
         configEnum *enumNode = data.enumd.enum_value;
         while(enumNode->name != NULL) {
-            enumerr = sdscatlen(enumerr, enumNode->name, strlen(enumNode->name));
+            enumerr = sdscatlen(enumerr, enumNode->name,
+                                strlen(enumNode->name));
             enumerr = sdscatlen(enumerr, ", ", 2);
             enumNode++;
         }
-
-        enumerr[sdslen(enumerr) - 2] = '\0';
+        sdsrange(enumerr,0,-3); /* Remove final ", ". */
 
         strncpy(loadbuf, enumerr, LOADBUF_SIZE);
         loadbuf[LOADBUF_SIZE - 1] = '\0';

From 45f318d2ffaeb7b44b3aebb803a012fef1c36f0a Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 27 Feb 2020 18:21:12 +0100
Subject: [PATCH 0142/1098] Show Redis version when not understanding a config
 directive.

This makes simpler to give people help when posting such kind of errors
in the mailing list or other help forums, because sometimes the
directive looks well spelled, but the version of Redis they are using is
not able to support it.
---
 src/config.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/config.c b/src/config.c
index aeb2fea7e..fd04b7c87 100644
--- a/src/config.c
+++ b/src/config.c
@@ -518,7 +518,8 @@ void loadServerConfigFromString(char *config) {
     return;
 
 loaderr:
-    fprintf(stderr, "\n*** FATAL CONFIG FILE ERROR ***\n");
+    fprintf(stderr, "\n*** FATAL CONFIG FILE ERROR (Redis %s) ***\n",
+        REDIS_VERSION);
     fprintf(stderr, "Reading the configuration file, at line %d\n", linenum);
     fprintf(stderr, ">>> '%s'\n", lines[i]);
     fprintf(stderr, "%s\n", err);

From 0285c22a2dff319ea359f6ab7f4099b18fae0fa9 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Fri, 28 Feb 2020 13:35:10 +0200
Subject: [PATCH 0143/1098] Adds `BIN_PATH` to create-cluster

Allows for setting the binaries path if used outside the upstream repo.

Also documents `call` in usage clause (TODO: port to
`redis-cli --cluster call` or just deprecate it).
---
 utils/create-cluster/create-cluster | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster
index 4aef0816a..5b8bfe250 100755
--- a/utils/create-cluster/create-cluster
+++ b/utils/create-cluster/create-cluster
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 # Settings
+BIN_PATH="../../"
 CLUSTER_HOST=127.0.0.1
 PORT=30000
 TIMEOUT=2000
@@ -25,7 +26,7 @@ then
     while [ $((PORT < ENDPORT)) != "0" ]; do
         PORT=$((PORT+1))
         echo "Starting $PORT"
-        ../../src/redis-server --port $PORT  --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS}
+        $BIN_PATH/redis-server --port $PORT  --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS}
     done
     exit 0
 fi
@@ -37,7 +38,7 @@ then
         PORT=$((PORT+1))
         HOSTS="$HOSTS $CLUSTER_HOST:$PORT"
     done
-    ../../src/redis-cli --cluster create $HOSTS --cluster-replicas $REPLICAS
+    $BIN_PATH/redis-cli --cluster create $HOSTS --cluster-replicas $REPLICAS
     exit 0
 fi
 
@@ -46,7 +47,7 @@ then
     while [ $((PORT < ENDPORT)) != "0" ]; do
         PORT=$((PORT+1))
         echo "Stopping $PORT"
-        ../../src/redis-cli -p $PORT shutdown nosave
+        $BIN_PATH/redis-cli -p $PORT shutdown nosave
     done
     exit 0
 fi
@@ -57,7 +58,7 @@ then
     while [ 1 ]; do
         clear
         date
-        ../../src/redis-cli -p $PORT cluster nodes | head -30
+        $BIN_PATH/redis-cli -p $PORT cluster nodes | head -30
         sleep 1
     done
     exit 0
@@ -81,7 +82,7 @@ if [ "$1" == "call" ]
 then
     while [ $((PORT < ENDPORT)) != "0" ]; do
         PORT=$((PORT+1))
-        ../../src/redis-cli -p $PORT $2 $3 $4 $5 $6 $7 $8 $9
+        $BIN_PATH/redis-cli -p $PORT $2 $3 $4 $5 $6 $7 $8 $9
     done
     exit 0
 fi
@@ -101,7 +102,7 @@ then
     exit 0
 fi
 
-echo "Usage: $0 [start|create|stop|watch|tail|clean]"
+echo "Usage: $0 [start|create|stop|watch|tail|clean|call]"
 echo "start       -- Launch Redis Cluster instances."
 echo "create      -- Create a cluster using redis-cli --cluster create."
 echo "stop        -- Stop Redis Cluster instances."
@@ -110,3 +111,4 @@ echo "tail <id>   -- Run tail -f of instance at base port + ID."
 echo "tailall     -- Run tail -f for all the log files at once."
 echo "clean       -- Remove all instances data, logs, configs."
 echo "clean-logs  -- Remove just instances logs."
+echo "call <cmd>  -- Call a command (up to 7 arguments) on all nodes. "

From d25709cdeab78800a79bc2105f11a281fa9c6d36 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Fri, 28 Feb 2020 13:36:50 +0200
Subject: [PATCH 0144/1098] Update create-cluster

---
 utils/create-cluster/create-cluster | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster
index 5b8bfe250..4db0a6619 100755
--- a/utils/create-cluster/create-cluster
+++ b/utils/create-cluster/create-cluster
@@ -111,4 +111,4 @@ echo "tail <id>   -- Run tail -f of instance at base port + ID."
 echo "tailall     -- Run tail -f for all the log files at once."
 echo "clean       -- Remove all instances data, logs, configs."
 echo "clean-logs  -- Remove just instances logs."
-echo "call <cmd>  -- Call a command (up to 7 arguments) on all nodes. "
+echo "call <cmd>  -- Call a command (up to 7 arguments) on all nodes."

From 8a3e9a6d446230f7fe0db6651af1ee0314617b0c Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 28 Feb 2020 18:06:30 +0100
Subject: [PATCH 0145/1098] Modules: more details in RM_Scan API top comment.

---
 src/module.c | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/src/module.c b/src/module.c
index b821f0c31..47371fb32 100644
--- a/src/module.c
+++ b/src/module.c
@@ -6557,9 +6557,21 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) {
  * possibly setting errno if the call failed.
  * It is also possible to restart and existing cursor using RM_CursorRestart.
  *
- * NOTE: You must avoid doing any database changes from within the callback, you should avoid any
- * RedisModule_OpenKey or RedisModule_Call, if you need to do these, you need to keep the key name
- * and do any work you need to do after the call to Scan returns. */
+ * IMPORTANT: This API is very similar to the Redis SCAN command from the
+ * point of view of the guarantees it provides. This means that the API
+ * may report duplicated keys, but guarantees to report at least one time
+ * every key that was there from the start to the end of the scanning process.
+ *
+ * NOTE: If you do database changes within the callback, you should be aware
+ * that the internal state of the database may change. For instance it is safe
+ * to delete or modify the current key, but may not be safe to delete any
+ * other key.
+ * Moreover playing with the Redis keyspace while iterating may have the
+ * effect of returning more duplicates. A safe pattern is to store the keys
+ * names you want to modify elsewhere, and perform the actions on the keys
+ * later when the iteration is complete. Howerver this can cost a lot of
+ * memory, so it may make sense to just operate on the current key when
+ * possible during the iteration, given that this is safe. */
 int RM_Scan(RedisModuleCtx *ctx, RedisModuleScanCursor *cursor, RedisModuleScanCB fn, void *privdata) {
     if (cursor->done) {
         errno = ENOENT;
@@ -6641,9 +6653,13 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) {
  * possibly setting errno if the call failed.
  * It is also possible to restart and existing cursor using RM_CursorRestart.
  *
- * NOTE: You must avoid doing any database changes from within the callback, you should avoid any
- * RedisModule_OpenKey or RedisModule_Call, if you need to do these, you need to keep the field name
- * and do any work you need to do after the call to Scan returns. */
+ * NOTE: Certain operations are unsafe while iterating the object. For instance
+ * while the API guarantees to return at least one time all the elements that
+ * are present in the data structure consistently from the start to the end
+ * of the iteration (see HSCAN and similar commands documentation), the more
+ * you play with the elements, the more duplicates you may get. In general
+ * deleting the current element of the data structure is safe, while removing
+ * the key you are iterating is not safe. */
 int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleScanKeyCB fn, void *privdata) {
     if (key == NULL || key->value == NULL) {
         errno = EINVAL;

From c8ae90fef64c829cd1ac50c29e5912807c2577d5 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 28 Feb 2020 18:09:46 +0100
Subject: [PATCH 0146/1098] Modules: reformat RM_Scan() top comment a bit.

---
 src/module.c | 33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/src/module.c b/src/module.c
index 47371fb32..bbd54082c 100644
--- a/src/module.c
+++ b/src/module.c
@@ -6526,24 +6526,32 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) {
     zfree(cursor);
 }
 
-/* Scan api that allows a module to scan all the keys and value in the selected db.
+/* Scan API that allows a module to scan all the keys and value in
+ * the selected db.
  *
  * Callback for scan implementation.
- * void scan_callback(RedisModuleCtx *ctx, RedisModuleString *keyname, RedisModuleKey *key, void *privdata);
- * - ctx - the redis module context provided to for the scan.
- * - keyname - owned by the caller and need to be retained if used after this function.
- * - key - holds info on the key and value, it is provided as best effort, in some cases it might
- *   be NULL, in which case the user should (can) use RedisModule_OpenKey (and CloseKey too).
- *   when it is provided, it is owned by the caller and will be free when the callback returns.
- * - privdata - the user data provided to RedisModule_Scan.
+ * void scan_callback(RedisModuleCtx *ctx, RedisModuleString *keyname,
+ *                    RedisModuleKey *key, void *privdata);
+ * ctx - the redis module context provided to for the scan.
+ * keyname - owned by the caller and need to be retained if used after this
+ * function.
+ *
+ * key - holds info on the key and value, it is provided as best effort, in
+ * some cases it might be NULL, in which case the user should (can) use
+ * RedisModule_OpenKey (and CloseKey too).
+ * when it is provided, it is owned by the caller and will be free when the
+ * callback returns.
+ *
+ * privdata - the user data provided to RedisModule_Scan.
  *
  * The way it should be used:
  *      RedisModuleCursor *c = RedisModule_ScanCursorCreate();
  *      while(RedisModule_Scan(ctx, c, callback, privateData));
  *      RedisModule_ScanCursorDestroy(c);
  *
- * It is also possible to use this API from another thread while the lock is acquired durring 
- * the actuall call to RM_Scan:
+ * It is also possible to use this API from another thread while the lock
+ * is acquired durring the actuall call to RM_Scan:
+ *
  *      RedisModuleCursor *c = RedisModule_ScanCursorCreate();
  *      RedisModule_ThreadSafeContextLock(ctx);
  *      while(RedisModule_Scan(ctx, c, callback, privateData)){
@@ -6553,8 +6561,9 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) {
  *      }
  *      RedisModule_ScanCursorDestroy(c);
  *
- * The function will return 1 if there are more elements to scan and 0 otherwise,
- * possibly setting errno if the call failed.
+ * The function will return 1 if there are more elements to scan and
+ * 0 otherwise, possibly setting errno if the call failed.
+ *
  * It is also possible to restart and existing cursor using RM_CursorRestart.
  *
  * IMPORTANT: This API is very similar to the Redis SCAN command from the

From 98b23cce295d1ed584de00395695a012ef2dd2ab Mon Sep 17 00:00:00 2001
From: ShooterIT <wangyuancode@163.com>
Date: Sat, 29 Feb 2020 18:28:41 +0800
Subject: [PATCH 0147/1098] Avoid compiler warnings

---
 src/acl.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/acl.c b/src/acl.c
index b046785ff..efe6b96ad 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1830,6 +1830,7 @@ void aclCommand(client *c) {
             case ACL_DENIED_CMD: reasonstr="command"; break;
             case ACL_DENIED_KEY: reasonstr="key"; break;
             case ACL_DENIED_AUTH: reasonstr="auth"; break;
+            default: reasonstr="unknown";
             }
             addReplyBulkCString(c,reasonstr);
 

From 6dd8de177418da61e3d6466fe01fb5b3413c8308 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 2 Mar 2020 16:49:11 +0100
Subject: [PATCH 0148/1098] Use a smaller getkeys global buffer.

The idea is that very few commands have a lot of keys, and when this
happens the allocation time becomes neglegible.
---
 src/db.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/db.c b/src/db.c
index 6c5d4b4d4..04e26c33b 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1305,7 +1305,7 @@ int expireIfNeeded(redisDb *db, robj *key) {
 /* -----------------------------------------------------------------------------
  * API to get key arguments from commands
  * ---------------------------------------------------------------------------*/
-#define MAX_KEYS_BUFFER 65536
+#define MAX_KEYS_BUFFER 256
 static int getKeysTempBuffer[MAX_KEYS_BUFFER];
 
 /* The base case is to use the keys position as given in the command table

From 5e2319c3266c6a4e0a3745c6b3a6af63dc3d9a94 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 3 Mar 2020 14:58:11 +0100
Subject: [PATCH 0149/1098] Remove RDB files used for replication in
 persistence-less instances.

---
 src/replication.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++-
 src/server.c      |  8 ++++++++
 src/server.h      |  1 +
 3 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/src/replication.c b/src/replication.c
index c497051c8..e71b269d8 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -45,6 +45,11 @@ void replicationSendAck(void);
 void putSlaveOnline(client *slave);
 int cancelReplicationHandshake(void);
 
+/* We take a global flag to remember if this instance generated an RDB
+ * because of replication, so that we can remove the RDB file in case
+ * the instance is configured to have no persistence. */
+int RDBGeneratedByReplication = 0;
+
 /* --------------------------- Utility functions ---------------------------- */
 
 /* Return the pointer to a string representing the slave ip:listening_port
@@ -591,6 +596,10 @@ int startBgsaveForReplication(int mincapa) {
         retval = C_ERR;
     }
 
+    /* If we succeeded to start a BGSAVE with disk target, let's remember
+     * this fact, so that we can later delete the file if needed. */
+    if (retval == C_OK && !socket_target) RDBGeneratedByReplication = 1;
+
     /* If we failed to BGSAVE, remove the slaves waiting for a full
      * resynchronization from the list of slaves, inform them with
      * an error about what happened, close the connection ASAP. */
@@ -883,6 +892,36 @@ void putSlaveOnline(client *slave) {
         replicationGetSlaveName(slave));
 }
 
+/* We call this function periodically to remove an RDB file that was
+ * generated because of replication, in an instance that is otherwise
+ * without any persistence. We don't want instances without persistence
+ * to take RDB files around, this violates certain policies in certain
+ * environments. */
+void removeRDBUsedToSyncReplicas(void) {
+    if (allPersistenceDisabled() && RDBGeneratedByReplication) {
+        client *slave;
+        listNode *ln;
+        listIter li;
+
+        int delrdb = 1;
+        listRewind(server.slaves,&li);
+        while((ln = listNext(&li))) {
+            slave = ln->value;
+            if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START ||
+                slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END ||
+                slave->replstate == SLAVE_STATE_SEND_BULK)
+            {
+                delrdb = 0;
+                break; /* No need to check the other replicas. */
+            }
+        }
+        if (delrdb) {
+            RDBGeneratedByReplication = 0;
+            unlink(server.rdb_filename);
+        }
+    }
+}
+
 void sendBulkToSlave(connection *conn) {
     client *slave = connGetPrivateData(conn);
     char buf[PROTO_IOBUF_LEN];
@@ -894,7 +933,8 @@ void sendBulkToSlave(connection *conn) {
     if (slave->replpreamble) {
         nwritten = connWrite(conn,slave->replpreamble,sdslen(slave->replpreamble));
         if (nwritten == -1) {
-            serverLog(LL_VERBOSE,"Write error sending RDB preamble to replica: %s",
+            serverLog(LL_VERBOSE,
+                "Write error sending RDB preamble to replica: %s",
                 connGetLastError(conn));
             freeClient(slave);
             return;
@@ -1639,12 +1679,14 @@ void readSyncBulkPayload(connection *conn) {
                 "Failed trying to load the MASTER synchronization "
                 "DB from disk");
             cancelReplicationHandshake();
+            if (allPersistenceDisabled()) unlink(server.rdb_filename);
             /* Note that there's no point in restarting the AOF on sync failure,
                it'll be restarted when sync succeeds or replica promoted. */
             return;
         }
 
         /* Cleanup. */
+        if (allPersistenceDisabled()) unlink(server.rdb_filename);
         zfree(server.repl_transfer_tmpfile);
         close(server.repl_transfer_fd);
         server.repl_transfer_fd = -1;
@@ -3149,6 +3191,10 @@ void replicationCron(void) {
         }
     }
 
+    /* Remove the RDB file used for replication if Redis is not running
+     * with any persistence. */
+    removeRDBUsedToSyncReplicas();
+
     /* Refresh the number of slaves with lag <= min-slaves-max-lag. */
     refreshGoodSlavesCount();
     replication_cron_loops++; /* Incremented with frequency 1 HZ. */
diff --git a/src/server.c b/src/server.c
index bb8b3b103..a6d4b357e 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1455,12 +1455,20 @@ void updateDictResizePolicy(void) {
         dictDisableResize();
 }
 
+/* Return true if there are no active children processes doing RDB saving,
+ * AOF rewriting, or some side process spawned by a loaded module. */
 int hasActiveChildProcess() {
     return server.rdb_child_pid != -1 ||
            server.aof_child_pid != -1 ||
            server.module_child_pid != -1;
 }
 
+/* Return true if this instance has persistence completely turned off:
+ * both RDB and AOF are disabled. */
+int allPersistenceDisabled(void) {
+    return server.saveparamslen == 0 && server.aof_state == AOF_OFF;
+}
+
 /* ======================= Cron: called every 100 ms ======================== */
 
 /* Add a sample to the operations per second array of samples. */
diff --git a/src/server.h b/src/server.h
index 87c293c26..5763671e4 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1786,6 +1786,7 @@ void loadingProgress(off_t pos);
 void stopLoading(int success);
 void startSaving(int rdbflags);
 void stopSaving(int success);
+int allPersistenceDisabled(void);
 
 #define DISK_ERROR_TYPE_AOF 1       /* Don't accept writes: AOF errors. */
 #define DISK_ERROR_TYPE_RDB 2       /* Don't accept writes: RDB errors. */

From 2f3bfd1c135738722ba16c9a8320c188b465c9b7 Mon Sep 17 00:00:00 2001
From: Jamie Scott <jamie@redislabs.com>
Date: Tue, 3 Mar 2020 18:03:16 -0800
Subject: [PATCH 0150/1098] Update Redis.conf to improve TLS usability

When using TLS with a Redis.conf file the line for TLS reading tls-cert-file redis.crt tls-key-file redis.key is interpreted as one complete directive. I am separating this on two separate lines to improve usability so users do not get the below error.

ubuntu@ip-172-31-29-250:~/redis-6.0-rc1$ ./src/redis-server redis.conf

*** FATAL CONFIG FILE ERROR ***
Reading the configuration file, at line 145
>>> 'tls-cert-file redis.crt tls-key-file redis.key'
wrong number of arguments
ubuntu@ip-172-31-29-250:~/redis-6.0-rc1$ vi redis.conf
ubuntu@ip-172-31-29-250:~/redis-6.0-rc1$ ./src/redis-server redis.conf
23085:C 04 Mar 2020 01:58:12.631 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
23085:C 04 Mar 2020 01:58:12.631 # Redis version=5.9.101, bits=64, commit=00000000, modified=0, pid=23085, just started
23085:C 04 Mar 2020 01:58:12.631 # Configuration loaded
23085:M 04 Mar 2020 01:58:12.632 * Increased maximum number of open files to 10032 (it was originally set to 1024).
---
 redis.conf | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/redis.conf b/redis.conf
index c04880f32..048b16302 100644
--- a/redis.conf
+++ b/redis.conf
@@ -142,7 +142,8 @@ tcp-keepalive 300
 # server to connected clients, masters or cluster peers.  These files should be
 # PEM formatted.
 #
-# tls-cert-file redis.crt tls-key-file redis.key
+# tls-cert-file redis.crt 
+# tls-key-file redis.key
 
 # Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange:
 #

From f0acdee4c533164f76e28b889aa299acc5672269 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 4 Mar 2020 11:10:54 +0100
Subject: [PATCH 0151/1098] Introduce bg_unlink().

---
 src/replication.c | 34 +++++++++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index e71b269d8..8a05b9696 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -79,6 +79,34 @@ char *replicationGetSlaveName(client *c) {
     return buf;
 }
 
+/* Plain unlink() can block for quite some time in order to actually apply
+ * the file deletion to the filesystem. This call removes the file in a
+ * background thread instead. We actually just do close() in the thread,
+ * by using the fact that if there is another instance of the same file open,
+ * the foreground unlink() will not really do anything, and deleting the
+ * file will only happen once the last reference is lost. */
+int bg_unlink(const char *filename) {
+    int fd = open(filename,O_RDONLY|O_NONBLOCK);
+    if (fd == -1) {
+        /* Can't open the file? Fall back to unlinking in the main thread. */
+        return unlink(filename);
+    } else {
+        /* The following unlink() will not do anything since file
+         * is still open. */
+        int retval = unlink(filename);
+        if (retval == -1) {
+            /* If we got an unlink error, we just return it, closing the
+             * new reference we have to the file. */
+            int old_errno = errno;
+            close(fd);  /* This would overwrite our errno. So we saved it. */
+            errno = old_errno;
+            return -1;
+        }
+        bioCreateBackgroundJob(BIO_CLOSE_FILE,(void*)(long)fd,NULL,NULL);
+        return 0; /* Success. */
+    }
+}
+
 /* ---------------------------------- MASTER -------------------------------- */
 
 void createReplicationBacklog(void) {
@@ -917,7 +945,7 @@ void removeRDBUsedToSyncReplicas(void) {
         }
         if (delrdb) {
             RDBGeneratedByReplication = 0;
-            unlink(server.rdb_filename);
+            bg_unlink(server.rdb_filename);
         }
     }
 }
@@ -1679,14 +1707,14 @@ void readSyncBulkPayload(connection *conn) {
                 "Failed trying to load the MASTER synchronization "
                 "DB from disk");
             cancelReplicationHandshake();
-            if (allPersistenceDisabled()) unlink(server.rdb_filename);
+            if (allPersistenceDisabled()) bg_unlink(server.rdb_filename);
             /* Note that there's no point in restarting the AOF on sync failure,
                it'll be restarted when sync succeeds or replica promoted. */
             return;
         }
 
         /* Cleanup. */
-        if (allPersistenceDisabled()) unlink(server.rdb_filename);
+        if (allPersistenceDisabled()) bg_unlink(server.rdb_filename);
         zfree(server.repl_transfer_tmpfile);
         close(server.repl_transfer_fd);
         server.repl_transfer_fd = -1;

From ff024e3064b392018623209279284568caff899e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 4 Mar 2020 11:19:55 +0100
Subject: [PATCH 0152/1098] Log RDB deletion in persistence-less instances.

---
 src/replication.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index 8a05b9696..acecdd098 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -944,6 +944,8 @@ void removeRDBUsedToSyncReplicas(void) {
             }
         }
         if (delrdb) {
+            serverLog(LL_NOTICE,"Removing the RDB file used to feed replicas "
+                                "in a persistence-less instance");
             RDBGeneratedByReplication = 0;
             bg_unlink(server.rdb_filename);
         }
@@ -1707,14 +1709,25 @@ void readSyncBulkPayload(connection *conn) {
                 "Failed trying to load the MASTER synchronization "
                 "DB from disk");
             cancelReplicationHandshake();
-            if (allPersistenceDisabled()) bg_unlink(server.rdb_filename);
+            if (allPersistenceDisabled()) {
+                serverLog(LL_NOTICE,"Removing the RDB file obtained from "
+                                    "the master. This replica has persistence "
+                                    "disabled");
+                bg_unlink(server.rdb_filename);
+            }
             /* Note that there's no point in restarting the AOF on sync failure,
                it'll be restarted when sync succeeds or replica promoted. */
             return;
         }
 
         /* Cleanup. */
-        if (allPersistenceDisabled()) bg_unlink(server.rdb_filename);
+        if (allPersistenceDisabled()) {
+            serverLog(LL_NOTICE,"Removing the RDB file obtained from "
+                                "the master. This replica has persistence "
+                                "disabled");
+            bg_unlink(server.rdb_filename);
+        }
+
         zfree(server.repl_transfer_tmpfile);
         close(server.repl_transfer_fd);
         server.repl_transfer_fd = -1;

From d16bb64fa4522ea814d5eda71dc81a36f74716e6 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 4 Mar 2020 12:55:49 +0100
Subject: [PATCH 0153/1098] Check that the file exists in
 removeRDBUsedToSyncReplicas().

---
 src/replication.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index acecdd098..20666bd20 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -944,10 +944,14 @@ void removeRDBUsedToSyncReplicas(void) {
             }
         }
         if (delrdb) {
-            serverLog(LL_NOTICE,"Removing the RDB file used to feed replicas "
-                                "in a persistence-less instance");
-            RDBGeneratedByReplication = 0;
-            bg_unlink(server.rdb_filename);
+            struct stat sb;
+            if (lstat(server.rdb_filename,&sb) != -1) {
+                RDBGeneratedByReplication = 0;
+                serverLog(LL_NOTICE,
+                    "Removing the RDB file used to feed replicas "
+                    "in a persistence-less instance");
+                bg_unlink(server.rdb_filename);
+            }
         }
     }
 }

From 94376f46adc693ebac1f54b313527a1bac46e6e1 Mon Sep 17 00:00:00 2001
From: "bodong.ybd" <bodong.ybd@alibaba-inc.com>
Date: Wed, 4 Mar 2020 20:51:45 +0800
Subject: [PATCH 0154/1098] Added BITFIELD_RO variants for read-only
 operations.

---
 src/bitops.c            | 19 ++++++++++++++++++-
 src/server.c            |  4 ++++
 src/server.h            |  1 +
 tests/unit/bitfield.tcl | 31 +++++++++++++++++++++++++++++++
 4 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/src/bitops.c b/src/bitops.c
index ee1ce0460..ffb330013 100644
--- a/src/bitops.c
+++ b/src/bitops.c
@@ -902,6 +902,9 @@ void bitposCommand(client *c) {
  * OVERFLOW [WRAP|SAT|FAIL]
  */
 
+#define BITFIELD_COMMON (1<<0)
+#define BITFIELD_READONLY (1<<1)
+
 struct bitfieldOp {
     uint64_t offset;    /* Bitfield offset. */
     int64_t i64;        /* Increment amount (INCRBY) or SET value */
@@ -911,7 +914,7 @@ struct bitfieldOp {
     int sign;           /* True if signed, otherwise unsigned op. */
 };
 
-void bitfieldCommand(client *c) {
+void bitfieldGeneric(client *c, int flags) {
     robj *o;
     size_t bitoffset;
     int j, numops = 0, changes = 0;
@@ -999,6 +1002,12 @@ void bitfieldCommand(client *c) {
             return;
         }
     } else {
+        if (flags & BITFIELD_READONLY) {
+            zfree(ops);
+            addReplyError(c, "bitfield_ro only support get subcommand");
+            return;
+        }
+
         /* Lookup by making room up to the farest bit reached by
          * this operation. */
         if ((o = lookupStringForBitCommand(c,
@@ -1129,3 +1138,11 @@ void bitfieldCommand(client *c) {
     }
     zfree(ops);
 }
+
+void bitfieldCommand(client *c) {
+    bitfieldGeneric(c, BITFIELD_COMMON);
+}
+
+void bitfieldroCommand(client *c) {
+    bitfieldGeneric(c, BITFIELD_READONLY);
+}
diff --git a/src/server.c b/src/server.c
index bb8b3b103..ab2afd47f 100644
--- a/src/server.c
+++ b/src/server.c
@@ -238,6 +238,10 @@ struct redisCommand redisCommandTable[] = {
      "write use-memory @bitmap",
      0,NULL,1,1,1,0,0,0},
 
+    {"bitfield_ro",bitfieldroCommand,-2,
+     "read-only fast @bitmap",
+     0,NULL,1,1,1,0,0,0},
+
     {"setrange",setrangeCommand,4,
      "write use-memory @string",
      0,NULL,1,1,1,0,0,0},
diff --git a/src/server.h b/src/server.h
index 87c293c26..e5043855c 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2171,6 +2171,7 @@ void existsCommand(client *c);
 void setbitCommand(client *c);
 void getbitCommand(client *c);
 void bitfieldCommand(client *c);
+void bitfieldroCommand(client *c);
 void setrangeCommand(client *c);
 void getrangeCommand(client *c);
 void incrCommand(client *c);
diff --git a/tests/unit/bitfield.tcl b/tests/unit/bitfield.tcl
index d76452b1b..819d8f36d 100644
--- a/tests/unit/bitfield.tcl
+++ b/tests/unit/bitfield.tcl
@@ -199,3 +199,34 @@ start_server {tags {"bitops"}} {
         r del mystring
     }
 }
+
+start_server {tags {"repl"}} {
+    start_server {} {
+        set master [srv -1 client]
+        set master_host [srv -1 host]
+        set master_port [srv -1 port]
+        set slave [srv 0 client]
+
+        test {setup slave} {
+            $slave slaveof $master_host $master_port
+            wait_for_condition 50 100 {
+                [s 0 master_link_status] eq {up}
+            } else {
+                fail "Replication not started."
+            }
+        }
+
+        test {write on master, read on slave} {
+            $master del bits
+            assert_equal 0 [$master bitfield bits set u8 0 255]
+            assert_equal 255 [$master bitfield bits set u8 0 100]
+            wait_for_ofs_sync $master $slave
+            assert_equal 100 [$slave bitfield_ro bits get u8 0]
+        }
+
+        test {bitfield_ro with write option} {
+            catch {$slave bitfield_ro bits set u8 0 100 get u8 0} err
+            assert_match {*ERR bitfield_ro only support get subcommand*} $err
+        }
+    }
+}

From 3bab69e921039ad2e4f56c5842c4cf3e73a598dd Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 4 Mar 2020 17:44:21 +0100
Subject: [PATCH 0155/1098] Make sync RDB deletion configurable. Default to no.

---
 src/config.c      |  1 +
 src/replication.c | 23 +++++++++++++++++++----
 src/server.h      |  2 ++
 3 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/src/config.c b/src/config.c
index fd04b7c87..211b6d003 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2084,6 +2084,7 @@ standardConfig configs[] = {
     createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, 0, NULL, NULL),
     createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, 1, NULL, NULL),
     createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, 1, NULL, NULL),
+    createBoolConfig("rdb-del-sync-files", NULL, MODIFIABLE_CONFIG, server.rdb_del_sync_files, 0, NULL, NULL),
     createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, 1, NULL, NULL),
     createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, 1, NULL, NULL),
     createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, 1, NULL, NULL), /* Adapt hz to # of clients.*/
diff --git a/src/replication.c b/src/replication.c
index 20666bd20..429e1d8a8 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -625,8 +625,12 @@ int startBgsaveForReplication(int mincapa) {
     }
 
     /* If we succeeded to start a BGSAVE with disk target, let's remember
-     * this fact, so that we can later delete the file if needed. */
-    if (retval == C_OK && !socket_target) RDBGeneratedByReplication = 1;
+     * this fact, so that we can later delete the file if needed. Note
+     * that we don't set the flag to 1 if the feature is disabled, otherwise
+     * it would never be cleared: the file is not deleted. This way if
+     * the user enables it later with CONFIG SET, we are fine. */
+    if (retval == C_OK && !socket_target && server.rdb_del_sync_files)
+        RDBGeneratedByReplication = 1;
 
     /* If we failed to BGSAVE, remove the slaves waiting for a full
      * resynchronization from the list of slaves, inform them with
@@ -926,6 +930,17 @@ void putSlaveOnline(client *slave) {
  * to take RDB files around, this violates certain policies in certain
  * environments. */
 void removeRDBUsedToSyncReplicas(void) {
+    /* If the feature is disabled, return ASAP but also clear the
+     * RDBGeneratedByReplication flag in case it was set. Otherwise if the
+     * feature was enabled, but gets disabled later with CONFIG SET, the
+     * flag may remain set to one: then next time the feature is re-enabled
+     * via CONFIG SET we have have it set even if no RDB was generated
+     * because of replication recently. */
+    if (!server.rdb_del_sync_files) {
+        RDBGeneratedByReplication = 0;
+        return;
+    }
+
     if (allPersistenceDisabled() && RDBGeneratedByReplication) {
         client *slave;
         listNode *ln;
@@ -1713,7 +1728,7 @@ void readSyncBulkPayload(connection *conn) {
                 "Failed trying to load the MASTER synchronization "
                 "DB from disk");
             cancelReplicationHandshake();
-            if (allPersistenceDisabled()) {
+            if (server.rdb_del_sync_files && allPersistenceDisabled()) {
                 serverLog(LL_NOTICE,"Removing the RDB file obtained from "
                                     "the master. This replica has persistence "
                                     "disabled");
@@ -1725,7 +1740,7 @@ void readSyncBulkPayload(connection *conn) {
         }
 
         /* Cleanup. */
-        if (allPersistenceDisabled()) {
+        if (server.rdb_del_sync_files && allPersistenceDisabled()) {
             serverLog(LL_NOTICE,"Removing the RDB file obtained from "
                                 "the master. This replica has persistence "
                                 "disabled");
diff --git a/src/server.h b/src/server.h
index 5763671e4..3c19a17ea 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1202,6 +1202,8 @@ struct redisServer {
     char *rdb_filename;             /* Name of RDB file */
     int rdb_compression;            /* Use compression in RDB? */
     int rdb_checksum;               /* Use RDB checksum? */
+    int rdb_del_sync_files;         /* Remove RDB files used only for SYNC if
+                                       the instance does not use persistence. */
     time_t lastsave;                /* Unix time of last successful save */
     time_t lastbgsave_try;          /* Unix time of last attempted bgsave */
     time_t rdb_save_time_last;      /* Time used by last RDB save run. */

From 4983eb6a23032d0e7e0b06a218ba5a2aa818714f Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 4 Mar 2020 17:58:05 +0100
Subject: [PATCH 0156/1098] RDB deletion: document it in example redis.conf.

---
 redis.conf | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/redis.conf b/redis.conf
index c04880f32..8609a9f57 100644
--- a/redis.conf
+++ b/redis.conf
@@ -321,6 +321,19 @@ rdbchecksum yes
 # The filename where to dump the DB
 dbfilename dump.rdb
 
+# Remove RDB files used by replication in instances without persistence
+# enabled. By default this option is disabled, however there are environments
+# where for regulations or other security concerns, RDB files persisted on
+# disk by masters in order to feed replicas, or stored on disk by replicas
+# in order to load them for the initial synchronization, should be deleted
+# ASAP. Note that this option ONLY WORKS in instances that have both AOF
+# and RDB persistence disabled, otherwise is completely ignored.
+#
+# An alternative (and sometimes better) way to obtain the same effect is
+# to use diskless replication on both master and replicas instances. However
+# in the case of replicas, diskless is not always an option.
+rdb-del-sync-files no
+
 # The working directory.
 #
 # The DB will be written inside this directory, with the filename specified

From 2b7b74ad68303f1d6964f1d87bb887e604c34cf9 Mon Sep 17 00:00:00 2001
From: lifubang <lifubang@acmcoder.com>
Date: Thu, 5 Mar 2020 18:13:43 +0800
Subject: [PATCH 0157/1098] update linenoise to
 https://github.com/antirez/linenoise/tree/fc9667a81d43911a6690fb1e68c16e6e3bb8df05

Signed-off-by: lifubang <lifubang@acmcoder.com>
---
 deps/linenoise/README.markdown | 25 ++++++++++++++++++++++++-
 deps/linenoise/example.c       |  5 +++++
 deps/linenoise/linenoise.c     | 31 ++++++++++++++++++++++++++++---
 deps/linenoise/linenoise.h     |  2 ++
 4 files changed, 59 insertions(+), 4 deletions(-)

diff --git a/deps/linenoise/README.markdown b/deps/linenoise/README.markdown
index e01642cf8..1afea2ae6 100644
--- a/deps/linenoise/README.markdown
+++ b/deps/linenoise/README.markdown
@@ -21,7 +21,7 @@ So what usually happens is either:
  
 The result is a pollution of binaries without line editing support.
 
-So I spent more or less two hours doing a reality check resulting in this little library: is it *really* needed for a line editing library to be 20k lines of code? Apparently not, it is possibe to get a very small, zero configuration, trivial to embed library, that solves the problem. Smaller programs will just include this, supporing line editing out of the box. Larger programs may use this little library or just checking with configure if readline/libedit is available and resorting to Linenoise if not.
+So I spent more or less two hours doing a reality check resulting in this little library: is it *really* needed for a line editing library to be 20k lines of code? Apparently not, it is possibe to get a very small, zero configuration, trivial to embed library, that solves the problem. Smaller programs will just include this, supporting line editing out of the box. Larger programs may use this little library or just checking with configure if readline/libedit is available and resorting to Linenoise if not.
 
 ## Terminals, in 2010.
 
@@ -126,6 +126,24 @@ Linenoise has direct support for persisting the history into an history
 file. The functions `linenoiseHistorySave` and `linenoiseHistoryLoad` do
 just that. Both functions return -1 on error and 0 on success.
 
+## Mask mode
+
+Sometimes it is useful to allow the user to type passwords or other
+secrets that should not be displayed. For such situations linenoise supports
+a "mask mode" that will just replace the characters the user is typing 
+with `*` characters, like in the following example:
+
+    $ ./linenoise_example
+    hello> get mykey
+    echo: 'get mykey'
+    hello> /mask
+    hello> *********
+
+You can enable and disable mask mode using the following two functions:
+
+    void linenoiseMaskModeEnable(void);
+    void linenoiseMaskModeDisable(void);
+
 ## Completion
 
 Linenoise supports completion, which is the ability to complete the user
@@ -222,3 +240,8 @@ Sometimes you may want to clear the screen as a result of something the
 user typed. You can do this by calling the following function:
 
     void linenoiseClearScreen(void);
+
+## Related projects
+
+* [Linenoise NG](https://github.com/arangodb/linenoise-ng) is a fork of Linenoise that aims to add more advanced features like UTF-8 support, Windows support and other features. Uses C++ instead of C as development language.
+* [Linenoise-swift](https://github.com/andybest/linenoise-swift) is a reimplementation of Linenoise written in Swift.
diff --git a/deps/linenoise/example.c b/deps/linenoise/example.c
index 3a544d3c6..74358c323 100644
--- a/deps/linenoise/example.c
+++ b/deps/linenoise/example.c
@@ -55,6 +55,7 @@ int main(int argc, char **argv) {
      *
      * The typed string is returned as a malloc() allocated string by
      * linenoise, so the user needs to free() it. */
+    
     while((line = linenoise("hello> ")) != NULL) {
         /* Do something with the string. */
         if (line[0] != '\0' && line[0] != '/') {
@@ -65,6 +66,10 @@ int main(int argc, char **argv) {
             /* The "/historylen" command will change the history len. */
             int len = atoi(line+11);
             linenoiseHistorySetMaxLen(len);
+        } else if (!strncmp(line, "/mask", 5)) {
+            linenoiseMaskModeEnable();
+        } else if (!strncmp(line, "/unmask", 7)) {
+            linenoiseMaskModeDisable();
         } else if (line[0] == '/') {
             printf("Unreconized command: %s\n", line);
         }
diff --git a/deps/linenoise/linenoise.c b/deps/linenoise/linenoise.c
index fce14a7c5..01c3b9350 100644
--- a/deps/linenoise/linenoise.c
+++ b/deps/linenoise/linenoise.c
@@ -125,6 +125,7 @@ static linenoiseHintsCallback *hintsCallback = NULL;
 static linenoiseFreeHintsCallback *freeHintsCallback = NULL;
 
 static struct termios orig_termios; /* In order to restore at exit.*/
+static int maskmode = 0; /* Show "***" instead of input. For passwords. */
 static int rawmode = 0; /* For atexit() function to check if restore is needed*/
 static int mlmode = 0;  /* Multi line mode. Default is single line. */
 static int atexit_registered = 0; /* Register atexit just 1 time. */
@@ -197,6 +198,19 @@ FILE *lndebug_fp = NULL;
 
 /* ======================= Low level terminal handling ====================== */
 
+/* Enable "mask mode". When it is enabled, instead of the input that
+ * the user is typing, the terminal will just display a corresponding
+ * number of asterisks, like "****". This is useful for passwords and other
+ * secrets that should not be displayed. */
+void linenoiseMaskModeEnable(void) {
+    maskmode = 1;
+}
+
+/* Disable mask mode. */
+void linenoiseMaskModeDisable(void) {
+    maskmode = 0;
+}
+
 /* Set if to use or not the multi line mode. */
 void linenoiseSetMultiLine(int ml) {
     mlmode = ml;
@@ -485,6 +499,8 @@ void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) {
             if (bold == 1 && color == -1) color = 37;
             if (color != -1 || bold != 0)
                 snprintf(seq,64,"\033[%d;%d;49m",bold,color);
+            else
+                seq[0] = '\0';
             abAppend(ab,seq,strlen(seq));
             abAppend(ab,hint,hintlen);
             if (color != -1 || bold != 0)
@@ -523,7 +539,11 @@ static void refreshSingleLine(struct linenoiseState *l) {
     abAppend(&ab,seq,strlen(seq));
     /* Write the prompt and the current buffer content */
     abAppend(&ab,l->prompt,strlen(l->prompt));
-    abAppend(&ab,buf,len);
+    if (maskmode == 1) {
+        while (len--) abAppend(&ab,"*",1);
+    } else {
+        abAppend(&ab,buf,len);
+    }
     /* Show hits if any. */
     refreshShowHints(&ab,l,plen);
     /* Erase to right */
@@ -577,7 +597,11 @@ static void refreshMultiLine(struct linenoiseState *l) {
 
     /* Write the prompt and the current buffer content */
     abAppend(&ab,l->prompt,strlen(l->prompt));
-    abAppend(&ab,l->buf,l->len);
+    if (maskmode == 1) {
+        for (uint i = 0; i < l->len; i++) abAppend(&ab,"*",1);
+    } else {
+        abAppend(&ab,l->buf,l->len);
+    }
 
     /* Show hits if any. */
     refreshShowHints(&ab,l,plen);
@@ -645,7 +669,8 @@ int linenoiseEditInsert(struct linenoiseState *l, char c) {
             if ((!mlmode && l->plen+l->len < l->cols && !hintsCallback)) {
                 /* Avoid a full update of the line in the
                  * trivial case. */
-                if (write(l->ofd,&c,1) == -1) return -1;
+                char d = (maskmode==1) ? '*' : c;
+                if (write(l->ofd,&d,1) == -1) return -1;
             } else {
                 refreshLine(l);
             }
diff --git a/deps/linenoise/linenoise.h b/deps/linenoise/linenoise.h
index ed20232c5..6dfee73bc 100644
--- a/deps/linenoise/linenoise.h
+++ b/deps/linenoise/linenoise.h
@@ -65,6 +65,8 @@ int linenoiseHistoryLoad(const char *filename);
 void linenoiseClearScreen(void);
 void linenoiseSetMultiLine(int ml);
 void linenoisePrintKeyCodes(void);
+void linenoiseMaskModeEnable(void);
+void linenoiseMaskModeDisable(void);
 
 #ifdef __cplusplus
 }

From 1b72f4b74951d6abff055e0667f18e9833fd0c72 Mon Sep 17 00:00:00 2001
From: lifubang <lifubang@acmcoder.com>
Date: Thu, 5 Mar 2020 18:17:32 +0800
Subject: [PATCH 0158/1098] add askpass mode

Signed-off-by: lifubang <lifubang@acmcoder.com>
---
 src/redis-cli.c | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 54898f42e..b44db9a1e 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -229,6 +229,7 @@ static struct config {
     int hotkeys;
     int stdinarg; /* get last arg from stdin. (-x option) */
     char *auth;
+    int askpass;
     char *user;
     int output; /* output mode, see OUTPUT_* defines */
     sds mb_delim;
@@ -1450,6 +1451,8 @@ static int parseOptions(int argc, char **argv) {
             config.dbnum = atoi(argv[++i]);
         } else if (!strcmp(argv[i], "--no-auth-warning")) {
             config.no_auth_warning = 1;
+        } else if (!strcmp(argv[i], "--askpass")) {
+            config.askpass = 1;
         } else if ((!strcmp(argv[i],"-a") || !strcmp(argv[i],"--pass"))
                    && !lastarg)
         {
@@ -1690,6 +1693,9 @@ static void usage(void) {
 "                     (if both are used, this argument takes predecence).\n"
 "  --user <username>  Used to send ACL style 'AUTH username pass'. Needs -a.\n"
 "  --pass <password>  Alias of -a for consistency with the new --user option.\n"
+"  --askpass          Force user to input password with mask from STDIN.\n"
+"                     If this argument is used, '-a' and " REDIS_CLI_AUTH_ENV "\n"
+"                     environment variable will be ignored.\n"
 "  -u <uri>           Server URI.\n"
 "  -r <repeat>        Execute specified command N times.\n"
 "  -i <interval>      When -r is used, waits <interval> seconds per command.\n"
@@ -7858,6 +7864,13 @@ static void intrinsicLatencyMode(void) {
     }
 }
 
+static sds askPassword() {
+    linenoiseMaskModeEnable();
+    sds auth = linenoise("Please input password: ");
+    linenoiseMaskModeDisable();
+    return auth;
+}
+
 /*------------------------------------------------------------------------------
  * Program main()
  *--------------------------------------------------------------------------- */
@@ -7894,6 +7907,7 @@ int main(int argc, char **argv) {
     config.hotkeys = 0;
     config.stdinarg = 0;
     config.auth = NULL;
+    config.askpass = 0;
     config.user = NULL;
     config.eval = NULL;
     config.eval_ldb = 0;
@@ -7935,6 +7949,10 @@ int main(int argc, char **argv) {
 
     parseEnv();
 
+    if (config.askpass) {
+        config.auth = askPassword();
+    }
+
 #ifdef USE_OPENSSL
     if (config.tls) {
         ERR_load_crypto_strings();
@@ -8044,4 +8062,4 @@ int main(int argc, char **argv) {
     } else {
         return noninteractive(argc,convertToSds(argc,argv));
     }
-}
+}
\ No newline at end of file

From 27641ee490aafdbb38ec506edfcbdb041e829826 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 5 Mar 2020 16:55:14 +0200
Subject: [PATCH 0159/1098] fix for flaky psync2 test

*** [err]: PSYNC2: total sum of full synchronizations is exactly 4 in tests/integration/psync2.tcl
Expected 5 == 4 (context: type eval line 6 cmd {assert {$sum == 4}} proc ::test)

issue was that sometime the test got an unexpected full sync since it
tried to switch to the replica before it was in sync with it's master.
---
 tests/integration/psync2.tcl | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl
index d1212b640..333736ffa 100644
--- a/tests/integration/psync2.tcl
+++ b/tests/integration/psync2.tcl
@@ -114,6 +114,27 @@ start_server {} {
             }
         }
 
+        # wait for all the slaves to be in sync with the master
+        set master_ofs [status $R($master_id) master_repl_offset]
+        wait_for_condition 500 100 {
+            $master_ofs == [status $R(0) master_repl_offset] &&
+            $master_ofs == [status $R(1) master_repl_offset] &&
+            $master_ofs == [status $R(2) master_repl_offset] &&
+            $master_ofs == [status $R(3) master_repl_offset] &&
+            $master_ofs == [status $R(4) master_repl_offset]
+        } else {
+            if {$debug_msg} {
+                for {set j 0} {$j < 5} {incr j} {
+                    puts "$j: sync_full: [status $R($j) sync_full]"
+                    puts "$j: id1      : [status $R($j) master_replid]:[status $R($j) master_repl_offset]"
+                    puts "$j: id2      : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]"
+                    puts "$j: backlog  : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]"
+                    puts "---"
+                }
+            }
+            fail "Slaves are not in sync with the master after too long time."
+        }
+
         # Put down the old master so that it cannot generate more
         # replication stream, this way in the next master switch, the time at
         # which we move slaves away is not important, each will have full

From 654815729308c830e88a397b59c88768e0c73242 Mon Sep 17 00:00:00 2001
From: David Carlier <devnexen@gmail.com>
Date: Sat, 7 Mar 2020 10:43:41 +0000
Subject: [PATCH 0160/1098] debug, dump registers on arm too.

---
 src/debug.c | 82 +++++++++++++++++++++++++++++++++++------------------
 1 file changed, 55 insertions(+), 27 deletions(-)

diff --git a/src/debug.c b/src/debug.c
index 36af35aec..a3e434d21 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1045,6 +1045,61 @@ void logRegisters(ucontext_t *uc) {
         (unsigned long) uc->uc_mcontext.gregs[18]
     );
     logStackContent((void**)uc->uc_mcontext.gregs[15]);
+    #elif defined(__aarch64__) /* Linux AArch64 */
+    serverLog(LL_WARNING,
+	      "\n"
+	      "X18:%016lx X19:%016lx\nX20:%016lx X21:%016lx\n"
+	      "X22:%016lx X23:%016lx\nX24:%016lx X25:%016lx\n"
+	      "X26:%016lx X27:%016lx\nX28:%016lx X29:%016lx\n"
+	      "X30:%016lx\n"
+	      "pc:%016lx sp:%016lx\npstate:%016lx fault_address:%016lx\n",
+	      (unsigned long) uc->uc_mcontext.regs[18],
+	      (unsigned long) uc->uc_mcontext.regs[19],
+	      (unsigned long) uc->uc_mcontext.regs[20],
+	      (unsigned long) uc->uc_mcontext.regs[21],
+	      (unsigned long) uc->uc_mcontext.regs[22],
+	      (unsigned long) uc->uc_mcontext.regs[23],
+	      (unsigned long) uc->uc_mcontext.regs[24],
+	      (unsigned long) uc->uc_mcontext.regs[25],
+	      (unsigned long) uc->uc_mcontext.regs[26],
+	      (unsigned long) uc->uc_mcontext.regs[27],
+	      (unsigned long) uc->uc_mcontext.regs[28],
+	      (unsigned long) uc->uc_mcontext.regs[29],
+	      (unsigned long) uc->uc_mcontext.regs[30],
+	      (unsigned long) uc->uc_mcontext.pc,
+	      (unsigned long) uc->uc_mcontext.sp,
+	      (unsigned long) uc->uc_mcontext.pstate,
+	      (unsigned long) uc->uc_mcontext.fault_address
+		      );
+	      logStackContent((void**)uc->uc_mcontext.sp);
+    #elif defined(__arm__) /* Linux ARM */
+    serverLog(LL_WARNING,
+	      "\n"
+	      "R10:%016lx R9 :%016lx\nR8 :%016lx R7 :%016lx\n"
+	      "R6 :%016lx R5 :%016lx\nR4 :%016lx R3 :%016lx\n"
+	      "R2 :%016lx R1 :%016lx\nR0 :%016lx EC :%016lx\n"
+	      "fp: %016lx ip:%016lx\n",
+	      "pc:%016lx sp:%016lx\ncpsr:%016lx fault_address:%016lx\n",
+	      (unsigned long) uc->uc_mcontext.arm_r10,
+	      (unsigned long) uc->uc_mcontext.arm_r9,
+	      (unsigned long) uc->uc_mcontext.arm_r8,
+	      (unsigned long) uc->uc_mcontext.arm_r7,
+	      (unsigned long) uc->uc_mcontext.arm_r6,
+	      (unsigned long) uc->uc_mcontext.arm_r5,
+	      (unsigned long) uc->uc_mcontext.arm_r4,
+	      (unsigned long) uc->uc_mcontext.arm_r3,
+	      (unsigned long) uc->uc_mcontext.arm_r2,
+	      (unsigned long) uc->uc_mcontext.arm_r1,
+	      (unsigned long) uc->uc_mcontext.arm_r0,
+	      (unsigned long) uc->uc_mcontext.error_code,
+	      (unsigned long) uc->uc_mcontext.arm_fp,
+	      (unsigned long) uc->uc_mcontext.arm_ip,
+	      (unsigned long) uc->uc_mcontext.arm_pc,
+	      (unsigned long) uc->uc_mcontext.arm_sp,
+	      (unsigned long) uc->uc_mcontext.arm_cpsr,
+	      (unsigned long) uc->uc_mcontext.fault_address
+		      );
+	      logStackContent((void**)uc->uc_mcontext.arm_sp);
     #endif
 #elif defined(__FreeBSD__)
     #if defined(__x86_64__)
@@ -1185,33 +1240,6 @@ void logRegisters(ucontext_t *uc) {
         (unsigned long) uc->uc_mcontext.mc_cs
     );
     logStackContent((void**)uc->uc_mcontext.mc_rsp);
-#elif defined(__aarch64__) /* Linux AArch64 */
-    serverLog(LL_WARNING,
-	      "\n"
-	      "X18:%016lx X19:%016lx\nX20:%016lx X21:%016lx\n"
-	      "X22:%016lx X23:%016lx\nX24:%016lx X25:%016lx\n"
-	      "X26:%016lx X27:%016lx\nX28:%016lx X29:%016lx\n"
-	      "X30:%016lx\n"
-	      "pc:%016lx sp:%016lx\npstate:%016lx fault_address:%016lx\n",
-	      (unsigned long) uc->uc_mcontext.regs[18],
-	      (unsigned long) uc->uc_mcontext.regs[19],
-	      (unsigned long) uc->uc_mcontext.regs[20],
-	      (unsigned long) uc->uc_mcontext.regs[21],
-	      (unsigned long) uc->uc_mcontext.regs[22],
-	      (unsigned long) uc->uc_mcontext.regs[23],
-	      (unsigned long) uc->uc_mcontext.regs[24],
-	      (unsigned long) uc->uc_mcontext.regs[25],
-	      (unsigned long) uc->uc_mcontext.regs[26],
-	      (unsigned long) uc->uc_mcontext.regs[27],
-	      (unsigned long) uc->uc_mcontext.regs[28],
-	      (unsigned long) uc->uc_mcontext.regs[29],
-	      (unsigned long) uc->uc_mcontext.regs[30],
-	      (unsigned long) uc->uc_mcontext.pc,
-	      (unsigned long) uc->uc_mcontext.sp,
-	      (unsigned long) uc->uc_mcontext.pstate,
-	      (unsigned long) uc->uc_mcontext.fault_address
-		      );
-	      logStackContent((void**)uc->uc_mcontext.sp);
 #else
     serverLog(LL_WARNING,
         "  Dumping of registers not supported for this OS/arch");

From 106a57c5ae15a4f1f11c19e564eab4968093ee7a Mon Sep 17 00:00:00 2001
From: guodongxiaren <879231132@qq.com>
Date: Sat, 7 Mar 2020 19:38:27 +0800
Subject: [PATCH 0161/1098] string literal should be const char*

---
 src/asciilogo.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/asciilogo.h b/src/asciilogo.h
index 83c538b54..044ca0c55 100644
--- a/src/asciilogo.h
+++ b/src/asciilogo.h
@@ -27,7 +27,7 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-char *ascii_logo =
+const char *ascii_logo =
 "                _._                                                  \n"
 "           _.-``__ ''-._                                             \n"
 "      _.-``    `.  `_.  ''-._           Redis %s (%s/%d) %s bit\n"

From 374b1192a358525dd099d5d5fc67ff370e3b6b05 Mon Sep 17 00:00:00 2001
From: Jamie Scott <jamie@redislabs.com>
Date: Mon, 9 Mar 2020 12:53:44 -0700
Subject: [PATCH 0162/1098] Remove default guidance in Redis.conf

Removing the default guidance in Redis.conf since this is not an available value.
---
 redis.conf | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/redis.conf b/redis.conf
index 048b16302..a8e2fd89d 100644
--- a/redis.conf
+++ b/redis.conf
@@ -176,8 +176,7 @@ tcp-keepalive 300
 # tls-cluster yes
 
 # Explicitly specify TLS versions to support. Allowed values are case insensitive
-# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) or
-# "default" which is currently >= TLSv1.1.
+# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) 
 #
 # tls-protocols TLSv1.2
 

From 336458d4b5e54a2b7a637628d1237a6291e26299 Mon Sep 17 00:00:00 2001
From: "bodong.ybd" <bodong.ybd@alibaba-inc.com>
Date: Wed, 11 Mar 2020 20:55:51 +0800
Subject: [PATCH 0163/1098] Fix bug of tcl test using external server

---
 tests/support/server.tcl | 7 +++++--
 tests/test_helper.tcl    | 3 +++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index d086366dc..400017c5f 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -159,9 +159,12 @@ proc start_server {options {code undefined}} {
     if {$::external} {
         if {[llength $::servers] == 0} {
             set srv {}
+            # In test_server_main(tests/test_helper.tcl:215~218), increase the value of start_port
+            # and assign it to ::port through the `--port` option, so we need to reduce it.
+            set baseport [expr {$::port-100}]
             dict set srv "host" $::host
-            dict set srv "port" $::port
-            set client [redis $::host $::port 0 $::tls]
+            dict set srv "port" $baseport
+            set client [redis $::host $baseport 0 $::tls]
             dict set srv "client" $client
             $client select 9
 
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index fa5579669..4dbead193 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -505,6 +505,9 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
     } elseif {$opt eq {--host}} {
         set ::external 1
         set ::host $arg
+        # If we use an external server, we can only set numclients to 1,
+        # otherwise the port will be miscalculated.
+        set ::numclients 1
         incr j
     } elseif {$opt eq {--port}} {
         set ::port $arg

From 7d703de7e938e72041a395b0dee435550fee7c5b Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Wed, 11 Mar 2020 18:43:03 +0200
Subject: [PATCH 0164/1098] Adds keyspace notifications to migrate and restore

---
 src/cluster.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/cluster.c b/src/cluster.c
index c05e46f76..5f63d2b8f 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4981,6 +4981,7 @@ void restoreCommand(client *c) {
     }
     objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock,1000);
     signalModifiedKey(c->db,c->argv[1]);
+    notifyKeyspaceEvent(NOTIFY_GENERIC,"restore",c->argv[1],c->db->id);
     addReply(c,shared.ok);
     server.dirty++;
 }
@@ -5327,6 +5328,7 @@ try_again:
                 /* No COPY option: remove the local key, signal the change. */
                 dbDelete(c->db,kv[j]);
                 signalModifiedKey(c->db,kv[j]);
+                notifyKeyspaceEvent(NOTIFY_GENERIC,"del",kv[j],c->db->id);
                 server.dirty++;
 
                 /* Populate the argument vector to replace the old one. */
@@ -5489,7 +5491,7 @@ void readwriteCommand(client *c) {
  * already "down" but it is fragile to rely on the update of the global state,
  * so we also handle it here.
  *
- * CLUSTER_REDIR_DOWN_STATE and CLUSTER_REDIR_DOWN_RO_STATE if the cluster is 
+ * CLUSTER_REDIR_DOWN_STATE and CLUSTER_REDIR_DOWN_RO_STATE if the cluster is
  * down but the user attempts to execute a command that addresses one or more keys. */
 clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, int *error_code) {
     clusterNode *n = NULL;

From 63c4697b46ff9162722881eaacebab6cce457a1a Mon Sep 17 00:00:00 2001
From: "bodong.ybd" <bodong.ybd@alibaba-inc.com>
Date: Thu, 12 Mar 2020 11:12:37 +0800
Subject: [PATCH 0165/1098] Remove duplicate obj files in Makefile

---
 src/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Makefile b/src/Makefile
index 00b623a4b..bbfb06440 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -208,9 +208,9 @@ REDIS_SERVER_NAME=redis-server
 REDIS_SENTINEL_NAME=redis-sentinel
 REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o
 REDIS_CLI_NAME=redis-cli
-REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o
+REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crc64.o siphash.o crc16.o
 REDIS_BENCHMARK_NAME=redis-benchmark
-REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o redis-benchmark.o
+REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o
 REDIS_CHECK_RDB_NAME=redis-check-rdb
 REDIS_CHECK_AOF_NAME=redis-check-aof
 

From 513931dfea68ae866b91b197ca7fc0e9d845bb0a Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 12 Mar 2020 12:59:44 +0100
Subject: [PATCH 0166/1098] ae.c: fix crash when resizing the event loop.

See #6964. The root cause is that the event loop may be resized from an
event callback itself, causing the event pointer to be invalid.
---
 src/ae.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/ae.c b/src/ae.c
index d2248fe5c..1bf6cbfbf 100644
--- a/src/ae.c
+++ b/src/ae.c
@@ -464,6 +464,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
             if (!invert && fe->mask & mask & AE_READABLE) {
                 fe->rfileProc(eventLoop,fd,fe->clientData,mask);
                 fired++;
+                fe = &eventLoop->events[fd]; /* Refresh in case of resize. */
             }
 
             /* Fire the writable event. */
@@ -476,8 +477,11 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
 
             /* If we have to invert the call, fire the readable event now
              * after the writable one. */
-            if (invert && fe->mask & mask & AE_READABLE) {
-                if (!fired || fe->wfileProc != fe->rfileProc) {
+            if (invert) {
+                fe = &eventLoop->events[fd]; /* Refresh in case of resize. */
+                if ((fe->mask & mask & AE_READABLE) &&
+                    (!fired || fe->wfileProc != fe->rfileProc))
+                {
                     fe->rfileProc(eventLoop,fd,fe->clientData,mask);
                     fired++;
                 }

From a5f5091041c17586df96993c22300ef1664d9170 Mon Sep 17 00:00:00 2001
From: fengpf <18221167541@163.com>
Date: Thu, 12 Mar 2020 20:44:32 +0800
Subject: [PATCH 0167/1098] fix comments in latency.c

---
 .gitignore    | 1 +
 src/latency.c | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index de626d61b..e445fd201 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,3 +32,4 @@ deps/lua/src/liblua.a
 *.dSYM
 Makefile.dep
 .vscode/*
+.idea/*
diff --git a/src/latency.c b/src/latency.c
index 74ced72a5..9a291ac9b 100644
--- a/src/latency.c
+++ b/src/latency.c
@@ -85,7 +85,7 @@ int THPGetAnonHugePagesSize(void) {
 /* ---------------------------- Latency API --------------------------------- */
 
 /* Latency monitor initialization. We just need to create the dictionary
- * of time series, each time serie is craeted on demand in order to avoid
+ * of time series, each time serie is created on demand in order to avoid
  * having a fixed list to maintain. */
 void latencyMonitorInit(void) {
     server.latency_events = dictCreate(&latencyTimeSeriesDictType,NULL);

From 5babacad9b68932d07fcb3ee6ddae5a21e7b380d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 12 Mar 2020 15:53:08 +0100
Subject: [PATCH 0168/1098] Update linenoise.

---
 deps/linenoise/linenoise.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/deps/linenoise/linenoise.c b/deps/linenoise/linenoise.c
index 01c3b9350..cfe51e768 100644
--- a/deps/linenoise/linenoise.c
+++ b/deps/linenoise/linenoise.c
@@ -598,7 +598,8 @@ static void refreshMultiLine(struct linenoiseState *l) {
     /* Write the prompt and the current buffer content */
     abAppend(&ab,l->prompt,strlen(l->prompt));
     if (maskmode == 1) {
-        for (uint i = 0; i < l->len; i++) abAppend(&ab,"*",1);
+        unsigned int i;
+        for (i = 0; i < l->len; i++) abAppend(&ab,"*",1);
     } else {
         abAppend(&ab,l->buf,l->len);
     }

From 453e01a091b21c709b4e6163be7d03c8365cca13 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 13 Mar 2020 16:21:55 +0100
Subject: [PATCH 0169/1098] Restore newline at the end of redis-cli.c

---
 src/redis-cli.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index b44db9a1e..7ad80c0a1 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -8062,4 +8062,5 @@ int main(int argc, char **argv) {
     } else {
         return noninteractive(argc,convertToSds(argc,argv));
     }
-}
\ No newline at end of file
+}
+

From 606a01df708fa7b5cfc86ffcd2ce9aa805c825f9 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Sun, 15 Mar 2020 21:49:10 +0800
Subject: [PATCH 0170/1098] Threaded IO: bugfix #6988 process events while
 blocked

---
 src/networking.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index 4c394af70..261c9b24a 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -36,6 +36,7 @@
 
 static void setProtocolError(const char *errstr, client *c);
 int postponeClientRead(client *c);
+int process_while_blocked;
 
 /* Return the size consumed from the allocator, for the specified SDS string,
  * including internal fragmentation. This function is used in order to compute
@@ -2738,6 +2739,7 @@ int clientsArePaused(void) {
 int processEventsWhileBlocked(void) {
     int iterations = 4; /* See the function top-comment. */
     int count = 0;
+    process_while_blocked = 1;
     while (iterations--) {
         int events = 0;
         events += aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT);
@@ -2745,6 +2747,7 @@ int processEventsWhileBlocked(void) {
         if (!events) break;
         count += events;
     }
+    process_while_blocked = 0;
     return count;
 }
 
@@ -2816,6 +2819,7 @@ void *IOThreadMain(void *myid) {
 /* Initialize the data structures needed for threaded I/O. */
 void initThreadedIO(void) {
     io_threads_active = 0; /* We start with threads not active. */
+    process_while_blocked = 0; 
 
     /* Don't spawn any thread if the user selected a single thread:
      * we'll handle I/O directly from the main thread. */
@@ -2970,6 +2974,7 @@ int handleClientsWithPendingWritesUsingThreads(void) {
 int postponeClientRead(client *c) {
     if (io_threads_active &&
         server.io_threads_do_reads &&
+        !process_while_blocked &&
         !(c->flags & (CLIENT_MASTER|CLIENT_SLAVE|CLIENT_PENDING_READ)))
     {
         c->flags |= CLIENT_PENDING_READ;

From 573c4673eec95ca7d92d1a47f1461c3621b15f90 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Sun, 15 Mar 2020 16:10:37 +0100
Subject: [PATCH 0171/1098] Aesthetic changes in PR #6989.

---
 src/networking.c | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 261c9b24a..c7dd2f23c 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -36,7 +36,7 @@
 
 static void setProtocolError(const char *errstr, client *c);
 int postponeClientRead(client *c);
-int process_while_blocked;
+int ProcessingEventsWhileBlocked = 0; /* See processEventsWhileBlocked(). */
 
 /* Return the size consumed from the allocator, for the specified SDS string,
  * including internal fragmentation. This function is used in order to compute
@@ -2739,7 +2739,12 @@ int clientsArePaused(void) {
 int processEventsWhileBlocked(void) {
     int iterations = 4; /* See the function top-comment. */
     int count = 0;
-    process_while_blocked = 1;
+
+    /* Note: when we are processing events while blocked (for instance during
+     * busy Lua scripts), we set a global flag. When such flag is set, we
+     * avoid handling the read part of clients using threaded I/O.
+     * See https://github.com/antirez/redis/issues/6988 for more info. */
+    ProcessingEventsWhileBlocked = 1;
     while (iterations--) {
         int events = 0;
         events += aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT);
@@ -2747,7 +2752,7 @@ int processEventsWhileBlocked(void) {
         if (!events) break;
         count += events;
     }
-    process_while_blocked = 0;
+    ProcessingEventsWhileBlocked = 0;
     return count;
 }
 
@@ -2819,7 +2824,6 @@ void *IOThreadMain(void *myid) {
 /* Initialize the data structures needed for threaded I/O. */
 void initThreadedIO(void) {
     io_threads_active = 0; /* We start with threads not active. */
-    process_while_blocked = 0; 
 
     /* Don't spawn any thread if the user selected a single thread:
      * we'll handle I/O directly from the main thread. */
@@ -2974,7 +2978,7 @@ int handleClientsWithPendingWritesUsingThreads(void) {
 int postponeClientRead(client *c) {
     if (io_threads_active &&
         server.io_threads_do_reads &&
-        !process_while_blocked &&
+        !ProcessingEventsWhileBlocked &&
         !(c->flags & (CLIENT_MASTER|CLIENT_SLAVE|CLIENT_PENDING_READ)))
     {
         c->flags |= CLIENT_PENDING_READ;

From a6a0e05a1a3573f03d3c42af2ec9aee378bb66b9 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Sun, 15 Mar 2020 23:30:25 +0800
Subject: [PATCH 0172/1098] Threaded IO: bugfix client kill may crash redis

---
 src/networking.c | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 261c9b24a..b6ae9bd50 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -3036,16 +3036,22 @@ int handleClientsWithPendingReadsUsingThreads(void) {
     if (tio_debug) printf("I/O READ All threads finshed\n");
 
     /* Run the list of clients again to process the new buffers. */
-    listRewind(server.clients_pending_read,&li);
-    while((ln = listNext(&li))) {
+    while(listLength(server.clients_pending_read)) {
+        ln = listFirst(server.clients_pending_read);
         client *c = listNodeValue(ln);
         c->flags &= ~CLIENT_PENDING_READ;
+        listDelNode(server.clients_pending_read,ln);
+
         if (c->flags & CLIENT_PENDING_COMMAND) {
-            c->flags &= ~ CLIENT_PENDING_COMMAND;
-            processCommandAndResetClient(c);
+            c->flags &= ~CLIENT_PENDING_COMMAND;
+            if (processCommandAndResetClient(c) == C_ERR) {
+                /* If the client is no longer valid, we avoid
+                 * processing the client later. So we just go
+                 * to the next. */
+                continue;
+            }
         }
         processInputBufferAndReplicate(c);
     }
-    listEmpty(server.clients_pending_read);
     return processed;
 }

From c46c76a3991d769f3d8dff8d221555ac07981801 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Mon, 16 Mar 2020 11:20:48 +0800
Subject: [PATCH 0173/1098] Threaded IO: handle pending reads clients ASAP
 after event loop

---
 src/server.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index a6d4b357e..f702da94a 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2088,6 +2088,9 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
 void beforeSleep(struct aeEventLoop *eventLoop) {
     UNUSED(eventLoop);
 
+    /* We should handle pending reads clients ASAP after event loop. */
+    handleClientsWithPendingReadsUsingThreads();
+
     /* Handle TLS pending data. (must be done before flushAppendOnlyFile) */
     tlsProcessPendingData();
     /* If tls still has pending unread data don't sleep at all. */
@@ -2157,7 +2160,6 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
 void afterSleep(struct aeEventLoop *eventLoop) {
     UNUSED(eventLoop);
     if (moduleCount()) moduleAcquireGIL();
-    handleClientsWithPendingReadsUsingThreads();
 }
 
 /* =========================== Server initialization ======================== */

From 8609e681614f29da43209603f6dfbcd58e1b490d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 16 Mar 2020 13:48:29 +0100
Subject: [PATCH 0174/1098] Example sentinel conf: document requirepass.

---
 sentinel.conf | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/sentinel.conf b/sentinel.conf
index bc9a705ac..796f45088 100644
--- a/sentinel.conf
+++ b/sentinel.conf
@@ -112,6 +112,14 @@ sentinel monitor mymaster 127.0.0.1 6379 2
 # Default is 30 seconds.
 sentinel down-after-milliseconds mymaster 30000
 
+# requirepass <password>
+#
+# You can configure Sentinel itself to require a password, however when doing
+# so Sentinel will try to authenticate with the same password to all the
+# other Sentinels. So you need to configure all your Sentinels in a given
+# group with the same "requirepass" password. Check the following documentation
+# for more info: https://redis.io/topics/sentinel
+
 # sentinel parallel-syncs <master-name> <numreplicas>
 #
 # How many replicas we can reconfigure to point to the new replica simultaneously

From 9321c7871f41364fbe539d6b7711f5ccc7c90bdc Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 16 Mar 2020 15:59:29 +0100
Subject: [PATCH 0175/1098] Sentinel: implement auth-user directive for ACLs.

---
 src/sentinel.c | 45 ++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 7 deletions(-)

diff --git a/src/sentinel.c b/src/sentinel.c
index 10c003d03..83d6c00bb 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -205,7 +205,8 @@ typedef struct sentinelRedisInstance {
     dict *slaves;       /* Slaves for this master instance. */
     unsigned int quorum;/* Number of sentinels that need to agree on failure. */
     int parallel_syncs; /* How many slaves to reconfigure at same time. */
-    char *auth_pass;    /* Password to use for AUTH against master & slaves. */
+    char *auth_pass;    /* Password to use for AUTH against master & replica. */
+    char *auth_user;    /* Username for ACLs AUTH against master & replica. */
 
     /* Slave specific. */
     mstime_t master_link_down_time; /* Slave replication link down time. */
@@ -1231,6 +1232,7 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char *
                             SENTINEL_DEFAULT_DOWN_AFTER;
     ri->master_link_down_time = 0;
     ri->auth_pass = NULL;
+    ri->auth_user = NULL;
     ri->slave_priority = SENTINEL_DEFAULT_SLAVE_PRIORITY;
     ri->slave_reconf_sent_time = 0;
     ri->slave_master_host = NULL;
@@ -1289,6 +1291,7 @@ void releaseSentinelRedisInstance(sentinelRedisInstance *ri) {
     sdsfree(ri->slave_master_host);
     sdsfree(ri->leader);
     sdsfree(ri->auth_pass);
+    sdsfree(ri->auth_user);
     sdsfree(ri->info);
     releaseSentinelAddr(ri->addr);
     dictRelease(ri->renamed_commands);
@@ -1654,19 +1657,19 @@ char *sentinelHandleConfiguration(char **argv, int argc) {
         ri->failover_timeout = atoi(argv[2]);
         if (ri->failover_timeout <= 0)
             return "negative or zero time parameter.";
-   } else if (!strcasecmp(argv[0],"parallel-syncs") && argc == 3) {
+    } else if (!strcasecmp(argv[0],"parallel-syncs") && argc == 3) {
         /* parallel-syncs <name> <milliseconds> */
         ri = sentinelGetMasterByName(argv[1]);
         if (!ri) return "No such master with specified name.";
         ri->parallel_syncs = atoi(argv[2]);
-   } else if (!strcasecmp(argv[0],"notification-script") && argc == 3) {
+    } else if (!strcasecmp(argv[0],"notification-script") && argc == 3) {
         /* notification-script <name> <path> */
         ri = sentinelGetMasterByName(argv[1]);
         if (!ri) return "No such master with specified name.";
         if (access(argv[2],X_OK) == -1)
             return "Notification script seems non existing or non executable.";
         ri->notification_script = sdsnew(argv[2]);
-   } else if (!strcasecmp(argv[0],"client-reconfig-script") && argc == 3) {
+    } else if (!strcasecmp(argv[0],"client-reconfig-script") && argc == 3) {
         /* client-reconfig-script <name> <path> */
         ri = sentinelGetMasterByName(argv[1]);
         if (!ri) return "No such master with specified name.";
@@ -1674,11 +1677,16 @@ char *sentinelHandleConfiguration(char **argv, int argc) {
             return "Client reconfiguration script seems non existing or "
                    "non executable.";
         ri->client_reconfig_script = sdsnew(argv[2]);
-   } else if (!strcasecmp(argv[0],"auth-pass") && argc == 3) {
+    } else if (!strcasecmp(argv[0],"auth-pass") && argc == 3) {
         /* auth-pass <name> <password> */
         ri = sentinelGetMasterByName(argv[1]);
         if (!ri) return "No such master with specified name.";
         ri->auth_pass = sdsnew(argv[2]);
+    } else if (!strcasecmp(argv[0],"auth-user") && argc == 3) {
+        /* auth-user <name> <username> */
+        ri = sentinelGetMasterByName(argv[1]);
+        if (!ri) return "No such master with specified name.";
+        ri->auth_user = sdsnew(argv[2]);
     } else if (!strcasecmp(argv[0],"current-epoch") && argc == 2) {
         /* current-epoch <epoch> */
         unsigned long long current_epoch = strtoull(argv[1],NULL,10);
@@ -1836,7 +1844,7 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
             rewriteConfigRewriteLine(state,"sentinel",line,1);
         }
 
-        /* sentinel auth-pass */
+        /* sentinel auth-pass & auth-user */
         if (master->auth_pass) {
             line = sdscatprintf(sdsempty(),
                 "sentinel auth-pass %s %s",
@@ -1844,6 +1852,13 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
             rewriteConfigRewriteLine(state,"sentinel",line,1);
         }
 
+        if (master->auth_user) {
+            line = sdscatprintf(sdsempty(),
+                "sentinel auth-user %s %s",
+                master->name, master->auth_user);
+            rewriteConfigRewriteLine(state,"sentinel",line,1);
+        }
+
         /* sentinel config-epoch */
         line = sdscatprintf(sdsempty(),
             "sentinel config-epoch %s %llu",
@@ -1968,19 +1983,29 @@ werr:
  * will disconnect and reconnect the link and so forth. */
 void sentinelSendAuthIfNeeded(sentinelRedisInstance *ri, redisAsyncContext *c) {
     char *auth_pass = NULL;
+    char *auth_user = NULL;
 
     if (ri->flags & SRI_MASTER) {
         auth_pass = ri->auth_pass;
+        auth_user = ri->auth_user;
     } else if (ri->flags & SRI_SLAVE) {
         auth_pass = ri->master->auth_pass;
+        auth_user = ri->master->auth_user;
     } else if (ri->flags & SRI_SENTINEL) {
         auth_pass = ACLDefaultUserFirstPassword();
+        auth_user = NULL;
     }
 
-    if (auth_pass) {
+    if (auth_pass && auth_user == NULL) {
         if (redisAsyncCommand(c, sentinelDiscardReplyCallback, ri, "%s %s",
             sentinelInstanceMapCommand(ri,"AUTH"),
             auth_pass) == C_OK) ri->link->pending_commands++;
+    } else if (auth_pass && auth_user) {
+        /* If we also have an username, use the ACL-style AUTH command
+         * with two arguments, username and password. */
+        if (redisAsyncCommand(c, sentinelDiscardReplyCallback, ri, "%s %s %s",
+            sentinelInstanceMapCommand(ri,"AUTH"),
+            auth_user, auth_pass) == C_OK) ri->link->pending_commands++;
     }
 }
 
@@ -3522,6 +3547,12 @@ void sentinelSetCommand(client *c) {
             sdsfree(ri->auth_pass);
             ri->auth_pass = strlen(value) ? sdsnew(value) : NULL;
             changes++;
+        } else if (!strcasecmp(option,"auth-user") && moreargs > 0) {
+            /* auth-user <username> */
+            char *value = c->argv[++j]->ptr;
+            sdsfree(ri->auth_user);
+            ri->auth_user = strlen(value) ? sdsnew(value) : NULL;
+            changes++;
         } else if (!strcasecmp(option,"quorum") && moreargs > 0) {
             /* quorum <count> */
             robj *o = c->argv[++j];

From 771df8a436d7a01f95bb319ff2b342138c33d2bb Mon Sep 17 00:00:00 2001
From: artix <artix2@gmail.com>
Date: Tue, 18 Feb 2020 10:46:10 +0100
Subject: [PATCH 0176/1098] Support Redis Cluster Proxy PROXY INFO command

---
 src/redis-cli.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 7ad80c0a1..7e440d67c 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -1292,7 +1292,11 @@ static int cliSendCommand(int argc, char **argv, long repeat) {
         (argc == 3 && !strcasecmp(command,"latency") &&
                        !strcasecmp(argv[1],"graph")) ||
         (argc == 2 && !strcasecmp(command,"latency") &&
-                       !strcasecmp(argv[1],"doctor")))
+                       !strcasecmp(argv[1],"doctor")) ||
+        /* Format PROXY INFO command for Redis Cluster Proxy:
+         * https://github.com/artix75/redis-cluster-proxy */
+        (argc >= 2 && !strcasecmp(command,"proxy") &&
+                       !strcasecmp(argv[1],"info")))
     {
         output_raw = 1;
     }

From 29b9d0a245135a0b9e024032a7cccd2c238103a9 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 16 Mar 2020 16:56:50 +0100
Subject: [PATCH 0177/1098] ACL: Make Redis 6 more backward compatible with
 requirepass.

Note that this as a side effect fixes Sentinel "requirepass" mode.
---
 src/acl.c      | 11 +----------
 src/config.c   | 16 ++++++++++++----
 src/sentinel.c |  2 +-
 src/server.h   |  3 +++
 4 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index efe6b96ad..27f4bdb84 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -899,16 +899,6 @@ char *ACLSetUserStringError(void) {
     return errmsg;
 }
 
-/* Return the first password of the default user or NULL.
- * This function is needed for backward compatibility with the old
- * directive "requirepass" when Redis supported a single global
- * password. */
-sds ACLDefaultUserFirstPassword(void) {
-    if (listLength(DefaultUser->passwords) == 0) return NULL;
-    listNode *first = listFirst(DefaultUser->passwords);
-    return listNodeValue(first);
-}
-
 /* Initialize the default user, that will always exist for all the process
  * lifetime. */
 void ACLInitDefaultUser(void) {
@@ -925,6 +915,7 @@ void ACLInit(void) {
     UsersToLoad = listCreate();
     ACLLog = listCreate();
     ACLInitDefaultUser();
+    server.requirepass = NULL; /* Only used for backward compatibility. */
 }
 
 /* Check the username and password pair and return C_OK if they are valid,
diff --git a/src/config.c b/src/config.c
index 211b6d003..7c87ebe6e 100644
--- a/src/config.c
+++ b/src/config.c
@@ -411,11 +411,15 @@ void loadServerConfigFromString(char *config) {
                 goto loaderr;
             }
             /* The old "requirepass" directive just translates to setting
-             * a password to the default user. */
+             * a password to the default user. The only thing we do
+             * additionally is to remember the cleartext password in this
+             * case, for backward compatibility with Redis <= 5. */
             ACLSetUser(DefaultUser,"resetpass",-1);
             sds aclop = sdscatprintf(sdsempty(),">%s",argv[1]);
             ACLSetUser(DefaultUser,aclop,sdslen(aclop));
             sdsfree(aclop);
+            sdsfree(server.requirepass);
+            server.requirepass = sdsnew(argv[1]);
         } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){
             /* DEAD OPTION */
         } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) {
@@ -623,11 +627,15 @@ void configSetCommand(client *c) {
     config_set_special_field("requirepass") {
         if (sdslen(o->ptr) > CONFIG_AUTHPASS_MAX_LEN) goto badfmt;
         /* The old "requirepass" directive just translates to setting
-         * a password to the default user. */
+         * a password to the default user. The only thing we do
+         * additionally is to remember the cleartext password in this
+         * case, for backward compatibility with Redis <= 5. */
         ACLSetUser(DefaultUser,"resetpass",-1);
         sds aclop = sdscatprintf(sdsempty(),">%s",(char*)o->ptr);
         ACLSetUser(DefaultUser,aclop,sdslen(aclop));
         sdsfree(aclop);
+        sdsfree(server.requirepass);
+        server.requirepass = sdsnew(o->ptr);
     } config_set_special_field("save") {
         int vlen, j;
         sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen);
@@ -899,7 +907,7 @@ void configGetCommand(client *c) {
     }
     if (stringmatch(pattern,"requirepass",1)) {
         addReplyBulkCString(c,"requirepass");
-        sds password = ACLDefaultUserFirstPassword();
+        sds password = server.requirepass;
         if (password) {
             addReplyBulkCBuffer(c,password,sdslen(password));
         } else {
@@ -1341,7 +1349,7 @@ void rewriteConfigBindOption(struct rewriteConfigState *state) {
 void rewriteConfigRequirepassOption(struct rewriteConfigState *state, char *option) {
     int force = 1;
     sds line;
-    sds password = ACLDefaultUserFirstPassword();
+    sds password = server.requirepass;
 
     /* If there is no password set, we don't want the requirepass option
      * to be present in the configuration at all. */
diff --git a/src/sentinel.c b/src/sentinel.c
index 83d6c00bb..d091bf230 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -1992,7 +1992,7 @@ void sentinelSendAuthIfNeeded(sentinelRedisInstance *ri, redisAsyncContext *c) {
         auth_pass = ri->master->auth_pass;
         auth_user = ri->master->auth_user;
     } else if (ri->flags & SRI_SENTINEL) {
-        auth_pass = ACLDefaultUserFirstPassword();
+        auth_pass = server.requirepass;
         auth_user = NULL;
     }
 
diff --git a/src/server.h b/src/server.h
index 3c19a17ea..fa6770dfa 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1395,6 +1395,9 @@ struct redisServer {
     /* ACLs */
     char *acl_filename;     /* ACL Users file. NULL if not configured. */
     unsigned long acllog_max_len; /* Maximum length of the ACL LOG list. */
+    sds requirepass;        /* Remember the cleartext password set with the
+                               old "requirepass" directive for backward
+                               compatibility with Redis <= 5. */
     /* Assert & bug reporting */
     const char *assert_failed;
     const char *assert_file;

From b3a97004f4c5555158f774279e5e2131cf909a6d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 16 Mar 2020 17:11:43 +0100
Subject: [PATCH 0178/1098] Sentinel: document auth-user directive.

---
 sentinel.conf | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/sentinel.conf b/sentinel.conf
index 796f45088..4ca5e5f8f 100644
--- a/sentinel.conf
+++ b/sentinel.conf
@@ -102,6 +102,18 @@ sentinel monitor mymaster 127.0.0.1 6379 2
 #
 # sentinel auth-pass mymaster MySUPER--secret-0123passw0rd
 
+# sentinel auth-user <master-name> <username>
+#
+# This is useful in order to authenticate to instances having ACL capabilities,
+# that is, running Redis 6.0 or greater. When just auth-pass is provided the
+# Sentinel instance will authenticate to Redis using the old "AUTH <pass>"
+# method. When also an username is provided, it will use "AUTH <user> <pass>".
+# In the Redis servers side, the ACL to provide just minimal access to
+# Sentinel instances, should be configured along the following lines:
+#
+#     user sentinel-user >somepassword +client +subscribe +publish \
+#                        +ping +info +multi +slaveof +config +client +exec on
+
 # sentinel down-after-milliseconds <master-name> <milliseconds>
 #
 # Number of milliseconds the master (or any attached replica or sentinel) should

From d9de9d5478909231dd544c2e4782a891aadc6b8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=EB=B0=95=EC=8A=B9=ED=98=84?= <ppillip@gmail.com>
Date: Wed, 18 Mar 2020 14:40:50 +0900
Subject: [PATCH 0179/1098] Update redis.conf

---
 redis.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/redis.conf b/redis.conf
index c9d256bef..7c55a3ab0 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1628,7 +1628,7 @@ hz 10
 # offers, and enables by default, the ability to use an adaptive HZ value
 # which will temporary raise when there are many connected clients.
 #
-# When dynamic HZ is enabled, the actual configured HZ will be used as
+# When dynamic HZ is enabled, the actual configured HZ will be used
 # as a baseline, but multiples of the configured HZ value will be actually
 # used as needed once more clients are connected. In this way an idle
 # instance will use very little CPU time while a busy instance will be

From af5167b7f35972fd21cd8f8566da7339ee07809a Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Wed, 18 Mar 2020 16:17:46 +0800
Subject: [PATCH 0180/1098] Add 14-consistency-check.tcl to prove there is a
 data consistency issue.

---
 tests/cluster/tests/14-consistency-check.tcl | 87 ++++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100644 tests/cluster/tests/14-consistency-check.tcl

diff --git a/tests/cluster/tests/14-consistency-check.tcl b/tests/cluster/tests/14-consistency-check.tcl
new file mode 100644
index 000000000..a43725ebc
--- /dev/null
+++ b/tests/cluster/tests/14-consistency-check.tcl
@@ -0,0 +1,87 @@
+source "../tests/includes/init-tests.tcl"
+
+test "Create a 5 nodes cluster" {
+    create_cluster 5 5
+}
+
+test "Cluster should start ok" {
+    assert_cluster_state ok
+}
+
+test "Cluster is writable" {
+    cluster_write_test 0
+}
+
+proc find_non_empty_master {} {
+    set master_id_no {}
+    foreach_redis_id id {
+        if {[RI $id role] eq {master} && [R $id dbsize] > 0} {
+            set master_id_no $id
+        }
+    }
+    return $master_id_no
+}
+
+proc get_one_of_my_replica {id} {
+    set replica_port [lindex [lindex [lindex [R $id role] 2] 0] 1]
+    set replica_id_num [get_instance_id_by_port redis $replica_port]
+    return $replica_id_num
+}
+
+proc cluster_write_keys_with_expire {id ttl} {
+    set prefix [randstring 20 20 alpha]
+    set port [get_instance_attrib redis $id port]
+    set cluster [redis_cluster 127.0.0.1:$port]
+    for {set j 100} {$j < 200} {incr j} {
+        $cluster setex key_expire.$j $ttl $prefix.$j
+    }
+    $cluster close
+}
+
+proc test_slave_load_expired_keys {aof} {
+    test "Slave expired keys is loaded when restarted: appendonly=$aof" {
+        set master_id [find_non_empty_master]
+        set replica_id [get_one_of_my_replica $master_id]
+
+        set master_dbsize [R $master_id dbsize]
+        set slave_dbsize [R $replica_id dbsize]
+        assert_equal $master_dbsize $slave_dbsize
+
+        set data_ttl 5
+        cluster_write_keys_with_expire $master_id $data_ttl
+        after 100
+        set replica_dbsize_1 [R $replica_id dbsize]
+        assert {$replica_dbsize_1  > $slave_dbsize}
+
+        R $replica_id config set appendonly $aof
+        R $replica_id config rewrite
+
+        set start_time [clock seconds]
+        set end_time [expr $start_time+$data_ttl+2]
+        R $replica_id save
+        set replica_dbsize_2 [R $replica_id dbsize]
+        assert {$replica_dbsize_2  > $slave_dbsize}
+        kill_instance redis $replica_id
+
+        set master_port [get_instance_attrib redis $master_id port]
+        exec ../../../src/redis-cli -h 127.0.0.1 -p $master_port debug sleep [expr $data_ttl+3] > /dev/null &
+
+        while {[clock seconds] <= $end_time} {
+            #wait for $data_ttl seconds
+        }
+        restart_instance redis $replica_id
+
+        wait_for_condition 200 50 {
+            [R $replica_id ping] eq {PONG}
+        } else {
+            fail "replica #$replica_id not started"
+        }
+
+        set replica_dbsize_3 [R $replica_id dbsize]
+        assert {$replica_dbsize_3  > $slave_dbsize}
+    }
+}
+
+test_slave_load_expired_keys no
+after 5000
+test_slave_load_expired_keys yes

From f6029fb9256ee1ffa1491f6fa57bb56a9558e952 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Wed, 18 Mar 2020 16:20:10 +0800
Subject: [PATCH 0181/1098] Fix master replica inconsistency for upgrading
 scenario.

Before this commit, when upgrading a replica, expired keys will not
be loaded, thus causing replica having less keys in db. To this point,
master and replica's keys is logically consistent. However, before
the keys in master and replica are physically consistent, that is,
they have the same dbsize, if master got a problem and the replica
got promoted and becomes new master of that partition, and master
updates a key which does not exist on master, but physically exists
on the old master(new replica), the old master would refuse to update
the key, thus causing master and replica data inconsistent.

How could this happen?
That's all because of the wrong judgement of roles while starting up
the server. We can not use server.masterhost to judge if the server
is master or replica, since it fails in cluster mode.

When we start the server, we load rdb and do want to load expired keys,
and do not want to have the ability to active expire keys, if it is
a replica.
---
 src/rdb.c    | 2 +-
 src/server.c | 7 ++++++-
 src/server.h | 2 ++
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/rdb.c b/src/rdb.c
index cbcea96c6..5d34f5a32 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2231,7 +2231,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
          * received from the master. In the latter case, the master is
          * responsible for key expiry. If we would expire keys here, the
          * snapshot taken by the master may not be reflected on the slave. */
-        if (server.masterhost == NULL && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now) {
+        if (iAmMaster() && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now) {
             decrRefCount(key);
             decrRefCount(val);
         } else {
diff --git a/src/server.c b/src/server.c
index f702da94a..467d09b67 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1691,7 +1691,7 @@ void databasesCron(void) {
     /* Expire keys by random sampling. Not required for slaves
      * as master will synthesize DELs for us. */
     if (server.active_expire_enabled) {
-        if (server.masterhost == NULL) {
+        if (iAmMaster()) {
             activeExpireCycle(ACTIVE_EXPIRE_CYCLE_SLOW);
         } else {
             expireSlaveKeys();
@@ -4863,6 +4863,11 @@ int redisIsSupervised(int mode) {
     return 0;
 }
 
+int iAmMaster(void) {
+    return ((!server.cluster_enabled && server.masterhost == NULL) ||
+            (server.cluster_enabled && nodeIsMaster(server.cluster->myself)));
+}
+
 
 int main(int argc, char **argv) {
     struct timeval tv;
diff --git a/src/server.h b/src/server.h
index fa6770dfa..fdfe5b8ea 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2393,4 +2393,6 @@ int tlsConfigure(redisTLSContextConfig *ctx_config);
 #define redisDebugMark() \
     printf("-- MARK %s:%d --\n", __FILE__, __LINE__)
 
+int iAmMaster(void);
+
 #endif

From f16eaadd4ff4ebebc5661e5570301ab0c31bf1f6 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Wed, 18 Mar 2020 18:34:27 +0530
Subject: [PATCH 0182/1098] Allow RM_GetContextFlags to work with ctx==NULL

---
 src/module.c | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/src/module.c b/src/module.c
index bbd54082c..a5f4219fd 100644
--- a/src/module.c
+++ b/src/module.c
@@ -1848,20 +1848,22 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) {
 
     int flags = 0;
     /* Client specific flags */
-    if (ctx->client) {
-        if (ctx->client->flags & CLIENT_LUA)
-         flags |= REDISMODULE_CTX_FLAGS_LUA;
-        if (ctx->client->flags & CLIENT_MULTI)
-         flags |= REDISMODULE_CTX_FLAGS_MULTI;
-        /* Module command recieved from MASTER, is replicated. */
-        if (ctx->client->flags & CLIENT_MASTER)
-         flags |= REDISMODULE_CTX_FLAGS_REPLICATED;
-    }
+    if (ctx) {
+        if (ctx->client) {
+            if (ctx->client->flags & CLIENT_LUA)
+             flags |= REDISMODULE_CTX_FLAGS_LUA;
+            if (ctx->client->flags & CLIENT_MULTI)
+             flags |= REDISMODULE_CTX_FLAGS_MULTI;
+            /* Module command recieved from MASTER, is replicated. */
+            if (ctx->client->flags & CLIENT_MASTER)
+             flags |= REDISMODULE_CTX_FLAGS_REPLICATED;
+        }
 
-    /* For DIRTY flags, we need the blocked client if used */
-    client *c = ctx->blocked_client ? ctx->blocked_client->client : ctx->client;
-    if (c && (c->flags & (CLIENT_DIRTY_CAS|CLIENT_DIRTY_EXEC))) {
-        flags |= REDISMODULE_CTX_FLAGS_MULTI_DIRTY;
+        /* For DIRTY flags, we need the blocked client if used */
+        client *c = ctx->blocked_client ? ctx->blocked_client->client : ctx->client;
+        if (c && (c->flags & (CLIENT_DIRTY_CAS|CLIENT_DIRTY_EXEC))) {
+            flags |= REDISMODULE_CTX_FLAGS_MULTI_DIRTY;
+        }
     }
 
     if (server.cluster_enabled)

From 86691ccff51b0065d430836e15c81464cc691765 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Wed, 18 Mar 2020 09:33:52 -0400
Subject: [PATCH 0183/1098] fix potentical memory leak in redis-cli

---
 src/redis-cli.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 7ad80c0a1..c0cc69592 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -1989,6 +1989,8 @@ static void repl(void) {
                     if (config.eval) {
                         config.eval_ldb = 1;
                         config.output = OUTPUT_RAW;
+                        sdsfreesplitres(argv,argc);
+                        linenoiseFree(line);
                         return; /* Return to evalMode to restart the session. */
                     } else {
                         printf("Use 'restart' only in Lua debugging mode.");

From 262262fbb278b3fae5b64c085d182b1b3daec891 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Wed, 18 Mar 2020 09:48:03 -0400
Subject: [PATCH 0184/1098] fix spelling in cluster.c

---
 src/cluster.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cluster.c b/src/cluster.c
index 5f63d2b8f..f67e97abd 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -932,7 +932,7 @@ int clusterAddNode(clusterNode *node) {
     return (retval == DICT_OK) ? C_OK : C_ERR;
 }
 
-/* Remove a node from the cluster. The functio performs the high level
+/* Remove a node from the cluster. The function performs the high level
  * cleanup, calling freeClusterNode() for the low level cleanup.
  * Here we do the following:
  *

From f9c56dbb09fca67e2b82e5aa789cfb7af0b123be Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 20 Mar 2020 12:45:48 +0100
Subject: [PATCH 0185/1098] ACL: default user off should not allow automatic
 authentication.

This fixes issue #7011.
---
 src/networking.c | 3 ++-
 src/server.c     | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 0690bbdf6..69d59a59b 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -124,7 +124,8 @@ client *createClient(connection *conn) {
     c->ctime = c->lastinteraction = server.unixtime;
     /* If the default user does not require authentication, the user is
      * directly authenticated. */
-    c->authenticated = (c->user->flags & USER_FLAG_NOPASS) != 0;
+    c->authenticated = (c->user->flags & USER_FLAG_NOPASS) &&
+                       !(c->user->flags & USER_FLAG_DISABLED);
     c->replstate = REPL_STATE_NONE;
     c->repl_put_online_on_ack = 0;
     c->reploff = 0;
diff --git a/src/server.c b/src/server.c
index f702da94a..612805ce5 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3380,7 +3380,7 @@ int processCommand(client *c) {
     /* Check if the user is authenticated. This check is skipped in case
      * the default user is flagged as "nopass" and is active. */
     int auth_required = (!(DefaultUser->flags & USER_FLAG_NOPASS) ||
-                           DefaultUser->flags & USER_FLAG_DISABLED) &&
+                          (DefaultUser->flags & USER_FLAG_DISABLED)) &&
                         !c->authenticated;
     if (auth_required) {
         /* AUTH and HELLO and no auth modules are valid even in

From 5497a44037a01a862c6e522faa6dbc3fe96c738f Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 20 Mar 2020 12:52:06 +0100
Subject: [PATCH 0186/1098] Regression test for #7011.

---
 tests/unit/acl.tcl | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl
index fc1664a75..85c9b81a9 100644
--- a/tests/unit/acl.tcl
+++ b/tests/unit/acl.tcl
@@ -248,4 +248,11 @@ start_server {tags {"acl"}} {
         r AUTH default ""
         assert {[llength [r ACL LOG]] == 5}
     }
+
+    test {When default user is off, new connections are not authenticated} {
+        r ACL setuser default off
+        catch {set rd1 [redis_deferring_client]} e
+        r ACL setuser default on
+        set e
+    } {*NOAUTH*}
 }

From 93bb42a0b568cb3dbddf9fb6b4dd7f3a8e0240fc Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Fri, 20 Mar 2020 02:40:54 -0400
Subject: [PATCH 0187/1098] add missing commands in cluster help

---
 src/cluster.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/cluster.c b/src/cluster.c
index 5f63d2b8f..72755823a 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4261,7 +4261,7 @@ void clusterCommand(client *c) {
 "FORGET <node-id> -- Remove a node from the cluster.",
 "GETKEYSINSLOT <slot> <count> -- Return key names stored by current node in a slot.",
 "FLUSHSLOTS -- Delete current node own slots information.",
-"INFO - Return onformation about the cluster.",
+"INFO - Return information about the cluster.",
 "KEYSLOT <key> -- Return the hash slot for <key>.",
 "MEET <ip> <port> [bus-port] -- Connect nodes into a working cluster.",
 "MYID -- Return the node id.",
@@ -4272,6 +4272,7 @@ void clusterCommand(client *c) {
 "SET-config-epoch <epoch> - Set config epoch of current node.",
 "SETSLOT <slot> (importing|migrating|stable|node <node-id>) -- Set slot state.",
 "REPLICAS <node-id> -- Return <node-id> replicas.",
+"SAVECONFIG - Force saving cluster configuration on disk.",
 "SLOTS -- Return information about slots range mappings. Each range is made of:",
 "    start, end, master and replicas IP addresses, ports and ids",
 NULL

From fa9aa90813ae67509208022f1315cbe82869c2cc Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 22 Mar 2020 14:42:03 +0200
Subject: [PATCH 0188/1098] Conns: Fix connClose() / connAccept() behavior.

We assume accept handlers may choose to reject a connection and close
it, but connAccept() callers can't distinguish between this state and
other error states requiring connClose().

This makes it safe (and mandatory!) to always call connClose() if
connAccept() fails, and safe for accept handlers to close connections
(which will defer).
---
 src/connection.c  | 12 ++++++++---
 src/connection.h  | 15 ++++++++++----
 src/connhelpers.h | 53 +++++++++++++++++++++++++----------------------
 3 files changed, 48 insertions(+), 32 deletions(-)

diff --git a/src/connection.c b/src/connection.c
index 58d86c31b..2015c9195 100644
--- a/src/connection.c
+++ b/src/connection.c
@@ -152,7 +152,7 @@ static void connSocketClose(connection *conn) {
     /* If called from within a handler, schedule the close but
      * keep the connection until the handler returns.
      */
-    if (conn->flags & CONN_FLAG_IN_HANDLER) {
+    if (connHasRefs(conn)) {
         conn->flags |= CONN_FLAG_CLOSE_SCHEDULED;
         return;
     }
@@ -183,10 +183,16 @@ static int connSocketRead(connection *conn, void *buf, size_t buf_len) {
 }
 
 static int connSocketAccept(connection *conn, ConnectionCallbackFunc accept_handler) {
+    int ret = C_OK;
+
     if (conn->state != CONN_STATE_ACCEPTING) return C_ERR;
     conn->state = CONN_STATE_CONNECTED;
-    if (!callHandler(conn, accept_handler)) return C_ERR;
-    return C_OK;
+
+    connIncrRefs(conn);
+    if (!callHandler(conn, accept_handler)) ret = C_ERR;
+    connDecrRefs(conn);
+
+    return ret;
 }
 
 /* Register a write handler, to be called when the connection is writable.
diff --git a/src/connection.h b/src/connection.h
index 97622f8d6..db09dfd83 100644
--- a/src/connection.h
+++ b/src/connection.h
@@ -45,9 +45,8 @@ typedef enum {
     CONN_STATE_ERROR
 } ConnectionState;
 
-#define CONN_FLAG_IN_HANDLER        (1<<0)      /* A handler execution is in progress */
-#define CONN_FLAG_CLOSE_SCHEDULED   (1<<1)      /* Closed scheduled by a handler */
-#define CONN_FLAG_WRITE_BARRIER     (1<<2)      /* Write barrier requested */
+#define CONN_FLAG_CLOSE_SCHEDULED   (1<<0)      /* Closed scheduled by a handler */
+#define CONN_FLAG_WRITE_BARRIER     (1<<1)      /* Write barrier requested */
 
 typedef void (*ConnectionCallbackFunc)(struct connection *conn);
 
@@ -70,7 +69,8 @@ typedef struct ConnectionType {
 struct connection {
     ConnectionType *type;
     ConnectionState state;
-    int flags;
+    short int flags;
+    short int refs;
     int last_errno;
     void *private_data;
     ConnectionCallbackFunc conn_handler;
@@ -88,6 +88,13 @@ struct connection {
  * connAccept() may directly call accept_handler(), or return and call it
  * at a later time. This behavior is a bit awkward but aims to reduce the need
  * to wait for the next event loop, if no additional handshake is required.
+ *
+ * IMPORTANT: accept_handler may decide to close the connection, calling connClose().
+ * To make this safe, the connection is only marked with CONN_FLAG_CLOSE_SCHEDULED
+ * in this case, and connAccept() returns with an error.
+ *
+ * connAccept() callers must always check the return value and on error (C_ERR)
+ * a connClose() must be called.
  */
 
 static inline int connAccept(connection *conn, ConnectionCallbackFunc accept_handler) {
diff --git a/src/connhelpers.h b/src/connhelpers.h
index f237c9b1d..86250d09e 100644
--- a/src/connhelpers.h
+++ b/src/connhelpers.h
@@ -37,46 +37,49 @@
  * implementations (currently sockets in connection.c and TLS in tls.c).
  *
  * Currently helpers implement the mechanisms for invoking connection
- * handlers, tracking in-handler states and dealing with deferred
- * destruction (if invoked by a handler).
+ * handlers and tracking connection references, to allow safe destruction
+ * of connections from within a handler.
  */
 
-/* Called whenever a handler is invoked on a connection and sets the
- * CONN_FLAG_IN_HANDLER flag to indicate we're in a handler context.
+/* Incremenet connection references.
  *
- * An attempt to close a connection while CONN_FLAG_IN_HANDLER is
- * set will result with deferred close, i.e. setting the CONN_FLAG_CLOSE_SCHEDULED
- * instead of destructing it.
+ * Inside a connection handler, we guarantee refs >= 1 so it is always
+ * safe to connClose().
+ *
+ * In other cases where we don't want to prematurely lose the connection,
+ * it can go beyond 1 as well; currently it is only done by connAccept().
  */
-static inline void enterHandler(connection *conn) {
-    conn->flags |= CONN_FLAG_IN_HANDLER;
+static inline void connIncrRefs(connection *conn) {
+    conn->refs++;
 }
 
-/* Called whenever a handler returns. This unsets the CONN_FLAG_IN_HANDLER
- * flag and performs actual close/destruction if a deferred close was
- * scheduled by the handler.
+/* Decrement connection references.
+ *
+ * Note that this is not intended to provide any automatic free logic!
+ * callHandler() takes care of that for the common flows, and anywhere an
+ * explicit connIncrRefs() is used, the caller is expected to take care of
+ * that.
  */
-static inline int exitHandler(connection *conn) {
-    conn->flags &= ~CONN_FLAG_IN_HANDLER;
-    if (conn->flags & CONN_FLAG_CLOSE_SCHEDULED) {
-        connClose(conn);
-        return 0;
-    }
-    return 1;
+
+static inline void connDecrRefs(connection *conn) {
+    conn->refs--;
+}
+
+static inline int connHasRefs(connection *conn) {
+    return conn->refs;
 }
 
 /* Helper for connection implementations to call handlers:
- * 1. Mark the handler in use.
+ * 1. Increment refs to protect the connection.
  * 2. Execute the handler (if set).
- * 3. Mark the handler as NOT in use and perform deferred close if was
- * requested by the handler at any time.
+ * 3. Decrement refs and perform deferred close, if refs==0.
  */
 static inline int callHandler(connection *conn, ConnectionCallbackFunc handler) {
-    conn->flags |= CONN_FLAG_IN_HANDLER;
+    connIncrRefs(conn);
     if (handler) handler(conn);
-    conn->flags &= ~CONN_FLAG_IN_HANDLER;
+    connDecrRefs(conn);
     if (conn->flags & CONN_FLAG_CLOSE_SCHEDULED) {
-        connClose(conn);
+        if (!connHasRefs(conn)) connClose(conn);
         return 0;
     }
     return 1;

From 4c08ae3ff698025b7100648faae5a0ac3f9f31fc Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 22 Mar 2020 14:46:16 +0200
Subject: [PATCH 0189/1098] Cluster: fix misleading accept errors.

---
 src/cluster.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 72755823a..a2e9ff5b6 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -681,9 +681,10 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
          * or schedule it for later depending on connection implementation.
          */
         if (connAccept(conn, clusterConnAcceptHandler) == C_ERR) {
-            serverLog(LL_VERBOSE,
-                    "Error accepting cluster node connection: %s",
-                    connGetLastError(conn));
+            if (connGetState(conn) == CONN_STATE_ERROR)
+                serverLog(LL_VERBOSE,
+                        "Error accepting cluster node connection: %s",
+                        connGetLastError(conn));
             connClose(conn);
             return;
         }

From 2dab5015b79f7fe86c2efb0e4e7f26548c8096c9 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 22 Mar 2020 14:47:44 +0200
Subject: [PATCH 0190/1098] Fix crashes related to failed/rejected accepts.

---
 src/networking.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 69d59a59b..a550e4040 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -786,7 +786,7 @@ void clientAcceptHandler(connection *conn) {
         serverLog(LL_WARNING,
                 "Error accepting a client connection: %s",
                 connGetLastError(conn));
-        freeClient(c);
+        freeClientAsync(c);
         return;
     }
 
@@ -828,7 +828,7 @@ void clientAcceptHandler(connection *conn) {
                 /* Nothing to do, Just to avoid the warning... */
             }
             server.stat_rejected_conn++;
-            freeClient(c);
+            freeClientAsync(c);
             return;
         }
     }
@@ -887,9 +887,10 @@ static void acceptCommonHandler(connection *conn, int flags, char *ip) {
      */
     if (connAccept(conn, clientAcceptHandler) == C_ERR) {
         char conninfo[100];
-        serverLog(LL_WARNING,
-                "Error accepting a client connection: %s (conn: %s)",
-                connGetLastError(conn), connGetInfo(conn, conninfo, sizeof(conninfo)));
+        if (connGetState(conn) == CONN_STATE_ERROR)
+            serverLog(LL_WARNING,
+                    "Error accepting a client connection: %s (conn: %s)",
+                    connGetLastError(conn), connGetInfo(conn, conninfo, sizeof(conninfo)));
         freeClient(connGetPrivateData(conn));
         return;
     }

From a2732291cd73b1f99d697e11ae59a28a598f5ca9 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Mon, 23 Mar 2020 01:04:49 -0400
Subject: [PATCH 0191/1098] clean CLIENT_TRACKING_CACHING flag when disabled
 caching

---
 src/tracking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tracking.c b/src/tracking.c
index 45f83103a..5c1f48cba 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -94,7 +94,7 @@ void disableTracking(client *c) {
         server.tracking_clients--;
         c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR|
                       CLIENT_TRACKING_BCAST|CLIENT_TRACKING_OPTIN|
-                      CLIENT_TRACKING_OPTOUT);
+                      CLIENT_TRACKING_OPTOUT|CLIENT_TRACKING_CACHING);
     }
 }
 

From 34d5982bd50e2e9014dde0f6a53db87091cca44c Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Mon, 23 Mar 2020 01:07:46 -0400
Subject: [PATCH 0192/1098] remove redundant Semicolon

---
 src/tracking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tracking.c b/src/tracking.c
index 5c1f48cba..6f7929430 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -271,7 +271,7 @@ void trackingInvalidateKey(robj *keyobj) {
         trackingRememberKeyToBroadcast(sdskey,sdslen(sdskey));
 
     rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey));
-    if (ids == raxNotFound) return;;
+    if (ids == raxNotFound) return;
 
     raxIterator ri;
     raxStart(&ri,ids);

From 19f5be231d5b745f84f562d40b5e5c5aa1939393 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 23 Mar 2020 11:17:50 +0100
Subject: [PATCH 0193/1098] Modules: updated function doc after #7003.

---
 src/module.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/module.c b/src/module.c
index d6b055f78..6f61a5ca8 100644
--- a/src/module.c
+++ b/src/module.c
@@ -1795,7 +1795,12 @@ int RM_GetSelectedDb(RedisModuleCtx *ctx) {
  * current request context (whether the client is a Lua script or in a MULTI),
  * and about the Redis instance in general, i.e replication and persistence.
  *
- * The available flags are:
+ * It is possible to call this function even with a NULL context, however
+ * in this case the following flags will not be reported:
+ *
+ *  * LUA, MULTI, REPLICATED, DIRTY (see below for more info).
+ *
+ * Available flags and their meaning:
  *
  *  * REDISMODULE_CTX_FLAGS_LUA: The command is running in a Lua script
  *

From 38514e3c8dd9ad6c0b788c25afa7db38aa26f5c3 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 23 Mar 2020 11:28:09 +0100
Subject: [PATCH 0194/1098] Minor changes to BITFIELD_RO PR #6951.

---
 src/bitops.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/bitops.c b/src/bitops.c
index ffb330013..d4e82c937 100644
--- a/src/bitops.c
+++ b/src/bitops.c
@@ -902,8 +902,8 @@ void bitposCommand(client *c) {
  * OVERFLOW [WRAP|SAT|FAIL]
  */
 
-#define BITFIELD_COMMON (1<<0)
-#define BITFIELD_READONLY (1<<1)
+#define BITFIELD_FLAG_NONE      0
+#define BITFIELD_FLAG_READONLY  (1<<0)
 
 struct bitfieldOp {
     uint64_t offset;    /* Bitfield offset. */
@@ -914,6 +914,9 @@ struct bitfieldOp {
     int sign;           /* True if signed, otherwise unsigned op. */
 };
 
+/* This implements both the BITFIELD command and the BITFIELD_RO command
+ * when flags is set to BITFIELD_FLAG_READONLY: in this case only the
+ * GET subcommand is allowed, other subcommands will return an error. */
 void bitfieldGeneric(client *c, int flags) {
     robj *o;
     size_t bitoffset;
@@ -1002,9 +1005,9 @@ void bitfieldGeneric(client *c, int flags) {
             return;
         }
     } else {
-        if (flags & BITFIELD_READONLY) {
+        if (flags & BITFIELD_FLAG_READONLY) {
             zfree(ops);
-            addReplyError(c, "bitfield_ro only support get subcommand");
+            addReplyError(c, "BITFIELD_RO only support the GET subcommand");
             return;
         }
 
@@ -1140,9 +1143,9 @@ void bitfieldGeneric(client *c, int flags) {
 }
 
 void bitfieldCommand(client *c) {
-    bitfieldGeneric(c, BITFIELD_COMMON);
+    bitfieldGeneric(c, BITFIELD_FLAG_NONE);
 }
 
 void bitfieldroCommand(client *c) {
-    bitfieldGeneric(c, BITFIELD_READONLY);
+    bitfieldGeneric(c, BITFIELD_FLAG_READONLY);
 }

From 918086e253407f7f327c564bd650ac065053af0e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 23 Mar 2020 11:47:37 +0100
Subject: [PATCH 0195/1098] Abort transactions after -READONLY error. Fix
 #7014.

---
 src/server.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/server.c b/src/server.c
index f5fb339f9..ddc90b3dd 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3509,6 +3509,7 @@ int processCommand(client *c) {
         !(c->flags & CLIENT_MASTER) &&
         c->cmd->flags & CMD_WRITE)
     {
+        flagTransaction(c);
         addReply(c, shared.roslaveerr);
         return C_OK;
     }

From 61de1c11466030908c22ed18206ae9dba189ef8b Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 23 Mar 2020 12:00:46 +0100
Subject: [PATCH 0196/1098] Fix BITFIELD_RO test.

---
 src/bitops.c            | 2 +-
 tests/unit/bitfield.tcl | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/bitops.c b/src/bitops.c
index d4e82c937..f78e4fd34 100644
--- a/src/bitops.c
+++ b/src/bitops.c
@@ -1007,7 +1007,7 @@ void bitfieldGeneric(client *c, int flags) {
     } else {
         if (flags & BITFIELD_FLAG_READONLY) {
             zfree(ops);
-            addReplyError(c, "BITFIELD_RO only support the GET subcommand");
+            addReplyError(c, "BITFIELD_RO only supports the GET subcommand");
             return;
         }
 
diff --git a/tests/unit/bitfield.tcl b/tests/unit/bitfield.tcl
index 819d8f36d..1f2f6395e 100644
--- a/tests/unit/bitfield.tcl
+++ b/tests/unit/bitfield.tcl
@@ -207,7 +207,7 @@ start_server {tags {"repl"}} {
         set master_port [srv -1 port]
         set slave [srv 0 client]
 
-        test {setup slave} {
+        test {BITFIELD: setup slave} {
             $slave slaveof $master_host $master_port
             wait_for_condition 50 100 {
                 [s 0 master_link_status] eq {up}
@@ -216,7 +216,7 @@ start_server {tags {"repl"}} {
             }
         }
 
-        test {write on master, read on slave} {
+        test {BITFIELD: write on master, read on slave} {
             $master del bits
             assert_equal 0 [$master bitfield bits set u8 0 255]
             assert_equal 255 [$master bitfield bits set u8 0 100]
@@ -224,9 +224,9 @@ start_server {tags {"repl"}} {
             assert_equal 100 [$slave bitfield_ro bits get u8 0]
         }
 
-        test {bitfield_ro with write option} {
+        test {BITFIELD_RO fails when write option is used} {
             catch {$slave bitfield_ro bits set u8 0 100 get u8 0} err
-            assert_match {*ERR bitfield_ro only support get subcommand*} $err
+            assert_match {*ERR BITFIELD_RO only supports the GET subcommand*} $err
         }
     }
 }

From c80d81c80aa4421f48d9e3fd005ca82546d82239 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 23 Mar 2020 16:17:35 +0100
Subject: [PATCH 0197/1098] Improve comments of
 replicationCacheMasterUsingMyself().

---
 src/replication.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/replication.c b/src/replication.c
index 31e14d7fe..49c38f73f 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2725,9 +2725,14 @@ void replicationCacheMaster(client *c) {
  * current offset if no data was lost during the failover. So we use our
  * current replication ID and offset in order to synthesize a cached master. */
 void replicationCacheMasterUsingMyself(void) {
+    /* This will be used to populate the field server.master->reploff
+     * by replicationCreateMasterClient(). We'll later set the created
+     * master as server.cached_master, so the replica will use such
+     * offset for PSYNC. */
+    server.master_initial_offset = server.master_repl_offset;
+
     /* The master client we create can be set to any DBID, because
      * the new master will start its replication stream with SELECT. */
-    server.master_initial_offset = server.master_repl_offset;
     replicationCreateMasterClient(NULL,-1);
 
     /* Use our own ID / offset. */

From ec007559ff703d27916f54ad0a41d154a88d9ac4 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 23 Mar 2020 20:13:52 +0200
Subject: [PATCH 0198/1098] MULTI/EXEC during LUA script timeout are messed up

Redis refusing to run MULTI or EXEC during script timeout may cause partial
transactions to run.

1) if the client sends MULTI+commands+EXEC in pipeline without waiting for
response, but these arrive to the shards partially while there's a busy script,
and partially after it eventually finishes: we'll end up running only part of
the transaction (since multi was ignored, and exec would fail).

2) similar to the above if EXEC arrives during busy script, it'll be ignored and
the client state remains in a transaction.

the 3rd test which i added for a case where MULTI and EXEC are ok, and
only the body arrives during busy script was already handled correctly
since processCommand calls flagTransaction
---
 src/server.c         |  1 +
 tests/unit/multi.tcl | 72 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 73 insertions(+)

diff --git a/src/server.c b/src/server.c
index ddc90b3dd..5a54a3abb 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3553,6 +3553,7 @@ int processCommand(client *c) {
           c->cmd->proc != authCommand &&
           c->cmd->proc != helloCommand &&
           c->cmd->proc != replconfCommand &&
+          c->cmd->proc != multiCommand && c->cmd->proc != execCommand && c->cmd->proc != discardCommand &&
         !(c->cmd->proc == shutdownCommand &&
           c->argc == 2 &&
           tolower(((char*)c->argv[1]->ptr)[0]) == 'n') &&
diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl
index 9fcef71d6..55f18bec8 100644
--- a/tests/unit/multi.tcl
+++ b/tests/unit/multi.tcl
@@ -320,4 +320,76 @@ start_server {tags {"multi"}} {
         $rd close
         r ping
     } {PONG}
+
+    test {MULTI and script timeout} {
+        # check that if MULTI arrives during timeout, it is either refused, or
+        # allowed to pass, and we don't end up executing half of the transaction
+        set rd1 [redis_deferring_client]
+        set rd2 [redis_deferring_client]
+        r config set lua-time-limit 10
+        r set xx 1
+        $rd1 eval {while true do end} 0
+        after 200
+        catch { $rd2 multi; $rd2 read } e
+        catch { $rd2 incr xx; $rd2 read } e
+        r script kill
+        after 200 ; # Give some time to Lua to call the hook again...
+        catch { $rd2 incr xx; $rd2 read } e
+        catch { $rd2 exec; $rd2 read } e
+        set xx [r get xx]
+        # make sure that either the whole transcation passed or none of it (we actually expect none)
+        assert { $xx == 1 || $xx == 3}
+        # check that the connection is no longer in multi state
+        $rd2 ping asdf
+        set pong [$rd2 read]
+        assert_equal $pong "asdf"
+    }
+
+    test {EXEC and script timeout} {
+        # check that if EXEC arrives during timeout, we don't end up executing
+        # half of the transaction, and also that we exit the multi state
+        set rd1 [redis_deferring_client]
+        set rd2 [redis_deferring_client]
+        r config set lua-time-limit 10
+        r set xx 1
+        catch { $rd2 multi; $rd2 read } e
+        catch { $rd2 incr xx; $rd2 read } e
+        $rd1 eval {while true do end} 0
+        after 200
+        catch { $rd2 incr xx; $rd2 read } e
+        catch { $rd2 exec; $rd2 read } e
+        r script kill
+        after 200 ; # Give some time to Lua to call the hook again...
+        set xx [r get xx]
+        # make sure that either the whole transcation passed or none of it (we actually expect none)
+        assert { $xx == 1 || $xx == 3}
+        # check that the connection is no longer in multi state
+        $rd2 ping asdf
+        set pong [$rd2 read]
+        assert_equal $pong "asdf"
+    }
+
+    test {MULTI-EXEC body and script timeout} {
+        # check that we don't run an imcomplete transaction due to some commands
+        # arriving during busy script
+        set rd1 [redis_deferring_client]
+        set rd2 [redis_deferring_client]
+        r config set lua-time-limit 10
+        r set xx 1
+        catch { $rd2 multi; $rd2 read } e
+        catch { $rd2 incr xx; $rd2 read } e
+        $rd1 eval {while true do end} 0
+        after 200
+        catch { $rd2 incr xx; $rd2 read } e
+        r script kill
+        after 200 ; # Give some time to Lua to call the hook again...
+        catch { $rd2 exec; $rd2 read } e
+        set xx [r get xx]
+        # make sure that either the whole transcation passed or none of it (we actually expect none)
+        assert { $xx == 1 || $xx == 3}
+        # check that the connection is no longer in multi state
+        $rd2 ping asdf
+        set pong [$rd2 read]
+        assert_equal $pong "asdf"
+    }
 }

From f15042dbf0f0981c827126cf70bf335e755253f0 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 25 Mar 2020 12:46:59 +0100
Subject: [PATCH 0199/1098] Explain why we allow transactions in -BUSY state.

Related to #7022.
---
 src/server.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/server.c b/src/server.c
index 5a54a3abb..65a01db57 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3548,12 +3548,19 @@ int processCommand(client *c) {
         return C_OK;
     }
 
-    /* Lua script too slow? Only allow a limited number of commands. */
+    /* Lua script too slow? Only allow a limited number of commands.
+     * Note that we need to allow the transactions commands, otherwise clients
+     * sending a transaction with pipelining without error checking, may have
+     * the MULTI plus a few initial commands refused, then the timeout
+     * condition resolves, and the bottom-half of the transaction gets
+     * executed, see Github PR #7022. */
     if (server.lua_timedout &&
           c->cmd->proc != authCommand &&
           c->cmd->proc != helloCommand &&
           c->cmd->proc != replconfCommand &&
-          c->cmd->proc != multiCommand && c->cmd->proc != execCommand && c->cmd->proc != discardCommand &&
+          c->cmd->proc != multiCommand &&
+          c->cmd->proc != execCommand &&
+          c->cmd->proc != discardCommand &&
         !(c->cmd->proc == shutdownCommand &&
           c->argc == 2 &&
           tolower(((char*)c->argv[1]->ptr)[0]) == 'n') &&

From 57fa355e56540a6fab61a5b8ecdda96225815632 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 24 Mar 2020 11:02:40 +0100
Subject: [PATCH 0200/1098] PSYNC2: meaningful offset implemented.

A very commonly signaled operational problem with Redis master-replicas
sets is that, once the master becomes unavailable for some reason,
especially because of network problems, many times it wont be able to
perform a partial resynchronization with the new master, once it rejoins
the partition, for the following reason:

1. The master becomes isolated, however it keeps sending PINGs to the
replicas. Such PINGs will never be received since the link connection is
actually already severed.
2. On the other side, one of the replicas will turn into the new master,
setting its secondary replication ID offset to the one of the last
command received from the old master: this offset will not include the
PINGs sent by the master once the link was already disconnected.
3. When the master rejoins the partion and is turned into a replica, its
offset will be too advanced because of the PINGs, so a PSYNC will fail,
and a full synchronization will be required.

Related to issue #7002 and other discussion we had in the past around
this problem.
---
 src/replication.c | 36 +++++++++++++++++++++++++++++++++++-
 src/server.c      |  4 ++++
 src/server.h      |  1 +
 3 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/src/replication.c b/src/replication.c
index 49c38f73f..e62afb2fb 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -162,6 +162,7 @@ void feedReplicationBacklog(void *ptr, size_t len) {
     unsigned char *p = ptr;
 
     server.master_repl_offset += len;
+    server.master_repl_meaningful_offset = server.master_repl_offset;
 
     /* This is a circular buffer, so write as much data we can at every
      * iteration and rewind the "idx" index if we reach the limit. */
@@ -1768,6 +1769,7 @@ void readSyncBulkPayload(connection *conn) {
      * we are starting a new history. */
     memcpy(server.replid,server.master->replid,sizeof(server.replid));
     server.master_repl_offset = server.master->reploff;
+    server.master_repl_meaningful_offset = server.master->reploff;
     clearReplicationId2();
 
     /* Let's create the replication backlog if needed. Slaves need to
@@ -2725,12 +2727,37 @@ void replicationCacheMaster(client *c) {
  * current offset if no data was lost during the failover. So we use our
  * current replication ID and offset in order to synthesize a cached master. */
 void replicationCacheMasterUsingMyself(void) {
+    serverLog(LL_NOTICE,
+        "Before turning into a replica, using my own master parameters "
+        "to synthesize a cached master: I may be able to synchronize with "
+        "the new master with just a partial transfer.");
+
     /* This will be used to populate the field server.master->reploff
      * by replicationCreateMasterClient(). We'll later set the created
      * master as server.cached_master, so the replica will use such
      * offset for PSYNC. */
     server.master_initial_offset = server.master_repl_offset;
 
+    /* However if the "meaningful" offset, that is the offset without
+     * the final PINGs in the stream, is different, use this instead:
+     * often when the master is no longer reachable, replicas will never
+     * receive the PINGs, however the master will end with an incremented
+     * offset because of the PINGs and will not be able to incrementally
+     * PSYNC with the new master. */
+    if (server.master_repl_offset > server.master_repl_meaningful_offset) {
+        long long delta = server.master_repl_offset -
+                          server.master_repl_meaningful_offset;
+        serverLog(LL_NOTICE,
+            "Using the meaningful offset %lld instead of %lld to exclude "
+            "the final PINGs (%lld bytes difference)",
+                server.master_repl_meaningful_offset,
+                server.master_repl_offset,
+                delta);
+        server.master_initial_offset = server.master_repl_meaningful_offset;
+        server.repl_backlog_histlen -= delta;
+        if (server.repl_backlog_histlen < 0) server.repl_backlog_histlen = 0;
+    }
+
     /* The master client we create can be set to any DBID, because
      * the new master will start its replication stream with SELECT. */
     replicationCreateMasterClient(NULL,-1);
@@ -2742,7 +2769,6 @@ void replicationCacheMasterUsingMyself(void) {
     unlinkClient(server.master);
     server.cached_master = server.master;
     server.master = NULL;
-    serverLog(LL_NOTICE,"Before turning into a replica, using my master parameters to synthesize a cached master: I may be able to synchronize with the new master with just a partial transfer.");
 }
 
 /* Free a cached master, called when there are no longer the conditions for
@@ -3122,10 +3148,18 @@ void replicationCron(void) {
             clientsArePaused();
 
         if (!manual_failover_in_progress) {
+            long long before_ping = server.master_repl_meaningful_offset;
             ping_argv[0] = createStringObject("PING",4);
             replicationFeedSlaves(server.slaves, server.slaveseldb,
                 ping_argv, 1);
             decrRefCount(ping_argv[0]);
+            /* The server.master_repl_meaningful_offset variable represents
+             * the offset of the replication stream without the pending PINGs.
+             * This is useful to set the right replication offset for PSYNC
+             * when the master is turned into a replica. Otherwise pending
+             * PINGs may not allow it to perform an incremental sync with the
+             * new master. */
+            server.master_repl_meaningful_offset = before_ping;
         }
     }
 
diff --git a/src/server.c b/src/server.c
index 65a01db57..84439461e 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2355,6 +2355,7 @@ void initServerConfig(void) {
     server.repl_syncio_timeout = CONFIG_REPL_SYNCIO_TIMEOUT;
     server.repl_down_since = 0; /* Never connected, repl is down since EVER. */
     server.master_repl_offset = 0;
+    server.master_repl_meaningful_offset = 0;
 
     /* Replication partial resync backlog */
     server.repl_backlog = NULL;
@@ -4398,6 +4399,7 @@ sds genRedisInfoString(const char *section) {
             "master_replid:%s\r\n"
             "master_replid2:%s\r\n"
             "master_repl_offset:%lld\r\n"
+            "master_repl_meaningful_offset:%lld\r\n"
             "second_repl_offset:%lld\r\n"
             "repl_backlog_active:%d\r\n"
             "repl_backlog_size:%lld\r\n"
@@ -4406,6 +4408,7 @@ sds genRedisInfoString(const char *section) {
             server.replid,
             server.replid2,
             server.master_repl_offset,
+            server.master_repl_meaningful_offset,
             server.second_replid_offset,
             server.repl_backlog != NULL,
             server.repl_backlog_size,
@@ -4783,6 +4786,7 @@ void loadDataFromDisk(void) {
             {
                 memcpy(server.replid,rsi.repl_id,sizeof(server.replid));
                 server.master_repl_offset = rsi.repl_offset;
+                server.master_repl_meaningful_offset = rsi.repl_offset;
                 /* If we are a slave, create a cached master from this
                  * information, in order to allow partial resynchronizations
                  * with masters. */
diff --git a/src/server.h b/src/server.h
index ed4707d66..818fbc3b3 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1241,6 +1241,7 @@ struct redisServer {
     char replid[CONFIG_RUN_ID_SIZE+1];  /* My current replication ID. */
     char replid2[CONFIG_RUN_ID_SIZE+1]; /* replid inherited from master*/
     long long master_repl_offset;   /* My current replication offset */
+    long long master_repl_meaningful_offset; /* Offset minus latest PINGs. */
     long long second_replid_offset; /* Accept offsets up to this for replid2. */
     int slaveseldb;                 /* Last SELECTed DB in replication output */
     int repl_ping_slave_period;     /* Master pings the slave every N seconds */

From c4d7f30e250b4b291e6d5893d9a3323df4a57c7c Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 25 Mar 2020 15:43:34 +0100
Subject: [PATCH 0201/1098] PSYNC2: meaningful offset test.

---
 tests/integration/psync2-pingoff.tcl | 61 ++++++++++++++++++++++++++++
 tests/test_helper.tcl                |  1 +
 2 files changed, 62 insertions(+)
 create mode 100644 tests/integration/psync2-pingoff.tcl

diff --git a/tests/integration/psync2-pingoff.tcl b/tests/integration/psync2-pingoff.tcl
new file mode 100644
index 000000000..1cea290e7
--- /dev/null
+++ b/tests/integration/psync2-pingoff.tcl
@@ -0,0 +1,61 @@
+# Test the meaningful offset implementation to make sure masters
+# are able to PSYNC with replicas even if the replication stream
+# has pending PINGs at the end.
+
+start_server {tags {"psync2"}} {
+start_server {} {
+    # Config
+    set debug_msg 0                 ; # Enable additional debug messages
+
+    for {set j 0} {$j < 2} {incr j} {
+        set R($j) [srv [expr 0-$j] client]
+        set R_host($j) [srv [expr 0-$j] host]
+        set R_port($j) [srv [expr 0-$j] port]
+        $R($j) CONFIG SET repl-ping-replica-period 1
+        if {$debug_msg} {puts "Log file: [srv [expr 0-$j] stdout]"}
+    }
+
+    # Setup replication
+    test "PSYNC2 meaningful offset: setup" {
+        $R(1) replicaof $R_host(0) $R_port(0)
+        $R(0) set foo bar
+        wait_for_condition 50 1000 {
+            [$R(0) dbsize] == 1 && [$R(1) dbsize] == 1
+        } else {
+            fail "Replicas not replicating from master"
+        }
+    }
+
+    test "PSYNC2 meaningful offset: write and wait replication" {
+        $R(0) INCR counter
+        $R(0) INCR counter
+        $R(0) INCR counter
+        wait_for_condition 50 1000 {
+            [$R(0) GET counter] eq [$R(1) GET counter]
+        } else {
+            fail "Master and replica don't agree about counter"
+        }
+    }
+
+    # In this test we'll make sure the replica will get stuck, but with
+    # an active connection: this way the master will continue to send PINGs
+    # every second (we modified the PING period earlier)
+    test "PSYNC2 meaningful offset: pause replica and promote it" {
+        $R(1) MULTI
+        $R(1) DEBUG SLEEP 5
+        $R(1) SLAVEOF NO ONE
+        $R(1) EXEC
+        $R(1) ping ; # Wait for it to return back available
+    }
+
+    test "Make the old master a replica of the new one and check conditions" {
+        set sync_partial [status $R(1) sync_partial_ok]
+        assert {$sync_partial == 0}
+        $R(0) REPLICAOF $R_host(1) $R_port(1)
+        wait_for_condition 50 1000 {
+            [status $R(1) sync_partial_ok] == 1
+        } else {
+            fail "The new master was not able to partial sync"
+        }
+    }
+}}
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 4dbead193..5cb43104b 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -47,6 +47,7 @@ set ::all_tests {
     integration/logging
     integration/psync2
     integration/psync2-reg
+    integration/psync2-pingoff
     unit/pubsub
     unit/slowlog
     unit/scripting

From 3b29556a0cd9db0daa7828fe81df41fbe95e04dc Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 4 Dec 2019 17:29:29 +0200
Subject: [PATCH 0202/1098] AOFRW on an empty stream created with MKSTREAM
 loads badkly

the AOF will be loaded successfully, but the stream will be missing,
i.e inconsistencies with the original db.

this was because XADD with id of 0-0 would error.

add a test to reproduce.
---
 src/aof.c                          |  3 ++-
 tests/unit/type/stream-cgroups.tcl | 13 +++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/aof.c b/src/aof.c
index 8ab9349f0..6bb239252 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1212,12 +1212,13 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) {
         /* Use the XADD MAXLEN 0 trick to generate an empty stream if
          * the key we are serializing is an empty string, which is possible
          * for the Stream type. */
+        id.ms = 0; id.seq = 1; 
         if (rioWriteBulkCount(r,'*',7) == 0) return 0;
         if (rioWriteBulkString(r,"XADD",4) == 0) return 0;
         if (rioWriteBulkObject(r,key) == 0) return 0;
         if (rioWriteBulkString(r,"MAXLEN",6) == 0) return 0;
         if (rioWriteBulkString(r,"0",1) == 0) return 0;
-        if (rioWriteBulkStreamID(r,&s->last_id) == 0) return 0;
+        if (rioWriteBulkStreamID(r,&id) == 0) return 0;
         if (rioWriteBulkString(r,"x",1) == 0) return 0;
         if (rioWriteBulkString(r,"y",1) == 0) return 0;
     }
diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl
index 072ed14d6..6b9a4a9cd 100644
--- a/tests/unit/type/stream-cgroups.tcl
+++ b/tests/unit/type/stream-cgroups.tcl
@@ -311,4 +311,17 @@ start_server {
             }
         }
     }
+
+    start_server {tags {"stream"} overrides {appendonly yes aof-use-rdb-preamble no}} {
+        test {Empty stream with no lastid can be rewrite into AOF correctly} {
+            r XGROUP CREATE mystream group-name $ MKSTREAM
+            assert {[dict get [r xinfo stream mystream] length] == 0}
+            set grpinfo [r xinfo groups mystream]
+            r bgrewriteaof
+            waitForBgrewriteaof r
+            r debug loadaof
+            assert {[dict get [r xinfo stream mystream] length] == 0}
+            assert {[r xinfo groups mystream] == $grpinfo}
+        }
+    }
 }

From 1547d72cf3ec4ef5b67cbba7cf50726983be31e4 Mon Sep 17 00:00:00 2001
From: Valentino Geron <valentino@redislabs.com>
Date: Wed, 25 Mar 2020 21:54:14 +0200
Subject: [PATCH 0203/1098] XACK should be executed in a "all or nothing"
 fashion.

First, we must parse the IDs, so that we abort ASAP.
The return value of this command cannot be an error if
the client successfully acknowledged some messages,
so it should be executed in a "all or nothing" fashion.
---
 src/t_stream.c                     | 12 +++++++++++-
 tests/unit/type/stream-cgroups.tcl | 12 ++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 557d1d642..2bf6a26a7 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1921,11 +1921,21 @@ void xackCommand(client *c) {
         return;
     }
 
+    /* Start parsing the IDs, so that we abort ASAP if there is a syntax
+     * error: the return value of this command cannot be an error in case
+     * the client successfully acknowledged some messages, so it should be
+     * executed in a "all or nothing" fashion. */
+    for (int j = 3; j < c->argc; j++) {
+        streamID id;
+        if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return;
+    }
+
     int acknowledged = 0;
     for (int j = 3; j < c->argc; j++) {
         streamID id;
         unsigned char buf[sizeof(streamID)];
-        if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return;
+        if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK)
+            serverPanic("StreamID invalid after check. Should not be possible.");
         streamEncodeID(buf,&id);
 
         /* Lookup the ID in the group PEL: it will have a reference to the
diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl
index 072ed14d6..2fc0c51cd 100644
--- a/tests/unit/type/stream-cgroups.tcl
+++ b/tests/unit/type/stream-cgroups.tcl
@@ -93,6 +93,18 @@ start_server {
         assert {[r XACK mystream mygroup $id1 $id2] eq 1}
     }
 
+    test {XACK should fail if got at least one invalid ID} {
+        r del mystream
+        r xgroup create s g $ MKSTREAM
+        r xadd s * f1 v1
+        set c [llength [lindex [r xreadgroup group g c streams s >] 0 1]]
+        assert {$c == 1}
+        set pending [r xpending s g - + 10 c]
+        set id1 [lindex $pending 0 0]
+        assert_error "*Invalid stream ID specified*" {r xack s g $id1 invalid-id}
+        assert {[r xack s g $id1] eq 1}
+    }
+
     test {PEL NACK reassignment after XGROUP SETID event} {
         r del events
         r xadd events * f1 v1

From 9a1843ef2df09edcc28f4f1b2ace93a39d35cc59 Mon Sep 17 00:00:00 2001
From: Valentino Geron <valentino@redislabs.com>
Date: Thu, 26 Mar 2020 11:49:21 +0200
Subject: [PATCH 0204/1098] XREAD and XREADGROUP should not be allowed from
 scripts when BLOCK option is being used

---
 src/server.c             |  4 ++--
 src/t_stream.c           |  5 +++++
 tests/unit/scripting.tcl | 11 +++++++++++
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/server.c b/src/server.c
index 84439461e..4fc7e0103 100644
--- a/src/server.c
+++ b/src/server.c
@@ -947,11 +947,11 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,1,1,1,0,0,0},
 
     {"xread",xreadCommand,-4,
-     "read-only no-script @stream @blocking",
+     "read-only @stream @blocking",
      0,xreadGetKeys,1,1,1,0,0,0},
 
     {"xreadgroup",xreadCommand,-7,
-     "write no-script @stream @blocking",
+     "write @stream @blocking",
      0,xreadGetKeys,1,1,1,0,0,0},
 
     {"xgroup",xgroupCommand,-2,
diff --git a/src/t_stream.c b/src/t_stream.c
index 557d1d642..157f1eaeb 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1373,6 +1373,11 @@ void xreadCommand(client *c) {
         int moreargs = c->argc-i-1;
         char *o = c->argv[i]->ptr;
         if (!strcasecmp(o,"BLOCK") && moreargs) {
+            if (c->flags & CLIENT_LUA) {
+                /* There is no sense to use BLOCK option within LUA */
+                addReplyErrorFormat(c, "%s command is not allowed with BLOCK option from scripts", (char *)c->argv[0]->ptr);
+                return;
+            }
             i++;
             if (getTimeoutFromObjectOrReply(c,c->argv[i],&timeout,
                 UNIT_MILLISECONDS) != C_OK) return;
diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl
index fb36d0b80..8b364b287 100644
--- a/tests/unit/scripting.tcl
+++ b/tests/unit/scripting.tcl
@@ -146,6 +146,17 @@ start_server {tags {"scripting"}} {
         set e
     } {*not allowed*}
 
+    test {EVAL - Scripts can't run XREAD and XREADGROUP with BLOCK option} {
+        r del s
+        r xgroup create s g $ MKSTREAM
+        set res [r eval {return redis.pcall('xread','STREAMS','s','$')} 1 s]
+        assert {$res eq {}}
+        assert_error "*xread command is not allowed with BLOCK option from scripts" {r eval {return redis.pcall('xread','BLOCK',0,'STREAMS','s','$')} 1 s}
+        set res [r eval {return redis.pcall('xreadgroup','group','g','c','STREAMS','s','>')} 1 s]
+        assert {$res eq {}}
+        assert_error "*xreadgroup command is not allowed with BLOCK option from scripts" {r eval {return redis.pcall('xreadgroup','group','g','c','BLOCK',0,'STREAMS','s','>')} 1 s}
+    }
+
     test {EVAL - Scripts can't run certain commands} {
         set e {}
         r debug lua-always-replicate-commands 0

From 97f1c808ce522113935bad244a9505a87ba27f9e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 27 Mar 2020 16:19:56 +0100
Subject: [PATCH 0205/1098] PSYNC2: fix backlog_idx when adjusting for
 meaningful offset

See #7002.
---
 src/replication.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/replication.c b/src/replication.c
index e62afb2fb..20f9a2129 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2755,6 +2755,9 @@ void replicationCacheMasterUsingMyself(void) {
                 delta);
         server.master_initial_offset = server.master_repl_meaningful_offset;
         server.repl_backlog_histlen -= delta;
+        server.repl_backlog_idx =
+            (server.repl_backlog_idx + (server.repl_backlog_size - delta)) %
+            server.repl_backlog_size;
         if (server.repl_backlog_histlen < 0) server.repl_backlog_histlen = 0;
     }
 

From 8d116cc8a2b8168bdf0dd649ca933ba1b3404852 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 26 Mar 2020 11:33:18 +0100
Subject: [PATCH 0206/1098] Precise timeouts: refactor unblocking on timeout.

---
 src/server.c | 41 +++++++++++++++++++++++++++++------------
 src/server.h |  3 +++
 2 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/src/server.c b/src/server.c
index 84439461e..db56f6a0d 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1502,6 +1502,23 @@ long long getInstantaneousMetric(int metric) {
     return sum / STATS_METRIC_SAMPLES;
 }
 
+/* Check if this blocked client timedout (does nothing if the client is
+ * not blocked right now). If so send a reply, unblock it, and return 1.
+ * Otherwise 0 is returned and no operation is performed. */
+int checkBlockedClientTimeout(client *c, mstime_t now) {
+    if (c->flags & CLIENT_BLOCKED &&
+        c->bpop.timeout != 0
+        && c->bpop.timeout < now)
+    {
+        /* Handle blocking operation specific timeout. */
+        replyToBlockedClientTimedOut(c);
+        unblockClient(c);
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
 /* Check for timeouts. Returns non-zero if the client was terminated.
  * The function gets the current time in milliseconds as argument since
  * it gets called multiple times in a loop, so calling gettimeofday() for
@@ -1510,10 +1527,11 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) {
     time_t now = now_ms/1000;
 
     if (server.maxidletime &&
-        !(c->flags & CLIENT_SLAVE) &&    /* no timeout for slaves and monitors */
-        !(c->flags & CLIENT_MASTER) &&   /* no timeout for masters */
-        !(c->flags & CLIENT_BLOCKED) &&  /* no timeout for BLPOP */
-        !(c->flags & CLIENT_PUBSUB) &&   /* no timeout for Pub/Sub clients */
+        /* This handles the idle clients connection timeout if set. */
+        !(c->flags & CLIENT_SLAVE) &&   /* No timeout for slaves and monitors */
+        !(c->flags & CLIENT_MASTER) &&  /* No timeout for masters */
+        !(c->flags & CLIENT_BLOCKED) && /* No timeout for BLPOP */
+        !(c->flags & CLIENT_PUBSUB) &&  /* No timeout for Pub/Sub clients */
         (now - c->lastinteraction > server.maxidletime))
     {
         serverLog(LL_VERBOSE,"Closing idle client");
@@ -1522,15 +1540,14 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) {
     } else if (c->flags & CLIENT_BLOCKED) {
         /* Blocked OPS timeout is handled with milliseconds resolution.
          * However note that the actual resolution is limited by
-         * server.hz. */
+         * server.hz. So for short timeouts (less than SERVER_SHORT_TIMEOUT
+         * milliseconds) we populate a Radix tree and handle such timeouts
+         * in clientsHandleShortTimeout(). */
+        if (checkBlockedClientTimeout(c,now_ms)) return 0;
 
-        if (c->bpop.timeout != 0 && c->bpop.timeout < now_ms) {
-            /* Handle blocking operation specific timeout. */
-            replyToBlockedClientTimedOut(c);
-            unblockClient(c);
-        } else if (server.cluster_enabled) {
-            /* Cluster: handle unblock & redirect of clients blocked
-             * into keys no longer served by this server. */
+        /* Cluster: handle unblock & redirect of clients blocked
+         * into keys no longer served by this server. */
+        if (server.cluster_enabled) {
             if (clusterRedirectBlockedClientIfNeeded(c))
                 unblockClient(c);
         }
diff --git a/src/server.h b/src/server.h
index 818fbc3b3..93552b36e 100644
--- a/src/server.h
+++ b/src/server.h
@@ -277,6 +277,9 @@ typedef long long ustime_t; /* microsecond time type. */
                                     buffer configuration. Just the first
                                     three: normal, slave, pubsub. */
 
+/* Other client related defines. */
+#define CLIENT_SHORT_TIMEOUT 2000  /* See clientsHandleShortTimeout(). */
+
 /* Slave replication state. Used in server.repl_state for slaves to remember
  * what to do next. */
 #define REPL_STATE_NONE 0 /* No active replication */

From 324a8c91d05023420c7f15d9ad84ba41ad9eefe3 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 26 Mar 2020 13:28:39 +0100
Subject: [PATCH 0207/1098] Precise timeouts: working initial implementation.

---
 src/blocked.c |   1 +
 src/server.c  | 135 +++++++++++++++++++++++++++++++++++++++-----------
 src/server.h  |   2 +
 3 files changed, 110 insertions(+), 28 deletions(-)

diff --git a/src/blocked.c b/src/blocked.c
index 06aa5850e..c470cba00 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -619,6 +619,7 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo
         listAddNodeTail(l,c);
     }
     blockClient(c,btype);
+    addClientToShortTimeoutTable(c);
 }
 
 /* Unblock a client that's waiting in a blocking operation such as BLPOP.
diff --git a/src/server.c b/src/server.c
index db56f6a0d..bd63c4b4e 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1473,34 +1473,7 @@ int allPersistenceDisabled(void) {
     return server.saveparamslen == 0 && server.aof_state == AOF_OFF;
 }
 
-/* ======================= Cron: called every 100 ms ======================== */
-
-/* Add a sample to the operations per second array of samples. */
-void trackInstantaneousMetric(int metric, long long current_reading) {
-    long long t = mstime() - server.inst_metric[metric].last_sample_time;
-    long long ops = current_reading -
-                    server.inst_metric[metric].last_sample_count;
-    long long ops_sec;
-
-    ops_sec = t > 0 ? (ops*1000/t) : 0;
-
-    server.inst_metric[metric].samples[server.inst_metric[metric].idx] =
-        ops_sec;
-    server.inst_metric[metric].idx++;
-    server.inst_metric[metric].idx %= STATS_METRIC_SAMPLES;
-    server.inst_metric[metric].last_sample_time = mstime();
-    server.inst_metric[metric].last_sample_count = current_reading;
-}
-
-/* Return the mean of all the samples. */
-long long getInstantaneousMetric(int metric) {
-    int j;
-    long long sum = 0;
-
-    for (j = 0; j < STATS_METRIC_SAMPLES; j++)
-        sum += server.inst_metric[metric].samples[j];
-    return sum / STATS_METRIC_SAMPLES;
-}
+/* ========================== Clients timeouts ============================= */
 
 /* Check if this blocked client timedout (does nothing if the client is
  * not blocked right now). If so send a reply, unblock it, and return 1.
@@ -1555,6 +1528,107 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) {
     return 0;
 }
 
+/* For shor timeouts, less than < CLIENT_SHORT_TIMEOUT milliseconds, we
+ * populate a radix tree of 128 bit keys composed as such:
+ *
+ *  [8 byte big endian expire time]+[8 byte client ID]
+ *
+ * We don't do any cleanup in the Radix tree: when we run the clients that
+ * reached the timeout already, if they are no longer existing or no longer
+ * blocked with such timeout, we just go forward.
+ *
+ * Every time a client blocks with a short timeout, we add the client in
+ * the tree. In beforeSleep() we call clientsHandleShortTimeout() to run
+ * the tree and unblock the clients.
+ *
+ * Design hint: why we block only clients with short timeouts? For frugality:
+ * Clients blocking for 30 seconds usually don't need to be unblocked
+ * precisely, and anyway for the nature of Redis to *guarantee* unblock time
+ * precision is hard, so we can avoid putting a large number of clients in
+ * the radix tree without a good reason. This idea also has a role in memory
+ * usage as well given that we don't do cleanup, the shorter a client timeout,
+ * the less time it will stay in the radix tree. */
+
+#define CLIENT_ST_KEYLEN 16    /* 8 bytes mstime + 8 bytes client ID. */
+
+/* Given client ID and timeout, write the resulting radix tree key in buf. */
+void encodeTimeoutKey(unsigned char *buf, uint64_t timeout, uint64_t id) {
+    timeout = htonu64(timeout);
+    memcpy(buf,&timeout,sizeof(timeout));
+    memcpy(buf+8,&id,sizeof(id));
+}
+
+/* Given a key encoded with encodeTimeoutKey(), resolve the fields and write
+ * the timeout into *toptr and the client ID into *idptr. */
+void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) {
+    memcpy(toptr,buf,sizeof(*toptr));
+    *toptr = ntohu64(*toptr);
+    memcpy(idptr,buf+8,sizeof(*idptr));
+}
+
+/* Add the specified client id / timeout as a key in the radix tree we use
+ * to handle short timeouts. The client is not added to the list if its
+ * timeout is longer than CLIENT_SHORT_TIMEOUT milliseconds. */
+void addClientToShortTimeoutTable(client *c) {
+    if (c->bpop.timeout == 0 ||
+        c->bpop.timeout - mstime() > CLIENT_SHORT_TIMEOUT)
+    {
+        return;
+    }
+    uint64_t timeout = c->bpop.timeout;
+    uint64_t id = c->id;
+    unsigned char buf[CLIENT_ST_KEYLEN];
+    encodeTimeoutKey(buf,timeout,id);
+    raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL);
+}
+
+/* This function is called in beforeSleep() in order to unblock ASAP clients
+ * that are waiting in blocking operations with a short timeout set. */
+void clientsHandleShortTimeout(void) {
+    uint64_t now = mstime();
+    raxIterator ri;
+    raxStart(&ri,server.clients_timeout_table);
+
+    while(raxNext(&ri)) {
+        uint64_t id, timeout;
+        decodeTimeoutKey(ri.key,&timeout,&id);
+        if (timeout >= now) break; /* All the timeouts are in the future. */
+        client *c = lookupClientByID(id);
+        if (c) checkBlockedClientTimeout(c,now);
+        raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL);
+        raxSeek(&ri,"^",NULL,0);
+    }
+}
+
+/* ======================= Cron: called every 100 ms ======================== */
+
+/* Add a sample to the operations per second array of samples. */
+void trackInstantaneousMetric(int metric, long long current_reading) {
+    long long t = mstime() - server.inst_metric[metric].last_sample_time;
+    long long ops = current_reading -
+                    server.inst_metric[metric].last_sample_count;
+    long long ops_sec;
+
+    ops_sec = t > 0 ? (ops*1000/t) : 0;
+
+    server.inst_metric[metric].samples[server.inst_metric[metric].idx] =
+        ops_sec;
+    server.inst_metric[metric].idx++;
+    server.inst_metric[metric].idx %= STATS_METRIC_SAMPLES;
+    server.inst_metric[metric].last_sample_time = mstime();
+    server.inst_metric[metric].last_sample_count = current_reading;
+}
+
+/* Return the mean of all the samples. */
+long long getInstantaneousMetric(int metric) {
+    int j;
+    long long sum = 0;
+
+    for (j = 0; j < STATS_METRIC_SAMPLES; j++)
+        sum += server.inst_metric[metric].samples[j];
+    return sum / STATS_METRIC_SAMPLES;
+}
+
 /* The client query buffer is an sds.c string that can end with a lot of
  * free space not used, this function reclaims space if needed.
  *
@@ -2109,11 +2183,15 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
 void beforeSleep(struct aeEventLoop *eventLoop) {
     UNUSED(eventLoop);
 
+    /* Handle precise timeouts of blocked clients. */
+    clientsHandleShortTimeout();
+
     /* We should handle pending reads clients ASAP after event loop. */
     handleClientsWithPendingReadsUsingThreads();
 
     /* Handle TLS pending data. (must be done before flushAppendOnlyFile) */
     tlsProcessPendingData();
+
     /* If tls still has pending unread data don't sleep at all. */
     aeSetDontWait(server.el, tlsHasPendingData());
 
@@ -2738,6 +2816,7 @@ void initServer(void) {
     server.monitors = listCreate();
     server.clients_pending_write = listCreate();
     server.clients_pending_read = listCreate();
+    server.clients_timeout_table = raxNew();
     server.slaveseldb = -1; /* Force to emit the first SELECT command. */
     server.unblocked_clients = listCreate();
     server.ready_keys = listCreate();
diff --git a/src/server.h b/src/server.h
index 93552b36e..6f417b730 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1070,6 +1070,7 @@ struct redisServer {
     list *clients_pending_read;  /* Client has pending read socket buffers. */
     list *slaves, *monitors;    /* List of slaves and MONITORs */
     client *current_client;     /* Current client executing the command. */
+    rax *clients_timeout_table; /* Radix tree for clients with short timeout. */
     long fixed_time_expire;     /* If > 0, expire keys against server.mstime. */
     rax *clients_index;         /* Active clients dictionary by client ID. */
     int clients_paused;         /* True if clients are currently paused */
@@ -2139,6 +2140,7 @@ void disconnectAllBlockedClients(void);
 void handleClientsBlockedOnKeys(void);
 void signalKeyAsReady(redisDb *db, robj *key);
 void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids);
+void addClientToShortTimeoutTable(client *c);
 
 /* expire.c -- Handling of expired keys */
 void activeExpireCycle(int type);

From 8d11e0df7aeecd4f1bcdf1fbaa5ef19f44a4a18f Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 26 Mar 2020 14:37:00 +0100
Subject: [PATCH 0208/1098] Precise timeouts: fix bugs in initial
 implementation.

---
 src/blocked.c | 2 +-
 src/server.c  | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/blocked.c b/src/blocked.c
index c470cba00..84a5287d4 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -111,6 +111,7 @@ void blockClient(client *c, int btype) {
     c->btype = btype;
     server.blocked_clients++;
     server.blocked_clients_by_type[btype]++;
+    addClientToShortTimeoutTable(c);
 }
 
 /* This function is called in the beforeSleep() function of the event loop
@@ -619,7 +620,6 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo
         listAddNodeTail(l,c);
     }
     blockClient(c,btype);
-    addClientToShortTimeoutTable(c);
 }
 
 /* Unblock a client that's waiting in a blocking operation such as BLPOP.
diff --git a/src/server.c b/src/server.c
index bd63c4b4e..25af4e161 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1588,6 +1588,7 @@ void clientsHandleShortTimeout(void) {
     uint64_t now = mstime();
     raxIterator ri;
     raxStart(&ri,server.clients_timeout_table);
+    raxSeek(&ri,"^",NULL,0);
 
     while(raxNext(&ri)) {
         uint64_t id, timeout;
@@ -1745,6 +1746,9 @@ void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) {
  */
 #define CLIENTS_CRON_MIN_ITERATIONS 5
 void clientsCron(void) {
+    /* Unblock short timeout clients ASAP. */
+    clientsHandleShortTimeout();
+
     /* Try to process at least numclients/server.hz of clients
      * per call. Since normally (if there are no big latency events) this
      * function is called server.hz times per second, in the average case we

From 077f9654264d601a87b801f219e27b6097e9c861 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 26 Mar 2020 15:52:16 +0100
Subject: [PATCH 0209/1098] Precise timeouts: fast exit for
 clientsHandleShortTimeout().

---
 src/server.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/server.c b/src/server.c
index 25af4e161..7240794f1 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1585,6 +1585,7 @@ void addClientToShortTimeoutTable(client *c) {
 /* This function is called in beforeSleep() in order to unblock ASAP clients
  * that are waiting in blocking operations with a short timeout set. */
 void clientsHandleShortTimeout(void) {
+    if (raxSize(server.clients_timeout_table) == 0) return;
     uint64_t now = mstime();
     raxIterator ri;
     raxStart(&ri,server.clients_timeout_table);

From aa9d92d94a7f18f768e25bdbb94179b32e7b9e13 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 26 Mar 2020 16:02:26 +0100
Subject: [PATCH 0210/1098] Precise timeouts: use only radix tree for timeouts.

---
 src/blocked.c |  2 +-
 src/server.c  | 46 +++++++++++++---------------------------------
 src/server.h  |  5 +----
 3 files changed, 15 insertions(+), 38 deletions(-)

diff --git a/src/blocked.c b/src/blocked.c
index 84a5287d4..443daec7f 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -111,7 +111,7 @@ void blockClient(client *c, int btype) {
     c->btype = btype;
     server.blocked_clients++;
     server.blocked_clients_by_type[btype]++;
-    addClientToShortTimeoutTable(c);
+    addClientToTimeoutTable(c);
 }
 
 /* This function is called in the beforeSleep() function of the event loop
diff --git a/src/server.c b/src/server.c
index 7240794f1..fc21a128e 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1511,13 +1511,6 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) {
         freeClient(c);
         return 1;
     } else if (c->flags & CLIENT_BLOCKED) {
-        /* Blocked OPS timeout is handled with milliseconds resolution.
-         * However note that the actual resolution is limited by
-         * server.hz. So for short timeouts (less than SERVER_SHORT_TIMEOUT
-         * milliseconds) we populate a Radix tree and handle such timeouts
-         * in clientsHandleShortTimeout(). */
-        if (checkBlockedClientTimeout(c,now_ms)) return 0;
-
         /* Cluster: handle unblock & redirect of clients blocked
          * into keys no longer served by this server. */
         if (server.cluster_enabled) {
@@ -1528,8 +1521,8 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) {
     return 0;
 }
 
-/* For shor timeouts, less than < CLIENT_SHORT_TIMEOUT milliseconds, we
- * populate a radix tree of 128 bit keys composed as such:
+/* For blocked clients timeouts we populate a radix tree of 128 bit keys
+ * composed as such:
  *
  *  [8 byte big endian expire time]+[8 byte client ID]
  *
@@ -1538,16 +1531,8 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) {
  * blocked with such timeout, we just go forward.
  *
  * Every time a client blocks with a short timeout, we add the client in
- * the tree. In beforeSleep() we call clientsHandleShortTimeout() to run
- * the tree and unblock the clients.
- *
- * Design hint: why we block only clients with short timeouts? For frugality:
- * Clients blocking for 30 seconds usually don't need to be unblocked
- * precisely, and anyway for the nature of Redis to *guarantee* unblock time
- * precision is hard, so we can avoid putting a large number of clients in
- * the radix tree without a good reason. This idea also has a role in memory
- * usage as well given that we don't do cleanup, the shorter a client timeout,
- * the less time it will stay in the radix tree. */
+ * the tree. In beforeSleep() we call clientsHandleTimeout() to run
+ * the tree and unblock the clients. */
 
 #define CLIENT_ST_KEYLEN 16    /* 8 bytes mstime + 8 bytes client ID. */
 
@@ -1568,13 +1553,9 @@ void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) {
 
 /* Add the specified client id / timeout as a key in the radix tree we use
  * to handle short timeouts. The client is not added to the list if its
- * timeout is longer than CLIENT_SHORT_TIMEOUT milliseconds. */
-void addClientToShortTimeoutTable(client *c) {
-    if (c->bpop.timeout == 0 ||
-        c->bpop.timeout - mstime() > CLIENT_SHORT_TIMEOUT)
-    {
-        return;
-    }
+ * timeout is zero (block forever). */
+void addClientToTimeoutTable(client *c) {
+    if (c->bpop.timeout == 0) return;
     uint64_t timeout = c->bpop.timeout;
     uint64_t id = c->id;
     unsigned char buf[CLIENT_ST_KEYLEN];
@@ -1584,7 +1565,7 @@ void addClientToShortTimeoutTable(client *c) {
 
 /* This function is called in beforeSleep() in order to unblock ASAP clients
  * that are waiting in blocking operations with a short timeout set. */
-void clientsHandleShortTimeout(void) {
+void clientsHandleTimeout(void) {
     if (raxSize(server.clients_timeout_table) == 0) return;
     uint64_t now = mstime();
     raxIterator ri;
@@ -1747,9 +1728,6 @@ void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) {
  */
 #define CLIENTS_CRON_MIN_ITERATIONS 5
 void clientsCron(void) {
-    /* Unblock short timeout clients ASAP. */
-    clientsHandleShortTimeout();
-
     /* Try to process at least numclients/server.hz of clients
      * per call. Since normally (if there are no big latency events) this
      * function is called server.hz times per second, in the average case we
@@ -2189,7 +2167,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     UNUSED(eventLoop);
 
     /* Handle precise timeouts of blocked clients. */
-    clientsHandleShortTimeout();
+    clientsHandleTimeout();
 
     /* We should handle pending reads clients ASAP after event loop. */
     handleClientsWithPendingReadsUsingThreads();
@@ -4101,11 +4079,13 @@ sds genRedisInfoString(const char *section) {
             "client_recent_max_input_buffer:%zu\r\n"
             "client_recent_max_output_buffer:%zu\r\n"
             "blocked_clients:%d\r\n"
-            "tracking_clients:%d\r\n",
+            "tracking_clients:%d\r\n"
+            "clients_in_timeout_table:%lld\r\n",
             listLength(server.clients)-listLength(server.slaves),
             maxin, maxout,
             server.blocked_clients,
-            server.tracking_clients);
+            server.tracking_clients,
+            raxSize(server.clients_timeout_table));
     }
 
     /* Memory */
diff --git a/src/server.h b/src/server.h
index 6f417b730..7ccab28ba 100644
--- a/src/server.h
+++ b/src/server.h
@@ -277,9 +277,6 @@ typedef long long ustime_t; /* microsecond time type. */
                                     buffer configuration. Just the first
                                     three: normal, slave, pubsub. */
 
-/* Other client related defines. */
-#define CLIENT_SHORT_TIMEOUT 2000  /* See clientsHandleShortTimeout(). */
-
 /* Slave replication state. Used in server.repl_state for slaves to remember
  * what to do next. */
 #define REPL_STATE_NONE 0 /* No active replication */
@@ -2140,7 +2137,7 @@ void disconnectAllBlockedClients(void);
 void handleClientsBlockedOnKeys(void);
 void signalKeyAsReady(redisDb *db, robj *key);
 void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids);
-void addClientToShortTimeoutTable(client *c);
+void addClientToTimeoutTable(client *c);
 
 /* expire.c -- Handling of expired keys */
 void activeExpireCycle(int type);

From 13e4c2a9b12cbaa0361e1701926e2d398f78e6b9 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 26 Mar 2020 16:05:20 +0100
Subject: [PATCH 0211/1098] Precise timeouts: fix comments after functional
 change.

---
 src/server.c | 10 +++++-----
 src/server.h |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/server.c b/src/server.c
index fc21a128e..2624cbecd 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1530,7 +1530,7 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) {
  * reached the timeout already, if they are no longer existing or no longer
  * blocked with such timeout, we just go forward.
  *
- * Every time a client blocks with a short timeout, we add the client in
+ * Every time a client blocks with a timeout, we add the client in
  * the tree. In beforeSleep() we call clientsHandleTimeout() to run
  * the tree and unblock the clients. */
 
@@ -1552,8 +1552,8 @@ void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) {
 }
 
 /* Add the specified client id / timeout as a key in the radix tree we use
- * to handle short timeouts. The client is not added to the list if its
- * timeout is zero (block forever). */
+ * to handle blocked clients timeouts. The client is not added to the list
+ * if its timeout is zero (block forever). */
 void addClientToTimeoutTable(client *c) {
     if (c->bpop.timeout == 0) return;
     uint64_t timeout = c->bpop.timeout;
@@ -1563,8 +1563,8 @@ void addClientToTimeoutTable(client *c) {
     raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL);
 }
 
-/* This function is called in beforeSleep() in order to unblock ASAP clients
- * that are waiting in blocking operations with a short timeout set. */
+/* This function is called in beforeSleep() in order to unblock clients
+ * that are waiting in blocking operations with a timeout set. */
 void clientsHandleTimeout(void) {
     if (raxSize(server.clients_timeout_table) == 0) return;
     uint64_t now = mstime();
diff --git a/src/server.h b/src/server.h
index 7ccab28ba..c6862051a 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1067,7 +1067,7 @@ struct redisServer {
     list *clients_pending_read;  /* Client has pending read socket buffers. */
     list *slaves, *monitors;    /* List of slaves and MONITORs */
     client *current_client;     /* Current client executing the command. */
-    rax *clients_timeout_table; /* Radix tree for clients with short timeout. */
+    rax *clients_timeout_table; /* Radix tree for blocked clients timeouts. */
     long fixed_time_expire;     /* If > 0, expire keys against server.mstime. */
     rax *clients_index;         /* Active clients dictionary by client ID. */
     int clients_paused;         /* True if clients are currently paused */

From 0e22cb2680db9f87fd232bc54419d538629edc2d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 27 Mar 2020 11:13:38 +0100
Subject: [PATCH 0212/1098] Precise timeouts: cleaup the table on unblock.

Now that this mechanism is the sole one used for blocked clients
timeouts, it is more wise to cleanup the table when the client unblocks
for any reason. We use a flag: CLIENT_IN_TO_TABLE, in order to avoid a
radix tree lookup when the client was already removed from the table
because we processed it by scanning the radix tree.
---
 src/blocked.c |  1 +
 src/server.c  | 20 ++++++++++++++++++--
 src/server.h  |  2 ++
 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/src/blocked.c b/src/blocked.c
index 443daec7f..795985ea1 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -186,6 +186,7 @@ void unblockClient(client *c) {
     server.blocked_clients_by_type[c->btype]--;
     c->flags &= ~CLIENT_BLOCKED;
     c->btype = BLOCKED_NONE;
+    removeClientFromTimeoutTable(c);
     queueClientForReprocessing(c);
 }
 
diff --git a/src/server.c b/src/server.c
index 2624cbecd..cfed2f91b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1560,7 +1560,20 @@ void addClientToTimeoutTable(client *c) {
     uint64_t id = c->id;
     unsigned char buf[CLIENT_ST_KEYLEN];
     encodeTimeoutKey(buf,timeout,id);
-    raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL);
+    if (raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL))
+        c->flags |= CLIENT_IN_TO_TABLE;
+}
+
+/* Remove the client from the table when it is unblocked for reasons
+ * different than timing out. */
+void removeClientFromTimeoutTable(client *c) {
+    if (!(c->flags & CLIENT_IN_TO_TABLE)) return;
+    c->flags &= ~CLIENT_IN_TO_TABLE;
+    uint64_t timeout = c->bpop.timeout;
+    uint64_t id = c->id;
+    unsigned char buf[CLIENT_ST_KEYLEN];
+    encodeTimeoutKey(buf,timeout,id);
+    raxRemove(server.clients_timeout_table,buf,sizeof(buf),NULL);
 }
 
 /* This function is called in beforeSleep() in order to unblock clients
@@ -1577,7 +1590,10 @@ void clientsHandleTimeout(void) {
         decodeTimeoutKey(ri.key,&timeout,&id);
         if (timeout >= now) break; /* All the timeouts are in the future. */
         client *c = lookupClientByID(id);
-        if (c) checkBlockedClientTimeout(c,now);
+        if (c) {
+            c->flags &= ~CLIENT_IN_TO_TABLE;
+            checkBlockedClientTimeout(c,now);
+        }
         raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL);
         raxSeek(&ri,"^",NULL,0);
     }
diff --git a/src/server.h b/src/server.h
index c6862051a..a3d91a09e 100644
--- a/src/server.h
+++ b/src/server.h
@@ -252,6 +252,7 @@ typedef long long ustime_t; /* microsecond time type. */
 #define CLIENT_TRACKING_OPTOUT (1ULL<<35) /* Tracking in opt-out mode. */
 #define CLIENT_TRACKING_CACHING (1ULL<<36) /* CACHING yes/no was given,
                                               depending on optin/optout mode. */
+#define CLIENT_IN_TO_TABLE (1ULL<<37) /* This client is in the timeout table. */
 
 /* Client block type (btype field in client structure)
  * if CLIENT_BLOCKED flag is set. */
@@ -2138,6 +2139,7 @@ void handleClientsBlockedOnKeys(void);
 void signalKeyAsReady(redisDb *db, robj *key);
 void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids);
 void addClientToTimeoutTable(client *c);
+void removeClientFromTimeoutTable(client *c);
 
 /* expire.c -- Handling of expired keys */
 void activeExpireCycle(int type);

From dd7e61d77f8e74b2d502965cf43df8034cd7923e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 27 Mar 2020 16:34:45 +0100
Subject: [PATCH 0213/1098] timeout.c created: move client timeouts code there.

---
 src/Makefile  |   2 +-
 src/blocked.c |  39 ----------
 src/server.c  | 128 +--------------------------------
 src/server.h  |   4 ++
 src/timeout.c | 192 ++++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 198 insertions(+), 167 deletions(-)
 create mode 100644 src/timeout.c

diff --git a/src/Makefile b/src/Makefile
index bbfb06440..3f982cc8e 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -206,7 +206,7 @@ endif
 
 REDIS_SERVER_NAME=redis-server
 REDIS_SENTINEL_NAME=redis-sentinel
-REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o
+REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o
 REDIS_CLI_NAME=redis-cli
 REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crc64.o siphash.o crc16.o
 REDIS_BENCHMARK_NAME=redis-benchmark
diff --git a/src/blocked.c b/src/blocked.c
index 795985ea1..e3a803ae3 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -31,9 +31,6 @@
  *
  * API:
  *
- * getTimeoutFromObjectOrReply() is just an utility function to parse a
- * timeout argument since blocking operations usually require a timeout.
- *
  * blockClient() set the CLIENT_BLOCKED flag in the client, and set the
  * specified block type 'btype' filed to one of BLOCKED_* macros.
  *
@@ -67,42 +64,6 @@
 
 int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int where);
 
-/* Get a timeout value from an object and store it into 'timeout'.
- * The final timeout is always stored as milliseconds as a time where the
- * timeout will expire, however the parsing is performed according to
- * the 'unit' that can be seconds or milliseconds.
- *
- * Note that if the timeout is zero (usually from the point of view of
- * commands API this means no timeout) the value stored into 'timeout'
- * is zero. */
-int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int unit) {
-    long long tval;
-    long double ftval;
-
-    if (unit == UNIT_SECONDS) {
-        if (getLongDoubleFromObjectOrReply(c,object,&ftval,
-            "timeout is not an float or out of range") != C_OK)
-            return C_ERR;
-        tval = (long long) (ftval * 1000.0);
-    } else {
-        if (getLongLongFromObjectOrReply(c,object,&tval,
-            "timeout is not an integer or out of range") != C_OK)
-            return C_ERR;
-    }
-
-    if (tval < 0) {
-        addReplyError(c,"timeout is negative");
-        return C_ERR;
-    }
-
-    if (tval > 0) {
-        tval += mstime();
-    }
-    *timeout = tval;
-
-    return C_OK;
-}
-
 /* Block a client for the specific operation type. Once the CLIENT_BLOCKED
  * flag is set client query buffer is not longer processed, but accumulated,
  * and will be processed when the client is unblocked. */
diff --git a/src/server.c b/src/server.c
index cfed2f91b..c1f7436a1 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1473,132 +1473,6 @@ int allPersistenceDisabled(void) {
     return server.saveparamslen == 0 && server.aof_state == AOF_OFF;
 }
 
-/* ========================== Clients timeouts ============================= */
-
-/* Check if this blocked client timedout (does nothing if the client is
- * not blocked right now). If so send a reply, unblock it, and return 1.
- * Otherwise 0 is returned and no operation is performed. */
-int checkBlockedClientTimeout(client *c, mstime_t now) {
-    if (c->flags & CLIENT_BLOCKED &&
-        c->bpop.timeout != 0
-        && c->bpop.timeout < now)
-    {
-        /* Handle blocking operation specific timeout. */
-        replyToBlockedClientTimedOut(c);
-        unblockClient(c);
-        return 1;
-    } else {
-        return 0;
-    }
-}
-
-/* Check for timeouts. Returns non-zero if the client was terminated.
- * The function gets the current time in milliseconds as argument since
- * it gets called multiple times in a loop, so calling gettimeofday() for
- * each iteration would be costly without any actual gain. */
-int clientsCronHandleTimeout(client *c, mstime_t now_ms) {
-    time_t now = now_ms/1000;
-
-    if (server.maxidletime &&
-        /* This handles the idle clients connection timeout if set. */
-        !(c->flags & CLIENT_SLAVE) &&   /* No timeout for slaves and monitors */
-        !(c->flags & CLIENT_MASTER) &&  /* No timeout for masters */
-        !(c->flags & CLIENT_BLOCKED) && /* No timeout for BLPOP */
-        !(c->flags & CLIENT_PUBSUB) &&  /* No timeout for Pub/Sub clients */
-        (now - c->lastinteraction > server.maxidletime))
-    {
-        serverLog(LL_VERBOSE,"Closing idle client");
-        freeClient(c);
-        return 1;
-    } else if (c->flags & CLIENT_BLOCKED) {
-        /* Cluster: handle unblock & redirect of clients blocked
-         * into keys no longer served by this server. */
-        if (server.cluster_enabled) {
-            if (clusterRedirectBlockedClientIfNeeded(c))
-                unblockClient(c);
-        }
-    }
-    return 0;
-}
-
-/* For blocked clients timeouts we populate a radix tree of 128 bit keys
- * composed as such:
- *
- *  [8 byte big endian expire time]+[8 byte client ID]
- *
- * We don't do any cleanup in the Radix tree: when we run the clients that
- * reached the timeout already, if they are no longer existing or no longer
- * blocked with such timeout, we just go forward.
- *
- * Every time a client blocks with a timeout, we add the client in
- * the tree. In beforeSleep() we call clientsHandleTimeout() to run
- * the tree and unblock the clients. */
-
-#define CLIENT_ST_KEYLEN 16    /* 8 bytes mstime + 8 bytes client ID. */
-
-/* Given client ID and timeout, write the resulting radix tree key in buf. */
-void encodeTimeoutKey(unsigned char *buf, uint64_t timeout, uint64_t id) {
-    timeout = htonu64(timeout);
-    memcpy(buf,&timeout,sizeof(timeout));
-    memcpy(buf+8,&id,sizeof(id));
-}
-
-/* Given a key encoded with encodeTimeoutKey(), resolve the fields and write
- * the timeout into *toptr and the client ID into *idptr. */
-void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) {
-    memcpy(toptr,buf,sizeof(*toptr));
-    *toptr = ntohu64(*toptr);
-    memcpy(idptr,buf+8,sizeof(*idptr));
-}
-
-/* Add the specified client id / timeout as a key in the radix tree we use
- * to handle blocked clients timeouts. The client is not added to the list
- * if its timeout is zero (block forever). */
-void addClientToTimeoutTable(client *c) {
-    if (c->bpop.timeout == 0) return;
-    uint64_t timeout = c->bpop.timeout;
-    uint64_t id = c->id;
-    unsigned char buf[CLIENT_ST_KEYLEN];
-    encodeTimeoutKey(buf,timeout,id);
-    if (raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL))
-        c->flags |= CLIENT_IN_TO_TABLE;
-}
-
-/* Remove the client from the table when it is unblocked for reasons
- * different than timing out. */
-void removeClientFromTimeoutTable(client *c) {
-    if (!(c->flags & CLIENT_IN_TO_TABLE)) return;
-    c->flags &= ~CLIENT_IN_TO_TABLE;
-    uint64_t timeout = c->bpop.timeout;
-    uint64_t id = c->id;
-    unsigned char buf[CLIENT_ST_KEYLEN];
-    encodeTimeoutKey(buf,timeout,id);
-    raxRemove(server.clients_timeout_table,buf,sizeof(buf),NULL);
-}
-
-/* This function is called in beforeSleep() in order to unblock clients
- * that are waiting in blocking operations with a timeout set. */
-void clientsHandleTimeout(void) {
-    if (raxSize(server.clients_timeout_table) == 0) return;
-    uint64_t now = mstime();
-    raxIterator ri;
-    raxStart(&ri,server.clients_timeout_table);
-    raxSeek(&ri,"^",NULL,0);
-
-    while(raxNext(&ri)) {
-        uint64_t id, timeout;
-        decodeTimeoutKey(ri.key,&timeout,&id);
-        if (timeout >= now) break; /* All the timeouts are in the future. */
-        client *c = lookupClientByID(id);
-        if (c) {
-            c->flags &= ~CLIENT_IN_TO_TABLE;
-            checkBlockedClientTimeout(c,now);
-        }
-        raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL);
-        raxSeek(&ri,"^",NULL,0);
-    }
-}
-
 /* ======================= Cron: called every 100 ms ======================== */
 
 /* Add a sample to the operations per second array of samples. */
@@ -2183,7 +2057,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     UNUSED(eventLoop);
 
     /* Handle precise timeouts of blocked clients. */
-    clientsHandleTimeout();
+    handleBlockedClientsTimeout();
 
     /* We should handle pending reads clients ASAP after event loop. */
     handleClientsWithPendingReadsUsingThreads();
diff --git a/src/server.h b/src/server.h
index a3d91a09e..691f47c48 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2138,8 +2138,12 @@ void disconnectAllBlockedClients(void);
 void handleClientsBlockedOnKeys(void);
 void signalKeyAsReady(redisDb *db, robj *key);
 void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids);
+
+/* timeout.c -- Blocked clients timeout and connections timeout. */
 void addClientToTimeoutTable(client *c);
 void removeClientFromTimeoutTable(client *c);
+void handleBlockedClientsTimeout(void);
+int clientsCronHandleTimeout(client *c, mstime_t now_ms);
 
 /* expire.c -- Handling of expired keys */
 void activeExpireCycle(int type);
diff --git a/src/timeout.c b/src/timeout.c
new file mode 100644
index 000000000..ea2032e2a
--- /dev/null
+++ b/src/timeout.c
@@ -0,0 +1,192 @@
+/* Copyright (c) 2009-2020, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "server.h"
+#include "cluster.h"
+
+/* ========================== Clients timeouts ============================= */
+
+/* Check if this blocked client timedout (does nothing if the client is
+ * not blocked right now). If so send a reply, unblock it, and return 1.
+ * Otherwise 0 is returned and no operation is performed. */
+int checkBlockedClientTimeout(client *c, mstime_t now) {
+    if (c->flags & CLIENT_BLOCKED &&
+        c->bpop.timeout != 0
+        && c->bpop.timeout < now)
+    {
+        /* Handle blocking operation specific timeout. */
+        replyToBlockedClientTimedOut(c);
+        unblockClient(c);
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+/* Check for timeouts. Returns non-zero if the client was terminated.
+ * The function gets the current time in milliseconds as argument since
+ * it gets called multiple times in a loop, so calling gettimeofday() for
+ * each iteration would be costly without any actual gain. */
+int clientsCronHandleTimeout(client *c, mstime_t now_ms) {
+    time_t now = now_ms/1000;
+
+    if (server.maxidletime &&
+        /* This handles the idle clients connection timeout if set. */
+        !(c->flags & CLIENT_SLAVE) &&   /* No timeout for slaves and monitors */
+        !(c->flags & CLIENT_MASTER) &&  /* No timeout for masters */
+        !(c->flags & CLIENT_BLOCKED) && /* No timeout for BLPOP */
+        !(c->flags & CLIENT_PUBSUB) &&  /* No timeout for Pub/Sub clients */
+        (now - c->lastinteraction > server.maxidletime))
+    {
+        serverLog(LL_VERBOSE,"Closing idle client");
+        freeClient(c);
+        return 1;
+    } else if (c->flags & CLIENT_BLOCKED) {
+        /* Cluster: handle unblock & redirect of clients blocked
+         * into keys no longer served by this server. */
+        if (server.cluster_enabled) {
+            if (clusterRedirectBlockedClientIfNeeded(c))
+                unblockClient(c);
+        }
+    }
+    return 0;
+}
+
+/* For blocked clients timeouts we populate a radix tree of 128 bit keys
+ * composed as such:
+ *
+ *  [8 byte big endian expire time]+[8 byte client ID]
+ *
+ * We don't do any cleanup in the Radix tree: when we run the clients that
+ * reached the timeout already, if they are no longer existing or no longer
+ * blocked with such timeout, we just go forward.
+ *
+ * Every time a client blocks with a timeout, we add the client in
+ * the tree. In beforeSleep() we call handleBlockedClientsTimeout() to run
+ * the tree and unblock the clients. */
+
+#define CLIENT_ST_KEYLEN 16    /* 8 bytes mstime + 8 bytes client ID. */
+
+/* Given client ID and timeout, write the resulting radix tree key in buf. */
+void encodeTimeoutKey(unsigned char *buf, uint64_t timeout, uint64_t id) {
+    timeout = htonu64(timeout);
+    memcpy(buf,&timeout,sizeof(timeout));
+    memcpy(buf+8,&id,sizeof(id));
+}
+
+/* Given a key encoded with encodeTimeoutKey(), resolve the fields and write
+ * the timeout into *toptr and the client ID into *idptr. */
+void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) {
+    memcpy(toptr,buf,sizeof(*toptr));
+    *toptr = ntohu64(*toptr);
+    memcpy(idptr,buf+8,sizeof(*idptr));
+}
+
+/* Add the specified client id / timeout as a key in the radix tree we use
+ * to handle blocked clients timeouts. The client is not added to the list
+ * if its timeout is zero (block forever). */
+void addClientToTimeoutTable(client *c) {
+    if (c->bpop.timeout == 0) return;
+    uint64_t timeout = c->bpop.timeout;
+    uint64_t id = c->id;
+    unsigned char buf[CLIENT_ST_KEYLEN];
+    encodeTimeoutKey(buf,timeout,id);
+    if (raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL))
+        c->flags |= CLIENT_IN_TO_TABLE;
+}
+
+/* Remove the client from the table when it is unblocked for reasons
+ * different than timing out. */
+void removeClientFromTimeoutTable(client *c) {
+    if (!(c->flags & CLIENT_IN_TO_TABLE)) return;
+    c->flags &= ~CLIENT_IN_TO_TABLE;
+    uint64_t timeout = c->bpop.timeout;
+    uint64_t id = c->id;
+    unsigned char buf[CLIENT_ST_KEYLEN];
+    encodeTimeoutKey(buf,timeout,id);
+    raxRemove(server.clients_timeout_table,buf,sizeof(buf),NULL);
+}
+
+/* This function is called in beforeSleep() in order to unblock clients
+ * that are waiting in blocking operations with a timeout set. */
+void handleBlockedClientsTimeout(void) {
+    if (raxSize(server.clients_timeout_table) == 0) return;
+    uint64_t now = mstime();
+    raxIterator ri;
+    raxStart(&ri,server.clients_timeout_table);
+    raxSeek(&ri,"^",NULL,0);
+
+    while(raxNext(&ri)) {
+        uint64_t id, timeout;
+        decodeTimeoutKey(ri.key,&timeout,&id);
+        if (timeout >= now) break; /* All the timeouts are in the future. */
+        client *c = lookupClientByID(id);
+        if (c) {
+            c->flags &= ~CLIENT_IN_TO_TABLE;
+            checkBlockedClientTimeout(c,now);
+        }
+        raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL);
+        raxSeek(&ri,"^",NULL,0);
+    }
+}
+
+/* Get a timeout value from an object and store it into 'timeout'.
+ * The final timeout is always stored as milliseconds as a time where the
+ * timeout will expire, however the parsing is performed according to
+ * the 'unit' that can be seconds or milliseconds.
+ *
+ * Note that if the timeout is zero (usually from the point of view of
+ * commands API this means no timeout) the value stored into 'timeout'
+ * is zero. */
+int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int unit) {
+    long long tval;
+    long double ftval;
+
+    if (unit == UNIT_SECONDS) {
+        if (getLongDoubleFromObjectOrReply(c,object,&ftval,
+            "timeout is not an float or out of range") != C_OK)
+            return C_ERR;
+        tval = (long long) (ftval * 1000.0);
+    } else {
+        if (getLongLongFromObjectOrReply(c,object,&tval,
+            "timeout is not an integer or out of range") != C_OK)
+            return C_ERR;
+    }
+
+    if (tval < 0) {
+        addReplyError(c,"timeout is negative");
+        return C_ERR;
+    }
+
+    if (tval > 0) {
+        tval += mstime();
+    }
+    *timeout = tval;
+
+    return C_OK;
+}

From 36ee294e83e2c1e9d8f3ec04de3b6c734185f699 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Sat, 28 Mar 2020 20:59:01 +0800
Subject: [PATCH 0214/1098] PSYNC2: reset backlog_idx and master_repl_offset
 correctly

---
 src/replication.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index 20f9a2129..3e9910374 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2754,11 +2754,16 @@ void replicationCacheMasterUsingMyself(void) {
                 server.master_repl_offset,
                 delta);
         server.master_initial_offset = server.master_repl_meaningful_offset;
-        server.repl_backlog_histlen -= delta;
-        server.repl_backlog_idx =
-            (server.repl_backlog_idx + (server.repl_backlog_size - delta)) %
-            server.repl_backlog_size;
-        if (server.repl_backlog_histlen < 0) server.repl_backlog_histlen = 0;
+        server.master_repl_offset = server.master_repl_meaningful_offset;
+        if (server.repl_backlog_histlen <= delta) {
+            server.repl_backlog_histlen = 0;
+            server.repl_backlog_idx = 0;
+        } else {
+            server.repl_backlog_histlen -= delta;
+            server.repl_backlog_idx =
+                (server.repl_backlog_idx + (server.repl_backlog_size - delta)) %
+                server.repl_backlog_size;
+        }
     }
 
     /* The master client we create can be set to any DBID, because

From 6d2f1188ac7fdbb03c0f3f83bfa06a02f2bbca3d Mon Sep 17 00:00:00 2001
From: OMG-By <504094596@qq.com>
Date: Sun, 29 Mar 2020 00:04:59 +0800
Subject: [PATCH 0215/1098] fix: dict.c->dictResize()->minimal  type

---
 src/dict.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/dict.c b/src/dict.c
index 106467ef7..dec15771d 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -134,7 +134,7 @@ int _dictInit(dict *d, dictType *type,
  * but with the invariant of a USED/BUCKETS ratio near to <= 1 */
 int dictResize(dict *d)
 {
-    int minimal;
+    unsigned long minimal;
 
     if (!dict_can_resize || dictIsRehashing(d)) return DICT_ERR;
     minimal = d->ht[0].used;

From 3f0afab30187ecd177683a942dee819237c9c4b4 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Sun, 29 Mar 2020 17:50:42 +0300
Subject: [PATCH 0216/1098] PERSIST should notify a keyspace event

---
 src/expire.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/expire.c b/src/expire.c
index 5aff72ee0..c102a01ff 100644
--- a/src/expire.c
+++ b/src/expire.c
@@ -590,6 +590,7 @@ void pttlCommand(client *c) {
 void persistCommand(client *c) {
     if (lookupKeyWrite(c->db,c->argv[1])) {
         if (removeExpire(c->db,c->argv[1])) {
+            notifyKeyspaceEvent(NOTIFY_GENERIC,"persist",c->argv[1],c->db->id);
             addReply(c,shared.cone);
             server.dirty++;
         } else {

From 1ef44f8243229bb8976f52b4ba6ad9ddce2e5bde Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Sun, 29 Mar 2020 23:06:50 -0400
Subject: [PATCH 0217/1098] add check for not providing both optin optout flag

---
 src/networking.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index a550e4040..bad3e2c25 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2314,6 +2314,14 @@ NULL
                 return;
             }
 
+            if ((options & CLIENT_TRACKING_OPTIN) && (options & CLIENT_TRACKING_OPTOUT))
+            {
+                addReplyError(c,
+                "You can't specify both OPTIN mode and OPTOUT mode");
+                zfree(prefix);
+                return;
+            }
+
             enableTracking(c,redir,options,prefix,numprefix);
         } else if (!strcasecmp(c->argv[2]->ptr,"off")) {
             disableTracking(c);

From 86c76cad74f9effb861dd1fc2f7d9f761b52077f Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Sun, 29 Mar 2020 23:20:54 -0400
Subject: [PATCH 0218/1098] add check for not switching between optin optout
 mode directly

---
 src/networking.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index bad3e2c25..b6ab8eb5d 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2314,7 +2314,7 @@ NULL
                 return;
             }
 
-            if ((options & CLIENT_TRACKING_OPTIN) && (options & CLIENT_TRACKING_OPTOUT))
+            if (options & CLIENT_TRACKING_OPTIN && options & CLIENT_TRACKING_OPTOUT)
             {
                 addReplyError(c,
                 "You can't specify both OPTIN mode and OPTOUT mode");
@@ -2322,6 +2322,17 @@ NULL
                 return;
             }
 
+            if ((options & CLIENT_TRACKING_OPTIN && c->flags & CLIENT_TRACKING_OPTOUT) ||
+                (options & CLIENT_TRACKING_OPTOUT && c->flags & CLIENT_TRACKING_OPTIN))
+            {
+                addReplyError(c,
+                "You can't switch OPTIN/OPTOUT mode before disabling "
+                "tracking for this client, and then re-enabling it with "
+                "a different mode.");
+                zfree(prefix);
+                return;
+            }
+
             enableTracking(c,redir,options,prefix,numprefix);
         } else if (!strcasecmp(c->argv[2]->ptr,"off")) {
             disableTracking(c);

From 12d7479144c36bea03f7d61f8474e31603eabd84 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Mon, 30 Mar 2020 10:52:59 +0300
Subject: [PATCH 0219/1098] streamReplyWithRange: Redundant XSETIDs to replica

propagate_last_id is declared outside of the loop but used
only from within the loop. Once it's '1' it will never go
back to '0' and will replicate XSETID even for IDs that
don't actually change the last_id.
While not a serious bug (XSETID always used group->last_id
so there's no risk), it does causes redundant traffic
between master and its replicas
---
 src/t_stream.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 557d1d642..00d1cbf1c 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -935,7 +935,6 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
     streamIterator si;
     int64_t numfields;
     streamID id;
-    int propagate_last_id = 0;
 
     /* If the client is asking for some history, we serve it using a
      * different function, so that we return entries *solely* from its
@@ -951,6 +950,8 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
         arraylen_ptr = addReplyDeferredLen(c);
     streamIteratorStart(&si,s,start,end,rev);
     while(streamIteratorGetID(&si,&id,&numfields)) {
+        int propagate_last_id = 0;
+
         /* Update the group last_id if needed. */
         if (group && streamCompareID(&id,&group->last_id) > 0) {
             group->last_id = id;

From 3ef59b50c40c87d6d50d5a52f3859c1a152d805d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 30 Mar 2020 15:22:55 +0200
Subject: [PATCH 0220/1098] Precise timeouts: reference client pointer
 directly.

---
 src/timeout.c | 29 +++++++++++++----------------
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/src/timeout.c b/src/timeout.c
index ea2032e2a..bb5999418 100644
--- a/src/timeout.c
+++ b/src/timeout.c
@@ -93,18 +93,19 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) {
 #define CLIENT_ST_KEYLEN 16    /* 8 bytes mstime + 8 bytes client ID. */
 
 /* Given client ID and timeout, write the resulting radix tree key in buf. */
-void encodeTimeoutKey(unsigned char *buf, uint64_t timeout, uint64_t id) {
+void encodeTimeoutKey(unsigned char *buf, uint64_t timeout, client *c) {
     timeout = htonu64(timeout);
     memcpy(buf,&timeout,sizeof(timeout));
-    memcpy(buf+8,&id,sizeof(id));
+    memcpy(buf+8,&c,sizeof(c));
+    if (sizeof(c) == 4) memset(buf+12,0,4); /* Zero padding for 32bit target. */
 }
 
 /* Given a key encoded with encodeTimeoutKey(), resolve the fields and write
- * the timeout into *toptr and the client ID into *idptr. */
-void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) {
+ * the timeout into *toptr and the client pointer into *cptr. */
+void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, client **cptr) {
     memcpy(toptr,buf,sizeof(*toptr));
     *toptr = ntohu64(*toptr);
-    memcpy(idptr,buf+8,sizeof(*idptr));
+    memcpy(cptr,buf+8,sizeof(*cptr));
 }
 
 /* Add the specified client id / timeout as a key in the radix tree we use
@@ -113,9 +114,8 @@ void decodeTimeoutKey(unsigned char *buf, uint64_t *toptr, uint64_t *idptr) {
 void addClientToTimeoutTable(client *c) {
     if (c->bpop.timeout == 0) return;
     uint64_t timeout = c->bpop.timeout;
-    uint64_t id = c->id;
     unsigned char buf[CLIENT_ST_KEYLEN];
-    encodeTimeoutKey(buf,timeout,id);
+    encodeTimeoutKey(buf,timeout,c);
     if (raxTryInsert(server.clients_timeout_table,buf,sizeof(buf),NULL,NULL))
         c->flags |= CLIENT_IN_TO_TABLE;
 }
@@ -126,9 +126,8 @@ void removeClientFromTimeoutTable(client *c) {
     if (!(c->flags & CLIENT_IN_TO_TABLE)) return;
     c->flags &= ~CLIENT_IN_TO_TABLE;
     uint64_t timeout = c->bpop.timeout;
-    uint64_t id = c->id;
     unsigned char buf[CLIENT_ST_KEYLEN];
-    encodeTimeoutKey(buf,timeout,id);
+    encodeTimeoutKey(buf,timeout,c);
     raxRemove(server.clients_timeout_table,buf,sizeof(buf),NULL);
 }
 
@@ -142,14 +141,12 @@ void handleBlockedClientsTimeout(void) {
     raxSeek(&ri,"^",NULL,0);
 
     while(raxNext(&ri)) {
-        uint64_t id, timeout;
-        decodeTimeoutKey(ri.key,&timeout,&id);
+        uint64_t timeout;
+        client *c;
+        decodeTimeoutKey(ri.key,&timeout,&c);
         if (timeout >= now) break; /* All the timeouts are in the future. */
-        client *c = lookupClientByID(id);
-        if (c) {
-            c->flags &= ~CLIENT_IN_TO_TABLE;
-            checkBlockedClientTimeout(c,now);
-        }
+        c->flags &= ~CLIENT_IN_TO_TABLE;
+        checkBlockedClientTimeout(c,now);
         raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL);
         raxSeek(&ri,"^",NULL,0);
     }

From 7698316fda98018ee56c010e87a6ded46b79ad67 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 31 Mar 2020 10:27:47 +0200
Subject: [PATCH 0221/1098] Fix RM_Call() stale comment due to cut&paste.

---
 src/module.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/module.c b/src/module.c
index 6f61a5ca8..7d9eb3719 100644
--- a/src/module.c
+++ b/src/module.c
@@ -3341,9 +3341,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
     }
     call(c,call_flags);
 
-    /* Convert the result of the Redis command into a suitable Lua type.
-     * The first thing we need is to create a single string from the client
-     * output buffers. */
+    /* Convert the result of the Redis command into a module reply. */
     sds proto = sdsnewlen(c->buf,c->bufpos);
     c->bufpos = 0;
     while(listLength(c->reply)) {

From 9dcf878f1b735e4714a2f7bbea7ce567768605bf Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 31 Mar 2020 11:00:45 +0200
Subject: [PATCH 0222/1098] Fix module commands propagation double MULTI bug.

37a10cef introduced automatic wrapping of MULTI/EXEC for the
alsoPropagate API. However this collides with the built-in mechanism
already present in module.c. To avoid complex changes near Redis 6 GA
this commit introduces the ability to exclude call() MUTLI/EXEC wrapping
for also propagate in order to continue to use the old code paths in
module.c.
---
 src/module.c              |  7 +------
 src/server.c              |  8 ++++++--
 src/server.h              |  2 ++
 tests/modules/propagate.c | 16 ++++++++++++++++
 4 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/src/module.c b/src/module.c
index 7d9eb3719..61dc25169 100644
--- a/src/module.c
+++ b/src/module.c
@@ -3326,13 +3326,8 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
      * a Lua script in the context of AOF and slaves. */
     if (replicate) moduleReplicateMultiIfNeeded(ctx);
 
-    if (ctx->client->flags & CLIENT_MULTI ||
-        ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) {
-        c->flags |= CLIENT_MULTI;
-    }
-
     /* Run the command */
-    int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS;
+    int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS | CMD_CALL_NOWRAP;
     if (replicate) {
         if (!(flags & REDISMODULE_ARGV_NO_AOF))
             call_flags |= CMD_CALL_PROPAGATE_AOF;
diff --git a/src/server.c b/src/server.c
index c1f7436a1..93bafbae8 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3266,11 +3266,15 @@ void call(client *c, int flags) {
             int multi_emitted = 0;
             /* Wrap the commands in server.also_propagate array,
              * but don't wrap it if we are already in MULIT context,
-             * in case the nested MULIT/EXEC.
+             * in case the nested MULTI/EXEC.
              *
              * And if the array contains only one command, no need to
              * wrap it, since the single command is atomic. */
-            if (server.also_propagate.numops > 1 && !(c->flags & CLIENT_MULTI)) {
+            if (server.also_propagate.numops > 1 &&
+                !(c->cmd->flags & CMD_MODULE) &&
+                !(c->flags & CLIENT_MULTI) &&
+                !(flags & CMD_CALL_NOWRAP))
+            {
                 execCommandPropagateMulti(c);
                 multi_emitted = 1;
             }
diff --git a/src/server.h b/src/server.h
index 691f47c48..d5be5ed29 100644
--- a/src/server.h
+++ b/src/server.h
@@ -395,6 +395,8 @@ typedef long long ustime_t; /* microsecond time type. */
 #define CMD_CALL_PROPAGATE_REPL (1<<3)
 #define CMD_CALL_PROPAGATE (CMD_CALL_PROPAGATE_AOF|CMD_CALL_PROPAGATE_REPL)
 #define CMD_CALL_FULL (CMD_CALL_SLOWLOG | CMD_CALL_STATS | CMD_CALL_PROPAGATE)
+#define CMD_CALL_NOWRAP (1<<4)  /* Don't wrap also propagate array into
+                                   MULTI/EXEC: the caller will handle it.  */
 
 /* Command propagation flags, see propagate() function */
 #define PROPAGATE_NONE 0
diff --git a/tests/modules/propagate.c b/tests/modules/propagate.c
index f83af1799..2571ee43d 100644
--- a/tests/modules/propagate.c
+++ b/tests/modules/propagate.c
@@ -89,6 +89,16 @@ int propagateTestCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc
     return REDISMODULE_OK;
 }
 
+int propagateTest2Command(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+
+    RedisModule_Replicate(ctx,"INCR","c","counter");
+    RedisModule_ReplyWithSimpleString(ctx,"OK");
+    return REDISMODULE_OK;
+}
+
 int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
@@ -100,5 +110,11 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
                 propagateTestCommand,
                 "",1,1,1) == REDISMODULE_ERR)
             return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx,"propagate-test-2",
+                propagateTest2Command,
+                "",1,1,1) == REDISMODULE_ERR)
+            return REDISMODULE_ERR;
+
     return REDISMODULE_OK;
 }

From 95f154985c31d3925d6d8d296b44df43735622fc Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 31 Mar 2020 12:04:06 +0200
Subject: [PATCH 0223/1098] Modify the propagate unit test to show more cases.

---
 tests/modules/propagate.c | 32 ++++++++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/tests/modules/propagate.c b/tests/modules/propagate.c
index 2571ee43d..13277b19d 100644
--- a/tests/modules/propagate.c
+++ b/tests/modules/propagate.c
@@ -64,7 +64,8 @@ void *threadMain(void *arg) {
     RedisModule_SelectDb(ctx,9); /* Tests ran in database number 9. */
     for (int i = 0; i < 10; i++) {
         RedisModule_ThreadSafeContextLock(ctx);
-        RedisModule_Replicate(ctx,"INCR","c","thread");
+        RedisModule_Replicate(ctx,"INCR","c","a-from-thread");
+        RedisModule_Replicate(ctx,"INCR","c","b-from-thread");
         RedisModule_ThreadSafeContextUnlock(ctx);
     }
     RedisModule_FreeThreadSafeContext(ctx);
@@ -94,7 +95,29 @@ int propagateTest2Command(RedisModuleCtx *ctx, RedisModuleString **argv, int arg
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
 
-    RedisModule_Replicate(ctx,"INCR","c","counter");
+    /* Replicate two commands to test MULTI/EXEC wrapping. */
+    RedisModule_Replicate(ctx,"INCR","c","counter-1");
+    RedisModule_Replicate(ctx,"INCR","c","counter-2");
+    RedisModule_ReplyWithSimpleString(ctx,"OK");
+    return REDISMODULE_OK;
+}
+
+int propagateTest3Command(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+    RedisModuleCallReply *reply;
+
+    /* This test mixes multiple propagation systems. */
+    reply = RedisModule_Call(ctx, "INCR", "c!", "using-call");
+    RedisModule_FreeCallReply(reply);
+
+    RedisModule_Replicate(ctx,"INCR","c","counter-1");
+    RedisModule_Replicate(ctx,"INCR","c","counter-2");
+
+    reply = RedisModule_Call(ctx, "INCR", "c!", "after-call");
+    RedisModule_FreeCallReply(reply);
+
     RedisModule_ReplyWithSimpleString(ctx,"OK");
     return REDISMODULE_OK;
 }
@@ -116,5 +139,10 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
                 "",1,1,1) == REDISMODULE_ERR)
             return REDISMODULE_ERR;
 
+    if (RedisModule_CreateCommand(ctx,"propagate-test-3",
+                propagateTest3Command,
+                "",1,1,1) == REDISMODULE_ERR)
+            return REDISMODULE_ERR;
+
     return REDISMODULE_OK;
 }

From 4379b8b411e4940f137623984e2931f0448b4e89 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 31 Mar 2020 12:09:38 +0200
Subject: [PATCH 0224/1098] Fix the propagate Tcl test after module changes.

---
 tests/unit/moduleapi/propagate.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/moduleapi/propagate.tcl b/tests/unit/moduleapi/propagate.tcl
index 71307ce33..73f795c71 100644
--- a/tests/unit/moduleapi/propagate.tcl
+++ b/tests/unit/moduleapi/propagate.tcl
@@ -20,7 +20,7 @@ tags "modules" {
 
                 wait_for_condition 5000 10 {
                     ([$replica get timer] eq "10") && \
-                    ([$replica get thread] eq "10")
+                    ([$replica get a-from-thread] eq "10")
                 } else {
                     fail "The two counters don't match the expected value."
                 }

From d6eb3afd13b0f186d8d0de72d0c43c4cd0464e87 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Sun, 29 Mar 2020 13:08:21 +0300
Subject: [PATCH 0225/1098] Modules: Test MULTI/EXEC replication of
 RM_Replicate

Makse sure call() doesn't wrap replicated commands with
a redundant MULTI/EXEC

Other, unrelated changes:
1. Formatting compiler warning in INFO CLIENTS
2. Use CLIENT_ID_AOF instead of UINT64_MAX
---
 src/aof.c                          |  2 +-
 src/networking.c                   |  9 +++++---
 src/redismodule.h                  |  2 +-
 src/server.c                       | 11 ++++++----
 src/server.h                       |  1 +
 tests/unit/moduleapi/propagate.tcl | 33 ++++++++++++++++++++++++++++++
 6 files changed, 49 insertions(+), 9 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index 6bb239252..301a40848 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -831,7 +831,7 @@ int loadAppendOnlyFile(char *filename) {
         if (cmd == server.multiCommand) valid_before_multi = valid_up_to;
 
         /* Run the command in the context of a fake client */
-        fakeClient->cmd = cmd;
+        fakeClient->cmd = fakeClient->lastcmd = cmd;
         if (fakeClient->flags & CLIENT_MULTI &&
             fakeClient->cmd->proc != execCommand)
         {
diff --git a/src/networking.c b/src/networking.c
index a550e4040..fcaa164a9 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -373,13 +373,16 @@ void addReplyErrorLength(client *c, const char *s, size_t len) {
      * will produce an error. However it is useful to log such events since
      * they are rare and may hint at errors in a script or a bug in Redis. */
     int ctype = getClientType(c);
-    if (ctype == CLIENT_TYPE_MASTER || ctype == CLIENT_TYPE_SLAVE) {
-        char* to = ctype == CLIENT_TYPE_MASTER? "master": "replica";
-        char* from = ctype == CLIENT_TYPE_MASTER? "replica": "master";
+    if (ctype == CLIENT_TYPE_MASTER || ctype == CLIENT_TYPE_SLAVE || c->id == CLIENT_ID_AOF) {
+        char* to = c->id == CLIENT_ID_AOF ? "AOF-client" :
+                       ctype == CLIENT_TYPE_MASTER ? "master" : "replica";
+        char* from = c->id == CLIENT_ID_AOF ? "server" :
+                         ctype == CLIENT_TYPE_MASTER ? "replica" : "master";
         char *cmdname = c->lastcmd ? c->lastcmd->name : "<unknown>";
         serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error "
                              "to its %s: '%s' after processing the command "
                              "'%s'", from, to, s, cmdname);
+        server.stat_unexpected_error_replies++;
     }
 }
 
diff --git a/src/redismodule.h b/src/redismodule.h
index a43443f13..d26c41456 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -673,7 +673,7 @@ int REDISMODULE_API_FUNC(RedisModule_AuthenticateClientWithUser)(RedisModuleCtx
 void REDISMODULE_API_FUNC(RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id);
 #endif
 
-#define RedisModule_IsAOFClient(id) ((id) == UINT64_MAX)
+#define RedisModule_IsAOFClient(id) ((id) == CLIENT_ID_AOF)
 
 /* This is included inline inside each Redis module. */
 static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int apiver) __attribute__((unused));
diff --git a/src/server.c b/src/server.c
index 93bafbae8..33f875ba9 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2661,6 +2661,7 @@ void resetServerStats(void) {
     }
     server.stat_net_input_bytes = 0;
     server.stat_net_output_bytes = 0;
+    server.stat_unexpected_error_replies = 0;
     server.aof_delayed_fsync = 0;
 }
 
@@ -3265,7 +3266,7 @@ void call(client *c, int flags) {
         if (flags & CMD_CALL_PROPAGATE) {
             int multi_emitted = 0;
             /* Wrap the commands in server.also_propagate array,
-             * but don't wrap it if we are already in MULIT context,
+             * but don't wrap it if we are already in MULTI context,
              * in case the nested MULTI/EXEC.
              *
              * And if the array contains only one command, no need to
@@ -3974,7 +3975,7 @@ sds genRedisInfoString(const char *section) {
             "client_recent_max_output_buffer:%zu\r\n"
             "blocked_clients:%d\r\n"
             "tracking_clients:%d\r\n"
-            "clients_in_timeout_table:%lld\r\n",
+            "clients_in_timeout_table:%ld\r\n",
             listLength(server.clients)-listLength(server.slaves),
             maxin, maxout,
             server.blocked_clients,
@@ -4229,7 +4230,8 @@ sds genRedisInfoString(const char *section) {
             "active_defrag_key_hits:%lld\r\n"
             "active_defrag_key_misses:%lld\r\n"
             "tracking_total_keys:%lld\r\n"
-            "tracking_total_items:%lld\r\n",
+            "tracking_total_items:%lld\r\n"
+            "unexpected_error_replies:%lld\r\n",
             server.stat_numconnections,
             server.stat_numcommands,
             getInstantaneousMetric(STATS_METRIC_COMMAND),
@@ -4258,7 +4260,8 @@ sds genRedisInfoString(const char *section) {
             server.stat_active_defrag_key_hits,
             server.stat_active_defrag_key_misses,
             (unsigned long long) trackingGetTotalKeys(),
-            (unsigned long long) trackingGetTotalItems());
+            (unsigned long long) trackingGetTotalItems(),
+            server.stat_unexpected_error_replies);
     }
 
     /* Replication */
diff --git a/src/server.h b/src/server.h
index d5be5ed29..59cec9352 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1129,6 +1129,7 @@ struct redisServer {
     size_t stat_rdb_cow_bytes;      /* Copy on write bytes during RDB saving. */
     size_t stat_aof_cow_bytes;      /* Copy on write bytes during AOF rewrite. */
     size_t stat_module_cow_bytes;   /* Copy on write bytes during module fork. */
+    long long stat_unexpected_error_replies; /* Number of unexpected (aof-loading, replica to master, etc.) error replies */
     /* The following two are used to track instantaneous metrics, like
      * number of operations per second, network traffic. */
     struct {
diff --git a/tests/unit/moduleapi/propagate.tcl b/tests/unit/moduleapi/propagate.tcl
index 73f795c71..aa0f55e5e 100644
--- a/tests/unit/moduleapi/propagate.tcl
+++ b/tests/unit/moduleapi/propagate.tcl
@@ -24,7 +24,40 @@ tags "modules" {
                 } else {
                     fail "The two counters don't match the expected value."
                 }
+
+                $master propagate-test-2
+                $master propagate-test-3
+                $master multi
+                $master propagate-test-2
+                $master propagate-test-3
+                $master exec
+                wait_for_ofs_sync $master $replica
+
+                assert_equal [s -1 unexpected_error_replies] 0
             }
         }
     }
 }
+
+tags "modules aof" {
+    test {Modules RM_Replicate replicates MULTI/EXEC correctly} {
+        start_server [list overrides [list loadmodule "$testmodule"]] {
+            # Enable the AOF
+            r config set appendonly yes
+            r config set auto-aof-rewrite-percentage 0 ; # Disable auto-rewrite.
+            waitForBgrewriteaof r
+
+            r propagate-test-2
+            r propagate-test-3
+            r multi
+            r propagate-test-2
+            r propagate-test-3
+            r exec
+
+            # Load the AOF
+            r debug loadaof
+
+            assert_equal [s 0 unexpected_error_replies] 0
+        }
+    }
+}

From 6c8221580c4ed5d3602d10eb467603d576eaf9c8 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Tue, 31 Mar 2020 17:37:05 +0300
Subject: [PATCH 0226/1098] RENAME can unblock XREADGROUP

Other changes:
Support stream in serverLogObjectDebugInfo
---
 src/db.c                           |  3 ++-
 src/debug.c                        |  2 ++
 tests/unit/type/stream-cgroups.tcl | 21 +++++++++++++++++++++
 3 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/db.c b/src/db.c
index 04e26c33b..211bb978d 100644
--- a/src/db.c
+++ b/src/db.c
@@ -182,7 +182,8 @@ void dbAdd(redisDb *db, robj *key, robj *val) {
 
     serverAssertWithInfo(NULL,key,retval == DICT_OK);
     if (val->type == OBJ_LIST ||
-        val->type == OBJ_ZSET)
+        val->type == OBJ_ZSET ||
+        val->type == OBJ_STREAM)
         signalKeyAsReady(db, key);
     if (server.cluster_enabled) slotToKeyAdd(key);
 }
diff --git a/src/debug.c b/src/debug.c
index 36af35aec..83e5b6197 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -817,6 +817,8 @@ void serverLogObjectDebugInfo(const robj *o) {
         serverLog(LL_WARNING,"Sorted set size: %d", (int) zsetLength(o));
         if (o->encoding == OBJ_ENCODING_SKIPLIST)
             serverLog(LL_WARNING,"Skiplist level: %d", (int) ((const zset*)o->ptr)->zsl->level);
+    } else if (o->type == OBJ_STREAM) {
+        serverLog(LL_WARNING,"Stream size: %d", (int) streamLength(o));
     }
 }
 
diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl
index 6b9a4a9cd..a27e1f582 100644
--- a/tests/unit/type/stream-cgroups.tcl
+++ b/tests/unit/type/stream-cgroups.tcl
@@ -170,6 +170,27 @@ start_server {
         assert_error "*NOGROUP*" {$rd read}
     }
 
+    test {RENAME can unblock XREADGROUP with data} {
+        r del mystream
+        r XGROUP CREATE mystream mygroup $ MKSTREAM
+        set rd [redis_deferring_client]
+        $rd XREADGROUP GROUP mygroup Alice BLOCK 0 STREAMS mystream ">"
+        r XGROUP CREATE mystream2 mygroup $ MKSTREAM
+        r XADD mystream2 100 f1 v1
+        r RENAME mystream2 mystream
+        assert_equal "{mystream {{100-0 {f1 v1}}}}" [$rd read] ;# mystream2 had mygroup before RENAME
+    }
+
+    test {RENAME can unblock XREADGROUP with -NOGROUP} {
+        r del mystream
+        r XGROUP CREATE mystream mygroup $ MKSTREAM
+        set rd [redis_deferring_client]
+        $rd XREADGROUP GROUP mygroup Alice BLOCK 0 STREAMS mystream ">"
+        r XADD mystream2 100 f1 v1
+        r RENAME mystream2 mystream
+        assert_error "*NOGROUP*" {$rd read} ;# mystream2 didn't have mygroup before RENAME
+    }
+
     test {XCLAIM can claim PEL items from another consumer} {
         # Add 3 items into the stream, and create a consumer group
         r del mystream

From 1d13ff0b397797b34973b4c33b0fd60e8ca4bccc Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 31 Mar 2020 17:10:09 +0200
Subject: [PATCH 0227/1098] Minor changes to #7037.

---
 src/networking.c | 17 +++++++++++++----
 src/server.c     |  2 +-
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index fcaa164a9..3c754c376 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -374,10 +374,19 @@ void addReplyErrorLength(client *c, const char *s, size_t len) {
      * they are rare and may hint at errors in a script or a bug in Redis. */
     int ctype = getClientType(c);
     if (ctype == CLIENT_TYPE_MASTER || ctype == CLIENT_TYPE_SLAVE || c->id == CLIENT_ID_AOF) {
-        char* to = c->id == CLIENT_ID_AOF ? "AOF-client" :
-                       ctype == CLIENT_TYPE_MASTER ? "master" : "replica";
-        char* from = c->id == CLIENT_ID_AOF ? "server" :
-                         ctype == CLIENT_TYPE_MASTER ? "replica" : "master";
+        char *to, *from;
+
+        if (c->id == CLIENT_ID_AOF) {
+            to = "AOF-loading-client";
+            from = "server";
+        } else if (ctype == CLIENT_TYPE_MASTER) {
+            to = "master";
+            from = "replica";
+        } else {
+            to = "replica";
+            from = "master";
+        }
+
         char *cmdname = c->lastcmd ? c->lastcmd->name : "<unknown>";
         serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error "
                              "to its %s: '%s' after processing the command "
diff --git a/src/server.c b/src/server.c
index 6f1913e4d..4fdbf30ec 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3976,7 +3976,7 @@ sds genRedisInfoString(const char *section) {
             "client_recent_max_output_buffer:%zu\r\n"
             "blocked_clients:%d\r\n"
             "tracking_clients:%d\r\n"
-            "clients_in_timeout_table:%ld\r\n",
+            "clients_in_timeout_table:%llu\r\n",
             listLength(server.clients)-listLength(server.slaves),
             maxin, maxout,
             server.blocked_clients,

From 38076fd6ba5703e38310275994838b5516b1e042 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 31 Mar 2020 17:41:23 +0200
Subject: [PATCH 0228/1098] cast raxSize() to avoid warning with format spec.

---
 src/server.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index 4fdbf30ec..c89e9c075 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3981,7 +3981,7 @@ sds genRedisInfoString(const char *section) {
             maxin, maxout,
             server.blocked_clients,
             server.tracking_clients,
-            raxSize(server.clients_timeout_table));
+            (unsigned long long) raxSize(server.clients_timeout_table));
     }
 
     /* Memory */

From c4dc5b80b26027cba179c21f5b2be4eb81bf3b5e Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Tue, 21 Jan 2020 15:09:42 +0530
Subject: [PATCH 0229/1098] Fix memory corruption in moduleHandleBlockedClients

By using a "circular BRPOPLPUSH"-like scenario it was
possible the get the same client on db->blocking_keys
twice (See comment in moduleTryServeClientBlockedOnKey)

The fix was actually already implememnted in
moduleTryServeClientBlockedOnKey but it had a bug:
the funxction should return 0 or 1 (not OK or ERR)

Other changes:
1. Added two commands to blockonkeys.c test module (To
   reproduce the case described above)
2. Simplify blockonkeys.c in order to make testing easier
3. cast raxSize() to avoid warning with format spec
---
 src/module.c                         |  18 +++-
 src/server.c                         |   2 +-
 tests/modules/blockonkeys.c          | 122 ++++++++++++++++++++++-----
 tests/unit/moduleapi/blockonkeys.tcl |  55 +++++++-----
 4 files changed, 150 insertions(+), 47 deletions(-)

diff --git a/src/module.c b/src/module.c
index 61dc25169..85e1497fd 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4393,14 +4393,26 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF
  * can really be unblocked, since the module was able to serve the client.
  * If the callback returns REDISMODULE_OK, then the client can be unblocked,
  * otherwise the client remains blocked and we'll retry again when one of
- * the keys it blocked for becomes "ready" again. */
+ * the keys it blocked for becomes "ready" again.
+ * This function returns 1 if client was served (and should be unblocked) */
 int moduleTryServeClientBlockedOnKey(client *c, robj *key) {
     int served = 0;
     RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle;
     /* Protect against re-processing: don't serve clients that are already
      * in the unblocking list for any reason (including RM_UnblockClient()
-     * explicit call). */
-    if (bc->unblocked) return REDISMODULE_ERR;
+     * explicit call).
+     * For example, the following pathological case:
+     * Assume a module called LIST implements the same command as
+     * the Redis list data type.
+     * LIST.BRPOPLPUSH src dst 0 ('src' goes into db->blocking_keys)
+     * LIST.BRPOPLPUSH dst src 0 ('dst' goes into db->blocking_keys)
+     * LIST.LPUSH src foo
+     * 'src' is in db->blocking_keys after the first BRPOPLPUSH is served
+     * (and stays there until the next beforeSleep).
+     * The second BRPOPLPUSH will signal 'src' as ready, leading to the
+     * unblocking of the already unblocked (and worst, freed) reply_client
+     * of the first BRPOPLPUSH. */
+    if (bc->unblocked) return 0;
     RedisModuleCtx ctx = REDISMODULE_CTX_INIT;
     ctx.flags |= REDISMODULE_CTX_BLOCKED_REPLY;
     ctx.blocked_ready_key = key;
diff --git a/src/server.c b/src/server.c
index 4fdbf30ec..5d8b4fde9 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3981,7 +3981,7 @@ sds genRedisInfoString(const char *section) {
             maxin, maxout,
             server.blocked_clients,
             server.tracking_clients,
-            raxSize(server.clients_timeout_table));
+            (unsigned long long)raxSize(server.clients_timeout_table));
     }
 
     /* Memory */
diff --git a/tests/modules/blockonkeys.c b/tests/modules/blockonkeys.c
index 10dc65b1a..94f31d455 100644
--- a/tests/modules/blockonkeys.c
+++ b/tests/modules/blockonkeys.c
@@ -109,41 +109,33 @@ int fsl_push(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
         return RedisModule_ReplyWithError(ctx,"ERR new element has to be greater than the head element");
 
     fsl->list[fsl->length++] = ele;
-
-    if (fsl->length >= 2)
-        RedisModule_SignalKeyAsReady(ctx, argv[1]);
+    RedisModule_SignalKeyAsReady(ctx, argv[1]);
 
     return RedisModule_ReplyWithSimpleString(ctx, "OK");
 }
 
-int bpop2_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+int bpop_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
     RedisModuleString *keyname = RedisModule_GetBlockedClientReadyKey(ctx);
 
     fsl_t *fsl;
-    if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0))
+    if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0) || !fsl)
         return REDISMODULE_ERR;
 
-    if (!fsl || fsl->length < 2)
-        return REDISMODULE_ERR;
-
-    RedisModule_ReplyWithArray(ctx, 2);
-    RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]);
     RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]);
     return REDISMODULE_OK;
 }
 
-int bpop2_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+int bpop_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
     return RedisModule_ReplyWithSimpleString(ctx, "Request timedout");
 }
 
-
-/* FSL.BPOP2 <key> <timeout> - Block clients until list has two or more elements.
+/* FSL.BPOP <key> <timeout> - Block clients until list has two or more elements.
  * When that happens, unblock client and pop the last two elements (from the right). */
-int fsl_bpop2(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+int fsl_bpop(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     if (argc != 3)
         return RedisModule_WrongArity(ctx);
 
@@ -155,13 +147,10 @@ int fsl_bpop2(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     if (!get_fsl(ctx, argv[1], REDISMODULE_READ, 0, &fsl, 1))
         return REDISMODULE_OK;
 
-    if (!fsl || fsl->length < 2) {
-        /* Key is empty or has <2 elements, we must block */
-        RedisModule_BlockClientOnKeys(ctx, bpop2_reply_callback, bpop2_timeout_callback,
+    if (!fsl) {
+        RedisModule_BlockClientOnKeys(ctx, bpop_reply_callback, bpop_timeout_callback,
                                       NULL, timeout, &argv[1], 1, NULL);
     } else {
-        RedisModule_ReplyWithArray(ctx, 2);
-        RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]);
         RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]);
     }
 
@@ -175,10 +164,10 @@ int bpopgt_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int arg
     long long *pgt = RedisModule_GetBlockedClientPrivateData(ctx);
 
     fsl_t *fsl;
-    if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0))
+    if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0) || !fsl)
         return REDISMODULE_ERR;
 
-    if (!fsl || fsl->list[fsl->length-1] <= *pgt)
+    if (fsl->list[fsl->length-1] <= *pgt)
         return REDISMODULE_ERR;
 
     RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]);
@@ -218,7 +207,6 @@ int fsl_bpopgt(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
         /* We use malloc so the tests in blockedonkeys.tcl can check for memory leaks */
         long long *pgt = RedisModule_Alloc(sizeof(long long));
         *pgt = gt;
-        /* Key is empty or has <2 elements, we must block */
         RedisModule_BlockClientOnKeys(ctx, bpopgt_reply_callback, bpopgt_timeout_callback,
                                       bpopgt_free_privdata, timeout, &argv[1], 1, pgt);
     } else {
@@ -228,6 +216,88 @@ int fsl_bpopgt(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     return REDISMODULE_OK;
 }
 
+int bpoppush_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+    RedisModuleString *src_keyname = RedisModule_GetBlockedClientReadyKey(ctx);
+    RedisModuleString *dst_keyname = RedisModule_GetBlockedClientPrivateData(ctx);
+
+    fsl_t *src;
+    if (!get_fsl(ctx, src_keyname, REDISMODULE_READ, 0, &src, 0) || !src)
+        return REDISMODULE_ERR;
+
+    fsl_t *dst;
+    if (!get_fsl(ctx, dst_keyname, REDISMODULE_WRITE, 1, &dst, 0) || !dst)
+        return REDISMODULE_ERR;
+
+    long long ele = src->list[--src->length];
+    dst->list[dst->length++] = ele;
+    RedisModule_SignalKeyAsReady(ctx, dst_keyname);
+    return RedisModule_ReplyWithLongLong(ctx, ele);
+}
+
+int bpoppush_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+    return RedisModule_ReplyWithSimpleString(ctx, "Request timedout");
+}
+
+void bpoppush_free_privdata(RedisModuleCtx *ctx, void *privdata) {
+    RedisModule_FreeString(ctx, privdata);
+}
+
+/* FSL.BPOPPUSH <src> <dst> <timeout> - Block clients until <src> has an element.
+ * When that happens, unblock client, pop the last element from <src> and push it to <dst>
+ * (from the right). */
+int fsl_bpoppush(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 4)
+        return RedisModule_WrongArity(ctx);
+
+    long long timeout;
+    if (RedisModule_StringToLongLong(argv[3],&timeout) != REDISMODULE_OK || timeout < 0)
+        return RedisModule_ReplyWithError(ctx,"ERR invalid timeout");
+
+    fsl_t *src;
+    if (!get_fsl(ctx, argv[1], REDISMODULE_READ, 0, &src, 1))
+        return REDISMODULE_OK;
+
+    if (!src) {
+        /* Retain string for reply callback */
+        RedisModule_RetainString(ctx, argv[2]);
+        /* Key is empty, we must block */
+        RedisModule_BlockClientOnKeys(ctx, bpoppush_reply_callback, bpoppush_timeout_callback,
+                                      bpoppush_free_privdata, timeout, &argv[1], 1, argv[2]);
+    } else {
+        fsl_t *dst;
+        if (!get_fsl(ctx, argv[2], REDISMODULE_WRITE, 1, &dst, 1))
+            return REDISMODULE_OK;
+        long long ele = src->list[--src->length];
+        dst->list[dst->length++] = ele;
+        RedisModule_SignalKeyAsReady(ctx, argv[2]);
+        RedisModule_ReplyWithLongLong(ctx, ele);
+    }
+
+    return REDISMODULE_OK;
+}
+
+/* FSL.GETALL <key> - Reply with an array containing all elements. */
+int fsl_getall(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 2)
+        return RedisModule_WrongArity(ctx);
+
+    fsl_t *fsl;
+    if (!get_fsl(ctx, argv[1], REDISMODULE_READ, 0, &fsl, 1))
+        return REDISMODULE_OK;
+
+    if (!fsl)
+        return RedisModule_ReplyWithArray(ctx, 0);
+
+    RedisModule_ReplyWithArray(ctx, fsl->length);
+    for (int i = 0; i < fsl->length; i++)
+        RedisModule_ReplyWithLongLong(ctx, fsl->list[i]);
+    return REDISMODULE_OK;
+}
+
 int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
@@ -252,11 +322,17 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     if (RedisModule_CreateCommand(ctx,"fsl.push",fsl_push,"",0,0,0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
 
-    if (RedisModule_CreateCommand(ctx,"fsl.bpop2",fsl_bpop2,"",0,0,0) == REDISMODULE_ERR)
+    if (RedisModule_CreateCommand(ctx,"fsl.bpop",fsl_bpop,"",0,0,0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
 
     if (RedisModule_CreateCommand(ctx,"fsl.bpopgt",fsl_bpopgt,"",0,0,0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
 
+    if (RedisModule_CreateCommand(ctx,"fsl.bpoppush",fsl_bpoppush,"",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx,"fsl.getall",fsl_getall,"",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
     return REDISMODULE_OK;
 }
diff --git a/tests/unit/moduleapi/blockonkeys.tcl b/tests/unit/moduleapi/blockonkeys.tcl
index b380227e0..c8b8f23ed 100644
--- a/tests/unit/moduleapi/blockonkeys.tcl
+++ b/tests/unit/moduleapi/blockonkeys.tcl
@@ -3,37 +3,53 @@ set testmodule [file normalize tests/modules/blockonkeys.so]
 start_server {tags {"modules"}} {
     r module load $testmodule
 
+    test "Module client blocked on keys: Circular BPOPPUSH" {
+        set rd1 [redis_deferring_client]
+        set rd2 [redis_deferring_client]
+
+        r del src dst
+
+        $rd1 fsl.bpoppush src dst 0
+        $rd2 fsl.bpoppush dst src 0
+
+        r fsl.push src 42
+
+        assert_equal {42} [r fsl.getall src]
+        assert_equal {} [r fsl.getall dst]
+    }
+
+    test "Module client blocked on keys: Self-referential BPOPPUSH" {
+        set rd1 [redis_deferring_client]
+
+        r del src
+
+        $rd1 fsl.bpoppush src src 0
+
+        r fsl.push src 42
+
+        assert_equal {42} [r fsl.getall src]
+    }
+
     test {Module client blocked on keys (no metadata): No block} {
         r del k
         r fsl.push k 33
         r fsl.push k 34
-        r fsl.bpop2 k 0
-    } {34 33}
+        r fsl.bpop k 0
+    } {34}
 
     test {Module client blocked on keys (no metadata): Timeout} {
         r del k
         set rd [redis_deferring_client]
-        r fsl.push k 33
-        $rd fsl.bpop2 k 1
+        $rd fsl.bpop k 1
         assert_equal {Request timedout} [$rd read]
     }
 
-    test {Module client blocked on keys (no metadata): Blocked, case 1} {
+    test {Module client blocked on keys (no metadata): Blocked} {
         r del k
         set rd [redis_deferring_client]
-        r fsl.push k 33
-        $rd fsl.bpop2 k 0
+        $rd fsl.bpop k 0
         r fsl.push k 34
-        assert_equal {34 33} [$rd read]
-    }
-
-    test {Module client blocked on keys (no metadata): Blocked, case 2} {
-        r del k
-        set rd [redis_deferring_client]
-        r fsl.push k 33
-        r fsl.push k 34
-        $rd fsl.bpop2 k 0
-        assert_equal {34 33} [$rd read]
+        assert_equal {34} [$rd read]
     }
 
     test {Module client blocked on keys (with metadata): No block} {
@@ -108,13 +124,12 @@ start_server {tags {"modules"}} {
     test {Module client blocked on keys does not wake up on wrong type} {
         r del k
         set rd [redis_deferring_client]
-        $rd fsl.bpop2 k 0
+        $rd fsl.bpop k 0
         r lpush k 12
         r lpush k 13
         r lpush k 14
         r del k
-        r fsl.push k 33
         r fsl.push k 34
-        assert_equal {34 33} [$rd read]
+        assert_equal {34} [$rd read]
     }
 }

From 1010c1b43e9af03dbfbf7895f83ab519b75cfe91 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 1 Apr 2020 16:10:18 +0200
Subject: [PATCH 0230/1098] LCS: initial functionality implemented.

---
 src/db.c       |  26 +++++++++++
 src/server.c   |   6 ++-
 src/server.h   |   2 +
 src/t_string.c | 123 +++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 156 insertions(+), 1 deletion(-)

diff --git a/src/db.c b/src/db.c
index 211bb978d..bfa39564e 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1554,6 +1554,32 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk
     return keys;
 }
 
+/* LCS ... [STOREIDX <key>] ... */
+int *lcsGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys)
+{
+    int i;
+    int *keys;
+    UNUSED(cmd);
+
+    /* We need to parse the options of the command in order to check for the
+     * "STOREIDX" argument before the STRINGS argument. */
+    for (i = 1; i < argc; i++) {
+        char *arg = argv[i]->ptr;
+        int moreargs = (argc-1) - i;
+
+        if (!strcasecmp(arg, "strings")) {
+            break;
+        } else if (!strcasecmp(arg, "storeidx") && moreargs) {
+            keys = getKeysTempBuffer;
+            keys[0] = i+1;
+            *numkeys = 1;
+            return keys;
+        }
+    }
+    *numkeys = 0;
+    return NULL;
+}
+
 /* Helper function to extract keys from memory command.
  * MEMORY USAGE <key> */
 int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
diff --git a/src/server.c b/src/server.c
index c89e9c075..d06a9e29a 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1004,7 +1004,11 @@ struct redisCommand redisCommandTable[] = {
 
     {"acl",aclCommand,-2,
      "admin no-script no-slowlog ok-loading ok-stale",
-     0,NULL,0,0,0,0,0,0}
+     0,NULL,0,0,0,0,0,0},
+
+    {"lcs",lcsCommand,-4,
+     "write use-memory @string",
+     0,lcsGetKeys,0,0,0,0,0,0}
 };
 
 /*============================ Utility functions ============================ */
diff --git a/src/server.h b/src/server.h
index c4db4278e..1472bcee7 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2101,6 +2101,7 @@ int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkey
 int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
 int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
 int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
+int *lcsGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
 
 /* Cluster */
 void clusterInit(void);
@@ -2372,6 +2373,7 @@ void xdelCommand(client *c);
 void xtrimCommand(client *c);
 void lolwutCommand(client *c);
 void aclCommand(client *c);
+void lcsCommand(client *c);
 
 #if defined(__GNUC__)
 void *calloc(size_t count, size_t size) __attribute__ ((deprecated));
diff --git a/src/t_string.c b/src/t_string.c
index 8ccd69eb9..e19647845 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -479,3 +479,126 @@ void strlenCommand(client *c) {
         checkType(c,o,OBJ_STRING)) return;
     addReplyLongLong(c,stringObjectLen(o));
 }
+
+/* LCS -- Longest common subsequence.
+ *
+ * LCS [LEN] [IDX] [STOREIDX <key>] STRINGS <string> <string> */
+void lcsCommand(client *c) {
+    uint32_t i, j;
+    sds a = NULL, b = NULL;
+    int getlen = 0, getidx = 0;
+    robj *idxkey = NULL; /* STOREIDX will set this and getidx to 1. */
+
+    for (j = 1; j < (uint32_t)c->argc; j++) {
+        char *opt = c->argv[j]->ptr;
+        int moreargs = (c->argc-1) - j;
+
+        if (!strcasecmp(opt,"IDX")) {
+            getidx = 1;
+        } else if (!strcasecmp(opt,"STOREIDX") && moreargs) {
+            getidx = 1;
+            idxkey = c->argv[j+1];
+            j++;
+        } else if (!strcasecmp(opt,"LEN")) {
+            getlen++;
+        } else if (!strcasecmp(opt,"STRINGS")) {
+            if (moreargs != 2) {
+                addReplyError(c,"LCS requires exactly two strings");
+                return;
+            }
+            a = c->argv[j+1]->ptr;
+            b = c->argv[j+2]->ptr;
+            j += 2;
+        } else {
+            addReply(c,shared.syntaxerr);
+            return;
+        }
+    }
+
+    /* Complain if the user didn't pass the STRING option. */
+    if (a == NULL) {
+        addReplyError(c,"STRINGS <string-a> <string-b> is mandatory");
+        return;
+    }
+
+    /* Compute the LCS using the vanilla dynamic programming technique of
+     * building a table of LCS(x,y) substrings. */
+    uint32_t alen = sdslen(a);
+    uint32_t blen = sdslen(b);
+
+    /* Setup an uint32_t array to store at LCS[i,j] the length of the
+     * LCS A0..i-1, B0..j-1. Note that we have a linear array here, so
+     * we index it as LCS[i+alen*j] */
+    uint32_t *lcs = zmalloc((alen+1)*(blen+1)*sizeof(uint32_t));
+    #define LCS(A,B) lcs[(A)+((B)*(alen+1))]
+
+    /* Start building the LCS table. */
+    for (uint32_t i = 0; i <= alen; i++) {
+        for (uint32_t j = 0; j <= blen; j++) {
+            if (i == 0 || j == 0) {
+                /* If one substring has length of zero, the
+                 * LCS length is zero. */
+                LCS(i,j) = 0;
+            } else if (a[i-1] == b[j-1]) {
+                /* The len LCS (and the LCS itself) of two
+                 * sequences with the same final character, is the
+                 * LCS of the two sequences without the last char
+                 * plus that last char. */
+                LCS(i,j) = LCS(i-1,j-1)+1;
+            } else {
+                /* If the last character is different, take the longest
+                 * between the LCS of the first string and the second
+                 * minus the last char, and the reverse. */
+                uint32_t lcs1 = LCS(i-1,j);
+                uint32_t lcs2 = LCS(i,j-1);
+                LCS(i,j) = lcs1 > lcs2 ? lcs1 : lcs2;
+            }
+        }
+    }
+
+    /* Store the actual LCS string in "result" if needed. We create
+     * it backward, but the length is already known, we store it into idx. */
+    uint32_t idx = LCS(alen,blen);
+    sds result = NULL;
+
+    /* Do we need to compute the actual LCS string? Allocate it in that case. */
+    int computelcs = getidx || !getlen;
+    if (computelcs) result = sdsnewlen(SDS_NOINIT,idx);
+
+    i = alen, j = blen;
+    while (computelcs && i > 0 && j > 0) {
+        if (a[i-1] == b[j-1]) {
+            /* If there is a match, store the character and reduce
+             * the indexes to look for a new match. */
+            result[idx-1] = a[i-1];
+            idx--;
+            i--;
+            j--;
+        } else {
+            /* Otherwise reduce i and j depending on the largest
+             * LCS between, to understand what direction we need to go. */
+            uint32_t lcs1 = LCS(i-1,j);
+            uint32_t lcs2 = LCS(i,j-1);
+            if (lcs1 > lcs2)
+                i--;
+            else
+                j--;
+        }
+    }
+
+    /* Signal modified key, increment dirty, ... */
+
+    /* Reply depending on the given options. */
+    if (getlen) {
+        addReplyLongLong(c,LCS(alen,blen));
+    } else {
+        addReplyBulkSds(c,result);
+        result = NULL;
+    }
+
+    /* Cleanup. */
+    sdsfree(result);
+    zfree(lcs);
+    return;
+}
+

From b3400559be53ff77f7196c99d791d62d298875e9 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 1 Apr 2020 17:11:31 +0200
Subject: [PATCH 0231/1098] LCS: implement range indexes option.

---
 src/t_string.c | 68 +++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 59 insertions(+), 9 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index e19647845..6c096a539 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -482,7 +482,7 @@ void strlenCommand(client *c) {
 
 /* LCS -- Longest common subsequence.
  *
- * LCS [LEN] [IDX] [STOREIDX <key>] STRINGS <string> <string> */
+ * LCS [IDX] [STOREIDX <key>] STRINGS <string> <string> */
 void lcsCommand(client *c) {
     uint32_t i, j;
     sds a = NULL, b = NULL;
@@ -495,12 +495,12 @@ void lcsCommand(client *c) {
 
         if (!strcasecmp(opt,"IDX")) {
             getidx = 1;
+        } else if (!strcasecmp(opt,"LEN")) {
+            getlen = 1;
         } else if (!strcasecmp(opt,"STOREIDX") && moreargs) {
             getidx = 1;
             idxkey = c->argv[j+1];
             j++;
-        } else if (!strcasecmp(opt,"LEN")) {
-            getlen++;
         } else if (!strcasecmp(opt,"STRINGS")) {
             if (moreargs != 2) {
                 addReplyError(c,"LCS requires exactly two strings");
@@ -515,10 +515,17 @@ void lcsCommand(client *c) {
         }
     }
 
-    /* Complain if the user didn't pass the STRING option. */
+    /* Complain if the user passed ambiguous parameters. */
     if (a == NULL) {
         addReplyError(c,"STRINGS <string-a> <string-b> is mandatory");
         return;
+    } else if (getlen && (getidx && idxkey == NULL)) {
+        addReplyError(c,
+            "If you want both the LEN and indexes, please "
+            "store the indexes into a key with STOREIDX. However note "
+            "that the IDX output also includes both the LCS string and "
+            "its length");
+        return;
     }
 
     /* Compute the LCS using the vanilla dynamic programming technique of
@@ -559,21 +566,62 @@ void lcsCommand(client *c) {
     /* Store the actual LCS string in "result" if needed. We create
      * it backward, but the length is already known, we store it into idx. */
     uint32_t idx = LCS(alen,blen);
-    sds result = NULL;
+    sds result = NULL;        /* Resulting LCS string. */
+    void *arraylenptr = NULL; /* Deffered length of the array for IDX. */
+    uint32_t arange_start = alen, /* alen signals that values are not set. */
+             arange_end = 0,
+             brange_start = 0,
+             brange_end = 0;
 
     /* Do we need to compute the actual LCS string? Allocate it in that case. */
     int computelcs = getidx || !getlen;
     if (computelcs) result = sdsnewlen(SDS_NOINIT,idx);
 
+    /* Start with a deferred array if we have to emit the ranges. */
+    uint32_t arraylen = 0;  /* Number of ranges emitted in the array. */
+    if (getidx && idxkey == NULL)
+        arraylenptr = addReplyDeferredLen(c);
+
     i = alen, j = blen;
     while (computelcs && i > 0 && j > 0) {
         if (a[i-1] == b[j-1]) {
             /* If there is a match, store the character and reduce
              * the indexes to look for a new match. */
             result[idx-1] = a[i-1];
-            idx--;
-            i--;
-            j--;
+            /* Track the current range. */
+            int emit_range = 0;
+            if (arange_start == alen) {
+                arange_start = i-1;
+                arange_end = i-1;
+                brange_start = j-1;
+                brange_end = j-1;
+                if (i == 0 || j == 0) emit_range = 1;
+            } else {
+                /* Let's see if we can extend the range backward since
+                 * it is contiguous. */
+                if (arange_start == i && brange_start == j) {
+                    arange_start--;
+                    brange_start--;
+                } else {
+                    emit_range = 1;
+                }
+            }
+
+            /* Emit the current range if needed. */
+            if (emit_range) {
+                if (arraylenptr) {
+                    addReplyArrayLen(c,2);
+                    addReplyArrayLen(c,2);
+                    addReplyLongLong(c,arange_start);
+                    addReplyLongLong(c,arange_end);
+                    addReplyArrayLen(c,2);
+                    addReplyLongLong(c,brange_start);
+                    addReplyLongLong(c,brange_end);
+                }
+                arange_start = alen; /* Restart at the next match. */
+                arraylen++;
+            }
+            idx--; i--; j--;
         } else {
             /* Otherwise reduce i and j depending on the largest
              * LCS between, to understand what direction we need to go. */
@@ -589,7 +637,9 @@ void lcsCommand(client *c) {
     /* Signal modified key, increment dirty, ... */
 
     /* Reply depending on the given options. */
-    if (getlen) {
+    if (arraylenptr) {
+        setDeferredArrayLen(c,arraylenptr,arraylen);
+    } else if (getlen) {
         addReplyLongLong(c,LCS(alen,blen));
     } else {
         addReplyBulkSds(c,result);

From c9c03c3ee60b6a7a4918d4b2bbf40cfd21fcd284 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 1 Apr 2020 17:14:36 +0200
Subject: [PATCH 0232/1098] LCS: fix emission of last range starting at index
 0.

---
 src/t_string.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_string.c b/src/t_string.c
index 6c096a539..caccd11f1 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -595,7 +595,6 @@ void lcsCommand(client *c) {
                 arange_end = i-1;
                 brange_start = j-1;
                 brange_end = j-1;
-                if (i == 0 || j == 0) emit_range = 1;
             } else {
                 /* Let's see if we can extend the range backward since
                  * it is contiguous. */
@@ -606,6 +605,7 @@ void lcsCommand(client *c) {
                     emit_range = 1;
                 }
             }
+            if (arange_start == 0 || brange_start == 0) emit_range = 1;
 
             /* Emit the current range if needed. */
             if (emit_range) {

From 4cbf3f5dddc7c765269d8ce9eceb406ccde036d6 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 1 Apr 2020 17:36:32 +0200
Subject: [PATCH 0233/1098] LCS: other fixes to range emission.

---
 src/t_string.c | 36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index caccd11f1..30999ee70 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -584,12 +584,13 @@ void lcsCommand(client *c) {
 
     i = alen, j = blen;
     while (computelcs && i > 0 && j > 0) {
+        int emit_range = 0;
         if (a[i-1] == b[j-1]) {
             /* If there is a match, store the character and reduce
              * the indexes to look for a new match. */
             result[idx-1] = a[i-1];
+
             /* Track the current range. */
-            int emit_range = 0;
             if (arange_start == alen) {
                 arange_start = i-1;
                 arange_end = i-1;
@@ -605,22 +606,9 @@ void lcsCommand(client *c) {
                     emit_range = 1;
                 }
             }
+            /* Emit the range if we matched with the first byte of
+             * one of the two strings. We'll exit the loop ASAP. */
             if (arange_start == 0 || brange_start == 0) emit_range = 1;
-
-            /* Emit the current range if needed. */
-            if (emit_range) {
-                if (arraylenptr) {
-                    addReplyArrayLen(c,2);
-                    addReplyArrayLen(c,2);
-                    addReplyLongLong(c,arange_start);
-                    addReplyLongLong(c,arange_end);
-                    addReplyArrayLen(c,2);
-                    addReplyLongLong(c,brange_start);
-                    addReplyLongLong(c,brange_end);
-                }
-                arange_start = alen; /* Restart at the next match. */
-                arraylen++;
-            }
             idx--; i--; j--;
         } else {
             /* Otherwise reduce i and j depending on the largest
@@ -631,6 +619,22 @@ void lcsCommand(client *c) {
                 i--;
             else
                 j--;
+            if (arange_start != alen) emit_range = 1;
+        }
+
+        /* Emit the current range if needed. */
+        if (emit_range) {
+            if (arraylenptr) {
+                addReplyArrayLen(c,2);
+                addReplyArrayLen(c,2);
+                addReplyLongLong(c,arange_start);
+                addReplyLongLong(c,arange_end);
+                addReplyArrayLen(c,2);
+                addReplyLongLong(c,brange_start);
+                addReplyLongLong(c,brange_end);
+            }
+            arange_start = alen; /* Restart at the next match. */
+            arraylen++;
         }
     }
 

From 8cdc15c3093a14f4a9af45cfae5679c67eda3fa0 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 1 Apr 2020 22:11:59 +0200
Subject: [PATCH 0234/1098] LCS: implement KEYS option.

---
 src/t_string.c | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index 30999ee70..0b159e6b3 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -482,12 +482,13 @@ void strlenCommand(client *c) {
 
 /* LCS -- Longest common subsequence.
  *
- * LCS [IDX] [STOREIDX <key>] STRINGS <string> <string> */
+ * LCS [IDX] [STOREIDX <key>] STRINGS <string> <string> | KEYS <keya> <keyb> */
 void lcsCommand(client *c) {
     uint32_t i, j;
     sds a = NULL, b = NULL;
     int getlen = 0, getidx = 0;
     robj *idxkey = NULL; /* STOREIDX will set this and getidx to 1. */
+    robj *obja = NULL, *objb = NULL;
 
     for (j = 1; j < (uint32_t)c->argc; j++) {
         char *opt = c->argv[j]->ptr;
@@ -509,6 +510,18 @@ void lcsCommand(client *c) {
             a = c->argv[j+1]->ptr;
             b = c->argv[j+2]->ptr;
             j += 2;
+        } else if (!strcasecmp(opt,"KEYS")) {
+            if (moreargs != 2) {
+                addReplyError(c,"LCS requires exactly two keys");
+                return;
+            }
+            obja = lookupKeyRead(c->db,c->argv[j+1]);
+            objb = lookupKeyRead(c->db,c->argv[j+2]);
+            obja = obja ? getDecodedObject(obja) : createStringObject("",0);
+            objb = objb ? getDecodedObject(objb) : createStringObject("",0);
+            a = obja->ptr;
+            b = objb->ptr;
+            j += 2;
         } else {
             addReply(c,shared.syntaxerr);
             return;
@@ -517,7 +530,8 @@ void lcsCommand(client *c) {
 
     /* Complain if the user passed ambiguous parameters. */
     if (a == NULL) {
-        addReplyError(c,"STRINGS <string-a> <string-b> is mandatory");
+        addReplyError(c,"Please specify two strings: "
+                        "STRINGS or KEYS options are mandatory");
         return;
     } else if (getlen && (getidx && idxkey == NULL)) {
         addReplyError(c,
@@ -651,6 +665,8 @@ void lcsCommand(client *c) {
     }
 
     /* Cleanup. */
+    if (obja) decrRefCount(obja);
+    if (objb) decrRefCount(objb);
     sdsfree(result);
     zfree(lcs);
     return;

From 88e66ecf946f720db76b06dfb1a27834d16b7d61 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 1 Apr 2020 23:45:07 +0200
Subject: [PATCH 0235/1098] LCS: 7x speedup by accessing the array with better
 locality.

---
 src/t_string.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_string.c b/src/t_string.c
index 0b159e6b3..b715e144a 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -551,7 +551,7 @@ void lcsCommand(client *c) {
      * LCS A0..i-1, B0..j-1. Note that we have a linear array here, so
      * we index it as LCS[i+alen*j] */
     uint32_t *lcs = zmalloc((alen+1)*(blen+1)*sizeof(uint32_t));
-    #define LCS(A,B) lcs[(A)+((B)*(alen+1))]
+    #define LCS(A,B) lcs[(B)+((A)*(blen+1))]
 
     /* Start building the LCS table. */
     for (uint32_t i = 0; i <= alen; i++) {

From 80ec0431e86defaef1c78bc4fac52688187546e5 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 2 Apr 2020 11:20:09 +0200
Subject: [PATCH 0236/1098] Simplify comment in
 moduleTryServeClientBlockedOnKey().

---
 src/module.c | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/src/module.c b/src/module.c
index 85e1497fd..16d24152e 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4398,21 +4398,12 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF
 int moduleTryServeClientBlockedOnKey(client *c, robj *key) {
     int served = 0;
     RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle;
+
     /* Protect against re-processing: don't serve clients that are already
      * in the unblocking list for any reason (including RM_UnblockClient()
-     * explicit call).
-     * For example, the following pathological case:
-     * Assume a module called LIST implements the same command as
-     * the Redis list data type.
-     * LIST.BRPOPLPUSH src dst 0 ('src' goes into db->blocking_keys)
-     * LIST.BRPOPLPUSH dst src 0 ('dst' goes into db->blocking_keys)
-     * LIST.LPUSH src foo
-     * 'src' is in db->blocking_keys after the first BRPOPLPUSH is served
-     * (and stays there until the next beforeSleep).
-     * The second BRPOPLPUSH will signal 'src' as ready, leading to the
-     * unblocking of the already unblocked (and worst, freed) reply_client
-     * of the first BRPOPLPUSH. */
+     * explicit call). See #6798. */
     if (bc->unblocked) return 0;
+
     RedisModuleCtx ctx = REDISMODULE_CTX_INIT;
     ctx.flags |= REDISMODULE_CTX_BLOCKED_REPLY;
     ctx.blocked_ready_key = key;

From 3f96e1623d5270714dc94c75ca8d5daab336912c Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 2 Apr 2020 13:37:35 +0200
Subject: [PATCH 0237/1098] LCS: MINMATCHLEN and WITHMATCHLEN options.

---
 src/t_string.c | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index b715e144a..6b30c9751 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -482,11 +482,13 @@ void strlenCommand(client *c) {
 
 /* LCS -- Longest common subsequence.
  *
- * LCS [IDX] [STOREIDX <key>] STRINGS <string> <string> | KEYS <keya> <keyb> */
+ * LCS [IDX] [STOREIDX <key>] [MINMATCHLEN <len>]
+ *     STRINGS <string> <string> | KEYS <keya> <keyb> */
 void lcsCommand(client *c) {
     uint32_t i, j;
+    long long minmatchlen = 0;
     sds a = NULL, b = NULL;
-    int getlen = 0, getidx = 0;
+    int getlen = 0, getidx = 0, withmatchlen = 0;
     robj *idxkey = NULL; /* STOREIDX will set this and getidx to 1. */
     robj *obja = NULL, *objb = NULL;
 
@@ -498,10 +500,17 @@ void lcsCommand(client *c) {
             getidx = 1;
         } else if (!strcasecmp(opt,"LEN")) {
             getlen = 1;
+        } else if (!strcasecmp(opt,"WITHMATCHLEN")) {
+            withmatchlen = 1;
         } else if (!strcasecmp(opt,"STOREIDX") && moreargs) {
             getidx = 1;
             idxkey = c->argv[j+1];
             j++;
+        } else if (!strcasecmp(opt,"MINMATCHLEN") && moreargs) {
+            if (getLongLongFromObjectOrReply(c,c->argv[j+1],&minmatchlen,NULL)
+                != C_OK) return;
+            if (minmatchlen < 0) minmatchlen = 0;
+            j++;
         } else if (!strcasecmp(opt,"STRINGS")) {
             if (moreargs != 2) {
                 addReplyError(c,"LCS requires exactly two strings");
@@ -637,18 +646,22 @@ void lcsCommand(client *c) {
         }
 
         /* Emit the current range if needed. */
+        uint32_t match_len = arange_end - arange_start + 1;
         if (emit_range) {
-            if (arraylenptr) {
-                addReplyArrayLen(c,2);
-                addReplyArrayLen(c,2);
-                addReplyLongLong(c,arange_start);
-                addReplyLongLong(c,arange_end);
-                addReplyArrayLen(c,2);
-                addReplyLongLong(c,brange_start);
-                addReplyLongLong(c,brange_end);
+            if (minmatchlen == 0 || match_len >= minmatchlen) {
+                if (arraylenptr) {
+                    addReplyArrayLen(c,2+withmatchlen);
+                    addReplyArrayLen(c,2);
+                    addReplyLongLong(c,arange_start);
+                    addReplyLongLong(c,arange_end);
+                    addReplyArrayLen(c,2);
+                    addReplyLongLong(c,brange_start);
+                    addReplyLongLong(c,brange_end);
+                    if (withmatchlen) addReplyLongLong(c,match_len);
+                    arraylen++;
+                }
             }
             arange_start = alen; /* Restart at the next match. */
-            arraylen++;
         }
     }
 

From 5a3d85745ad5abe8b7e9ae11c062a0a1b4220387 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 2 Apr 2020 16:15:17 +0200
Subject: [PATCH 0238/1098] LCS: output LCS len as well in IDX mode.

---
 src/t_string.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/t_string.c b/src/t_string.c
index 6b30c9751..fb53f7c54 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -602,8 +602,11 @@ void lcsCommand(client *c) {
 
     /* Start with a deferred array if we have to emit the ranges. */
     uint32_t arraylen = 0;  /* Number of ranges emitted in the array. */
-    if (getidx && idxkey == NULL)
+    if (getidx && idxkey == NULL) {
+        addReplyMapLen(c,2);
+        addReplyBulkCString(c,"matches");
         arraylenptr = addReplyDeferredLen(c);
+    }
 
     i = alen, j = blen;
     while (computelcs && i > 0 && j > 0) {
@@ -669,6 +672,8 @@ void lcsCommand(client *c) {
 
     /* Reply depending on the given options. */
     if (arraylenptr) {
+        addReplyBulkCString(c,"len");
+        addReplyLongLong(c,LCS(alen,blen));
         setDeferredArrayLen(c,arraylenptr,arraylen);
     } else if (getlen) {
         addReplyLongLong(c,LCS(alen,blen));

From 85a0d29d04ac280151e3fed28ab4767fbcc7463b Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Tue, 25 Feb 2020 16:51:35 +0530
Subject: [PATCH 0239/1098] Stale replica should allow MULTI/EXEC

Example: Client uses a pipe to send the following to a
stale replica:

MULTI
.. do something ...
DISCARD

The replica will reply the MUTLI with -MASTERDOWN and
execute the rest of the commands... A client using a
pipe might not be aware that MULTI failed until it's
too late.

I can't think of a reason why MULTI/EXEC/DISCARD should
not be executed on stale replicas...

Also, enable MULTI/EXEC/DISCARD during loading
---
 src/server.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/server.c b/src/server.c
index c89e9c075..ff7a39df5 100644
--- a/src/server.c
+++ b/src/server.c
@@ -672,15 +672,15 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,1,1,1,0,0,0},
 
     {"multi",multiCommand,1,
-     "no-script fast @transaction",
+     "no-script fast ok-loading ok-stale @transaction",
      0,NULL,0,0,0,0,0,0},
 
     {"exec",execCommand,1,
-     "no-script no-monitor no-slowlog @transaction",
+     "no-script no-monitor no-slowlog ok-loading ok-stale @transaction",
      0,NULL,0,0,0,0,0,0},
 
     {"discard",discardCommand,1,
-     "no-script fast @transaction",
+     "no-script fast ok-loading ok-stale @transaction",
      0,NULL,0,0,0,0,0,0},
 
     {"sync",syncCommand,1,

From 4665b3ebfb01e46b746ab75f720dbbcf4de28784 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 2 Apr 2020 18:41:29 +0300
Subject: [PATCH 0240/1098] Fix no-negative-zero test

---
 tests/unit/type/incr.tcl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/unit/type/incr.tcl b/tests/unit/type/incr.tcl
index 63bf2e116..b7a135203 100644
--- a/tests/unit/type/incr.tcl
+++ b/tests/unit/type/incr.tcl
@@ -153,6 +153,7 @@ start_server {tags {"incr"}} {
     } {ERR*valid*}
 
     test {No negative zero} {
+        r del foo
         r incrbyfloat foo [expr double(1)/41]
         r incrbyfloat foo [expr double(-1)/41]
         r get foo

From 78df016584d2d7ab2467e6fb4ac45797972bc80a Mon Sep 17 00:00:00 2001
From: Xudong Zhang <zxdvd_zhang@163.com>
Date: Thu, 2 Apr 2020 23:43:19 +0800
Subject: [PATCH 0241/1098] fix integer overflow

---
 src/quicklist.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/quicklist.c b/src/quicklist.c
index ae183ffd8..52e3988f5 100644
--- a/src/quicklist.c
+++ b/src/quicklist.c
@@ -110,7 +110,7 @@ quicklist *quicklistCreate(void) {
     return quicklist;
 }
 
-#define COMPRESS_MAX (1 << QL_COMP_BITS)
+#define COMPRESS_MAX ((1 << QL_COMP_BITS)-1)
 void quicklistSetCompressDepth(quicklist *quicklist, int compress) {
     if (compress > COMPRESS_MAX) {
         compress = COMPRESS_MAX;
@@ -120,7 +120,7 @@ void quicklistSetCompressDepth(quicklist *quicklist, int compress) {
     quicklist->compress = compress;
 }
 
-#define FILL_MAX (1 << (QL_FILL_BITS-1))
+#define FILL_MAX ((1 << (QL_FILL_BITS-1))-1)
 void quicklistSetFill(quicklist *quicklist, int fill) {
     if (fill > FILL_MAX) {
         fill = FILL_MAX;

From ef610802c7262f60c3418ba7e19524636a5c0c8e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 2 Apr 2020 21:17:31 +0200
Subject: [PATCH 0242/1098] LCS: fix stale comment.

---
 src/t_string.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_string.c b/src/t_string.c
index fb53f7c54..80cdbb5e1 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -558,7 +558,7 @@ void lcsCommand(client *c) {
 
     /* Setup an uint32_t array to store at LCS[i,j] the length of the
      * LCS A0..i-1, B0..j-1. Note that we have a linear array here, so
-     * we index it as LCS[i+alen*j] */
+     * we index it as LCS[j+(blen+1)*j] */
     uint32_t *lcs = zmalloc((alen+1)*(blen+1)*sizeof(uint32_t));
     #define LCS(A,B) lcs[(B)+((A)*(blen+1))]
 

From a8857320a5f0a655d7c84939040283176144a5fd Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 6 Feb 2020 15:13:52 +0530
Subject: [PATCH 0243/1098] Modules: Perform printf-like format checks in
 variadic API

---
 src/redismodule.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/redismodule.h b/src/redismodule.h
index d26c41456..754ee4fc6 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -470,7 +470,7 @@ RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongLong)(Re
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromDouble)(RedisModuleCtx *ctx, double d);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongDouble)(RedisModuleCtx *ctx, long double ld, int humanfriendly);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromString)(RedisModuleCtx *ctx, const RedisModuleString *str);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...);
+RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...); __attribute__ ((format (printf, 2, 3)));
 void REDISMODULE_API_FUNC(RedisModule_FreeString)(RedisModuleCtx *ctx, RedisModuleString *str);
 const char *REDISMODULE_API_FUNC(RedisModule_StringPtrLen)(const RedisModuleString *str, size_t *len);
 int REDISMODULE_API_FUNC(RedisModule_ReplyWithError)(RedisModuleCtx *ctx, const char *err);
@@ -554,8 +554,8 @@ void REDISMODULE_API_FUNC(RedisModule_SaveLongDouble)(RedisModuleIO *io, long do
 long double REDISMODULE_API_FUNC(RedisModule_LoadLongDouble)(RedisModuleIO *io);
 void *REDISMODULE_API_FUNC(RedisModule_LoadDataTypeFromString)(const RedisModuleString *str, const RedisModuleType *mt);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_SaveDataTypeToString)(RedisModuleCtx *ctx, void *data, const RedisModuleType *mt);
-void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...);
-void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...);
+void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...); __attribute__ ((format (printf, 3, 4)));
+void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...); __attribute__ ((format (printf, 3, 4)));
 void REDISMODULE_API_FUNC(RedisModule__Assert)(const char *estr, const char *file, int line);
 void REDISMODULE_API_FUNC(RedisModule_LatencyAddSample)(const char *event, mstime_t latency);
 int REDISMODULE_API_FUNC(RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len);

From a76a961cdd6c9df9e08548eee5378b34a331558c Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Fri, 3 Apr 2020 14:49:40 +0300
Subject: [PATCH 0244/1098] Use __attribute__ only if __GNUC__ is defined

---
 src/redismodule.h | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/redismodule.h b/src/redismodule.h
index 754ee4fc6..23b4d26e0 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -470,7 +470,11 @@ RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongLong)(Re
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromDouble)(RedisModuleCtx *ctx, double d);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongDouble)(RedisModuleCtx *ctx, long double ld, int humanfriendly);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromString)(RedisModuleCtx *ctx, const RedisModuleString *str);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...); __attribute__ ((format (printf, 2, 3)));
+#ifdef __GNUC__
+RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...) __attribute__ ((format (printf, 2, 3)));
+#else
+RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...);
+#endif
 void REDISMODULE_API_FUNC(RedisModule_FreeString)(RedisModuleCtx *ctx, RedisModuleString *str);
 const char *REDISMODULE_API_FUNC(RedisModule_StringPtrLen)(const RedisModuleString *str, size_t *len);
 int REDISMODULE_API_FUNC(RedisModule_ReplyWithError)(RedisModuleCtx *ctx, const char *err);
@@ -554,8 +558,13 @@ void REDISMODULE_API_FUNC(RedisModule_SaveLongDouble)(RedisModuleIO *io, long do
 long double REDISMODULE_API_FUNC(RedisModule_LoadLongDouble)(RedisModuleIO *io);
 void *REDISMODULE_API_FUNC(RedisModule_LoadDataTypeFromString)(const RedisModuleString *str, const RedisModuleType *mt);
 RedisModuleString *REDISMODULE_API_FUNC(RedisModule_SaveDataTypeToString)(RedisModuleCtx *ctx, void *data, const RedisModuleType *mt);
-void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...); __attribute__ ((format (printf, 3, 4)));
-void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...); __attribute__ ((format (printf, 3, 4)));
+#ifdef __GNUC__
+void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...) __attribute__ ((format (printf, 3, 4)));
+void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...) __attribute__ ((format (printf, 3, 4)));
+#else
+void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...);
+void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...);
+#endif
 void REDISMODULE_API_FUNC(RedisModule__Assert)(const char *estr, const char *file, int line);
 void REDISMODULE_API_FUNC(RedisModule_LatencyAddSample)(const char *event, mstime_t latency);
 int REDISMODULE_API_FUNC(RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len);

From 1b0d30aeb7bd6d105bc8ab51c6fbe661ff6f1896 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Thu, 2 Apr 2020 14:57:17 +0300
Subject: [PATCH 0245/1098] Try to fix time-sensitive tests in blockonkey.tcl

There is an inherent race between the deferring client and the
"main" client of the test: While the deferring client issues a blocking
command, we can't know for sure that by the time the "main" client
tries to issue another command (Usually one that unblocks the deferring
client) the deferring client is even blocked...
For lack of a better choice this commit uses TCL's 'after' in order
to give some time for the deferring client to issues its blocking
command before the "main" client does its thing.
This problem probably exists in many other tests but this commit
tries to fix blockonkeys.tcl
---
 tests/unit/moduleapi/blockonkeys.tcl | 55 +++++++++++++++++++++++++++-
 1 file changed, 54 insertions(+), 1 deletion(-)

diff --git a/tests/unit/moduleapi/blockonkeys.tcl b/tests/unit/moduleapi/blockonkeys.tcl
index c8b8f23ed..5e5d93da3 100644
--- a/tests/unit/moduleapi/blockonkeys.tcl
+++ b/tests/unit/moduleapi/blockonkeys.tcl
@@ -11,6 +11,12 @@ start_server {tags {"modules"}} {
 
         $rd1 fsl.bpoppush src dst 0
         $rd2 fsl.bpoppush dst src 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {2}
+        } else {
+            fail "Clients are not blocked"
+        }
 
         r fsl.push src 42
 
@@ -24,7 +30,12 @@ start_server {tags {"modules"}} {
         r del src
 
         $rd1 fsl.bpoppush src src 0
-
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r fsl.push src 42
 
         assert_equal {42} [r fsl.getall src]
@@ -48,6 +59,12 @@ start_server {tags {"modules"}} {
         r del k
         set rd [redis_deferring_client]
         $rd fsl.bpop k 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r fsl.push k 34
         assert_equal {34} [$rd read]
     }
@@ -76,6 +93,12 @@ start_server {tags {"modules"}} {
         set cid [$rd read]
         r fsl.push k 33
         $rd fsl.bpopgt k 33 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r fsl.push k 34
         assert_equal {34} [$rd read]
         r client kill id $cid ;# try to smoke-out client-related memory leak
@@ -85,6 +108,12 @@ start_server {tags {"modules"}} {
         r del k
         set rd [redis_deferring_client]
         $rd fsl.bpopgt k 35 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r fsl.push k 33
         r fsl.push k 34
         r fsl.push k 35
@@ -98,6 +127,12 @@ start_server {tags {"modules"}} {
         $rd client id
         set cid [$rd read]
         $rd fsl.bpopgt k 35 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r client kill id $cid ;# try to smoke-out client-related memory leak
     }
 
@@ -107,6 +142,12 @@ start_server {tags {"modules"}} {
         $rd client id
         set cid [$rd read]
         $rd fsl.bpopgt k 35 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r client unblock $cid timeout ;# try to smoke-out client-related memory leak
         assert_equal {Request timedout} [$rd read]
     }
@@ -117,6 +158,12 @@ start_server {tags {"modules"}} {
         $rd client id
         set cid [$rd read]
         $rd fsl.bpopgt k 35 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r client unblock $cid error ;# try to smoke-out client-related memory leak
         assert_error "*unblocked*" {$rd read}
     }
@@ -125,6 +172,12 @@ start_server {tags {"modules"}} {
         r del k
         set rd [redis_deferring_client]
         $rd fsl.bpop k 0
+        ;# wait until clients are actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
         r lpush k 12
         r lpush k 13
         r lpush k 14

From cf3789f0455a867915740a9b6f8d682369dcbf90 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 6 Apr 2020 09:41:14 +0300
Subject: [PATCH 0246/1098] diffrent fix for runtest --host --port

---
 tests/support/server.tcl |  7 ++-----
 tests/test_helper.tcl    | 19 +++++++++++--------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index 400017c5f..d086366dc 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -159,12 +159,9 @@ proc start_server {options {code undefined}} {
     if {$::external} {
         if {[llength $::servers] == 0} {
             set srv {}
-            # In test_server_main(tests/test_helper.tcl:215~218), increase the value of start_port
-            # and assign it to ::port through the `--port` option, so we need to reduce it.
-            set baseport [expr {$::port-100}]
             dict set srv "host" $::host
-            dict set srv "port" $baseport
-            set client [redis $::host $baseport 0 $::tls]
+            dict set srv "port" $::port
+            set client [redis $::host $::port 0 $::tls]
             dict set srv "client" $client
             $client select 9
 
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 5cb43104b..d80cb6907 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -212,13 +212,19 @@ proc test_server_main {} {
 
     # Start the client instances
     set ::clients_pids {}
-    set start_port [expr {$::port+100}]
-    for {set j 0} {$j < $::numclients} {incr j} {
-        set start_port [find_available_port $start_port]
+    if {$::external} {
         set p [exec $tclsh [info script] {*}$::argv \
-            --client $port --port $start_port &]
+            --client $port --port $::port &]
         lappend ::clients_pids $p
-        incr start_port 10
+    } else {
+        set start_port [expr {$::port+100}]
+        for {set j 0} {$j < $::numclients} {incr j} {
+            set start_port [find_available_port $start_port]
+            set p [exec $tclsh [info script] {*}$::argv \
+                --client $port --port $start_port &]
+            lappend ::clients_pids $p
+            incr start_port 10
+        }
     }
 
     # Setup global state for the test server
@@ -506,9 +512,6 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
     } elseif {$opt eq {--host}} {
         set ::external 1
         set ::host $arg
-        # If we use an external server, we can only set numclients to 1,
-        # otherwise the port will be miscalculated.
-        set ::numclients 1
         incr j
     } elseif {$opt eq {--port}} {
         set ::port $arg

From 25063f75d73a4c40b9cd7b0229fc63d6cec15e9a Mon Sep 17 00:00:00 2001
From: mymilkbottles <javaxz@163.com>
Date: Mon, 6 Apr 2020 19:27:06 +0800
Subject: [PATCH 0247/1098] Judge the log level in advance

---
 src/scripting.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/scripting.c b/src/scripting.c
index 32a511e13..bccbcf637 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -959,6 +959,7 @@ int luaLogCommand(lua_State *lua) {
         lua_pushstring(lua, "Invalid debug level.");
         return lua_error(lua);
     }
+    if (level < server.verbosity) return 0;
 
     /* Glue together all the arguments */
     log = sdsempty();

From 7261a5550fc760bf2cfe7365ecb793e728bb6950 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 6 Apr 2020 13:23:07 +0200
Subject: [PATCH 0248/1098] LCS: get rid of STOREIDX option. Fix get keys
 helper.

---
 src/db.c       | 14 +++++++-------
 src/t_string.c | 27 +++++++++++++--------------
 2 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/src/db.c b/src/db.c
index bfa39564e..e9e10aeeb 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1554,30 +1554,30 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk
     return keys;
 }
 
-/* LCS ... [STOREIDX <key>] ... */
+/* LCS ... [KEYS <key1> <key2>] ... */
 int *lcsGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys)
 {
     int i;
-    int *keys;
+    int *keys = getKeysTempBuffer;
     UNUSED(cmd);
 
     /* We need to parse the options of the command in order to check for the
-     * "STOREIDX" argument before the STRINGS argument. */
+     * "KEYS" argument before the "STRINGS" argument. */
     for (i = 1; i < argc; i++) {
         char *arg = argv[i]->ptr;
         int moreargs = (argc-1) - i;
 
         if (!strcasecmp(arg, "strings")) {
             break;
-        } else if (!strcasecmp(arg, "storeidx") && moreargs) {
-            keys = getKeysTempBuffer;
+        } else if (!strcasecmp(arg, "keys") && moreargs >= 2) {
             keys[0] = i+1;
-            *numkeys = 1;
+            keys[1] = i+2;
+            *numkeys = 2;
             return keys;
         }
     }
     *numkeys = 0;
-    return NULL;
+    return keys;
 }
 
 /* Helper function to extract keys from memory command.
diff --git a/src/t_string.c b/src/t_string.c
index 80cdbb5e1..deba22253 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -482,14 +482,13 @@ void strlenCommand(client *c) {
 
 /* LCS -- Longest common subsequence.
  *
- * LCS [IDX] [STOREIDX <key>] [MINMATCHLEN <len>]
+ * LCS [IDX] [MINMATCHLEN <len>]
  *     STRINGS <string> <string> | KEYS <keya> <keyb> */
 void lcsCommand(client *c) {
     uint32_t i, j;
     long long minmatchlen = 0;
     sds a = NULL, b = NULL;
     int getlen = 0, getidx = 0, withmatchlen = 0;
-    robj *idxkey = NULL; /* STOREIDX will set this and getidx to 1. */
     robj *obja = NULL, *objb = NULL;
 
     for (j = 1; j < (uint32_t)c->argc; j++) {
@@ -502,17 +501,16 @@ void lcsCommand(client *c) {
             getlen = 1;
         } else if (!strcasecmp(opt,"WITHMATCHLEN")) {
             withmatchlen = 1;
-        } else if (!strcasecmp(opt,"STOREIDX") && moreargs) {
-            getidx = 1;
-            idxkey = c->argv[j+1];
-            j++;
         } else if (!strcasecmp(opt,"MINMATCHLEN") && moreargs) {
             if (getLongLongFromObjectOrReply(c,c->argv[j+1],&minmatchlen,NULL)
                 != C_OK) return;
             if (minmatchlen < 0) minmatchlen = 0;
             j++;
         } else if (!strcasecmp(opt,"STRINGS")) {
-            if (moreargs != 2) {
+            if (a != NULL) {
+                addReplyError(c,"Either use STRINGS or KEYS");
+                return;
+            } else if (moreargs != 2) {
                 addReplyError(c,"LCS requires exactly two strings");
                 return;
             }
@@ -520,7 +518,10 @@ void lcsCommand(client *c) {
             b = c->argv[j+2]->ptr;
             j += 2;
         } else if (!strcasecmp(opt,"KEYS")) {
-            if (moreargs != 2) {
+            if (a != NULL) {
+                addReplyError(c,"Either use STRINGS or KEYS");
+                return;
+            } else if (moreargs != 2) {
                 addReplyError(c,"LCS requires exactly two keys");
                 return;
             }
@@ -542,12 +543,10 @@ void lcsCommand(client *c) {
         addReplyError(c,"Please specify two strings: "
                         "STRINGS or KEYS options are mandatory");
         return;
-    } else if (getlen && (getidx && idxkey == NULL)) {
+    } else if (getlen && getidx) {
         addReplyError(c,
-            "If you want both the LEN and indexes, please "
-            "store the indexes into a key with STOREIDX. However note "
-            "that the IDX output also includes both the LCS string and "
-            "its length");
+            "If you want both the length and indexes, please "
+            "just use IDX.");
         return;
     }
 
@@ -602,7 +601,7 @@ void lcsCommand(client *c) {
 
     /* Start with a deferred array if we have to emit the ranges. */
     uint32_t arraylen = 0;  /* Number of ranges emitted in the array. */
-    if (getidx && idxkey == NULL) {
+    if (getidx) {
         addReplyMapLen(c,2);
         addReplyBulkCString(c,"matches");
         arraylenptr = addReplyDeferredLen(c);

From 8dc28b6c7525190f3205969427614d51d47ac451 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 6 Apr 2020 13:45:37 +0200
Subject: [PATCH 0249/1098] LCS tests.

---
 tests/unit/type/string.tcl | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tests/unit/type/string.tcl b/tests/unit/type/string.tcl
index 7122fd987..131a80ad0 100644
--- a/tests/unit/type/string.tcl
+++ b/tests/unit/type/string.tcl
@@ -419,4 +419,26 @@ start_server {tags {"string"}} {
         r set foo bar
         r getrange foo 0 4294967297
     } {bar}
+
+    set rna1 {CACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGTGTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTTCGTCCGGGTGTG}
+    set rna2 {ATTAAAGGTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGTGTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTT}
+    set rnalcs {ACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGTGTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTT}
+
+    test {LCS string output with STRINGS option} {
+        r LCS STRINGS $rna1 $rna2
+    } $rnalcs
+
+    test {LCS len} {
+        r LCS LEN STRINGS $rna1 $rna2
+    } [string length $rnalcs]
+
+    test {LCS with KEYS option} {
+        r set virus1 $rna1
+        r set virus2 $rna2
+        r LCS KEYS virus1 virus2
+    } $rnalcs
+
+    test {LCS indexes} {
+        dict get [r LCS IDX KEYS virus1 virus2] matches
+    } {{{238 238} {239 239}} {{236 236} {238 238}} {{229 230} {236 237}} {{224 224} {235 235}} {{1 222} {13 234}}}
 }

From ca8d6f1072e15fcb12827c1ca84b388c41e394b3 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 6 Apr 2020 13:48:31 +0200
Subject: [PATCH 0250/1098] LCS: allow KEYS / STRINGS to be anywhere.

Initially they needed to be at the end so that we could extend to N
strings in the future, but after further consideration I no longer
believe it's worth it.
---
 src/t_string.c | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index deba22253..4e693cefa 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -510,9 +510,6 @@ void lcsCommand(client *c) {
             if (a != NULL) {
                 addReplyError(c,"Either use STRINGS or KEYS");
                 return;
-            } else if (moreargs != 2) {
-                addReplyError(c,"LCS requires exactly two strings");
-                return;
             }
             a = c->argv[j+1]->ptr;
             b = c->argv[j+2]->ptr;
@@ -521,9 +518,6 @@ void lcsCommand(client *c) {
             if (a != NULL) {
                 addReplyError(c,"Either use STRINGS or KEYS");
                 return;
-            } else if (moreargs != 2) {
-                addReplyError(c,"LCS requires exactly two keys");
-                return;
             }
             obja = lookupKeyRead(c->db,c->argv[j+1]);
             objb = lookupKeyRead(c->db,c->argv[j+2]);

From af3c722feccb2d002e57f02ee5ef623f86e7ea2f Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 6 Apr 2020 13:51:49 +0200
Subject: [PATCH 0251/1098] LCS: more tests.

---
 tests/unit/type/string.tcl | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/unit/type/string.tcl b/tests/unit/type/string.tcl
index 131a80ad0..b9ef9de7a 100644
--- a/tests/unit/type/string.tcl
+++ b/tests/unit/type/string.tcl
@@ -441,4 +441,12 @@ start_server {tags {"string"}} {
     test {LCS indexes} {
         dict get [r LCS IDX KEYS virus1 virus2] matches
     } {{{238 238} {239 239}} {{236 236} {238 238}} {{229 230} {236 237}} {{224 224} {235 235}} {{1 222} {13 234}}}
+
+    test {LCS indexes with match len} {
+        dict get [r LCS IDX KEYS virus1 virus2 WITHMATCHLEN] matches
+    } {{{238 238} {239 239} 1} {{236 236} {238 238} 1} {{229 230} {236 237} 2} {{224 224} {235 235} 1} {{1 222} {13 234} 222}}
+
+    test {LCS indexes with match len and minimum match len} {
+        dict get [r LCS IDX KEYS virus1 virus2 WITHMATCHLEN MINMATCHLEN 5] matches
+    } {{{1 222} {13 234} 222}}
 }

From 2c42f6a8b8048d4b287afd58150e42b3fc7c97c9 Mon Sep 17 00:00:00 2001
From: qetu3790 <qetu3790@163.com>
Date: Mon, 6 Apr 2020 20:52:32 +0800
Subject: [PATCH 0252/1098] fix comments about RESIZE DB opcode in rdb.c

fix comments about RESIZE DB opcode in rdb.c
---
 src/rdb.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/rdb.c b/src/rdb.c
index fc8911979..78ec83cce 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1175,10 +1175,7 @@ int rdbSaveRio(rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi) {
         if (rdbSaveType(rdb,RDB_OPCODE_SELECTDB) == -1) goto werr;
         if (rdbSaveLen(rdb,j) == -1) goto werr;
 
-        /* Write the RESIZE DB opcode. We trim the size to UINT32_MAX, which
-         * is currently the largest type we are able to represent in RDB sizes.
-         * However this does not limit the actual size of the DB to load since
-         * these sizes are just hints to resize the hash tables. */
+        /* Write the RESIZE DB opcode. */
         uint64_t db_size, expires_size;
         db_size = dictSize(db->dict);
         expires_size = dictSize(db->expires);

From bb30b9f913ced7380cd4a46451cdebf3588f337d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 6 Apr 2020 17:32:04 +0200
Subject: [PATCH 0253/1098] Clarify redis.conf comment about
 lazyfree-lazy-user-del.

---
 redis.conf | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/redis.conf b/redis.conf
index 14388e320..5baeae65f 100644
--- a/redis.conf
+++ b/redis.conf
@@ -936,23 +936,27 @@ replica-priority 100
 #    or SORT with STORE option may delete existing keys. The SET command
 #    itself removes any old content of the specified key in order to replace
 #    it with the specified string.
-# 4) The DEL command itself, and normally it's not easy to replace DEL with
-#    UNLINK in user's codes.
-# 5) During replication, when a replica performs a full resynchronization with
+# 4) During replication, when a replica performs a full resynchronization with
 #    its master, the content of the whole database is removed in order to
 #    load the RDB file just transferred.
 #
 # In all the above cases the default is to delete objects in a blocking way,
 # like if DEL was called. However you can configure each case specifically
 # in order to instead release memory in a non-blocking way like if UNLINK
-# was called, using the following configuration directives:
+# was called, using the following configuration directives.
 
 lazyfree-lazy-eviction no
 lazyfree-lazy-expire no
 lazyfree-lazy-server-del no
-lazyfree-lazy-user-del no
 replica-lazy-flush no
 
+# It is also possible, for the case when to replace the user code DEL calls
+# with UNLINK calls is not easy, to modify the default behavior of the DEL
+# command to act exactly like UNLINK, using the following configuration
+# directive:
+
+lazyfree-lazy-user-del no
+
 ################################ THREADED I/O #################################
 
 # Redis is mostly single threaded, however there are certain threaded

From f69876280c630d49574fa094808d3a4791c5039b Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 7 Apr 2020 12:07:09 +0200
Subject: [PATCH 0254/1098] Speedup INFO by counting client memory
 incrementally.

Related to #5145.

Design note: clients may change type when they turn into replicas or are
moved into the Pub/Sub category and so forth. Moreover the recomputation
of the bytes used is problematic for obvious reasons: it changes
continuously, so as a conservative way to avoid accumulating errors,
each client remembers the contribution it gave to the sum, and removes
it when it is freed or before updating it with the new memory usage.
---
 src/networking.c |  7 +++++++
 src/object.c     | 33 +++++++++------------------------
 src/server.c     | 25 +++++++++++++++++++++++++
 src/server.h     | 13 +++++++++++--
 4 files changed, 52 insertions(+), 26 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 85c640e34..654fda517 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -157,6 +157,8 @@ client *createClient(connection *conn) {
     c->client_list_node = NULL;
     c->client_tracking_redirection = 0;
     c->client_tracking_prefixes = NULL;
+    c->client_cron_last_memory_usage = 0;
+    c->client_cron_last_memory_type = CLIENT_TYPE_NORMAL;
     c->auth_callback = NULL;
     c->auth_callback_privdata = NULL;
     c->auth_module = NULL;
@@ -1160,6 +1162,11 @@ void freeClient(client *c) {
         listDelNode(server.clients_to_close,ln);
     }
 
+    /* Remove the contribution that this client gave to our
+     * incrementally computed memory usage. */
+    server.stat_clients_type_memory[c->client_cron_last_memory_type] -=
+        c->client_cron_last_memory_usage;
+
     /* Release other dynamically allocated client structure fields,
      * and finally release the client structure itself. */
     if (c->name) decrRefCount(c->name);
diff --git a/src/object.c b/src/object.c
index 11e335afc..52d5b11f5 100644
--- a/src/object.c
+++ b/src/object.c
@@ -974,30 +974,15 @@ struct redisMemOverhead *getMemoryOverheadData(void) {
     mh->repl_backlog = mem;
     mem_total += mem;
 
-    mem = 0;
-    if (listLength(server.clients)) {
-        listIter li;
-        listNode *ln;
-        size_t mem_normal = 0, mem_slaves = 0;
-
-        listRewind(server.clients,&li);
-        while((ln = listNext(&li))) {
-            size_t mem_curr = 0;
-            client *c = listNodeValue(ln);
-            int type = getClientType(c);
-            mem_curr += getClientOutputBufferMemoryUsage(c);
-            mem_curr += sdsAllocSize(c->querybuf);
-            mem_curr += sizeof(client);
-            if (type == CLIENT_TYPE_SLAVE)
-                mem_slaves += mem_curr;
-            else
-                mem_normal += mem_curr;
-        }
-        mh->clients_slaves = mem_slaves;
-        mh->clients_normal = mem_normal;
-        mem = mem_slaves + mem_normal;
-    }
-    mem_total+=mem;
+    /* Computing the memory used by the clients would be O(N) if done
+     * here online. We use our values computed incrementally by
+     * clientsCronTrackClientsMemUsage(). */
+    mh->clients_slaves = server.stat_clients_type_memory[CLIENT_TYPE_SLAVE];
+    mh->clients_normal = server.stat_clients_type_memory[CLIENT_TYPE_MASTER]+
+                         server.stat_clients_type_memory[CLIENT_TYPE_PUBSUB]+
+                         server.stat_clients_type_memory[CLIENT_TYPE_NORMAL];
+    mem_total += mh->clients_slaves;
+    mem_total += mh->clients_normal;
 
     mem = 0;
     if (server.aof_state != AOF_OFF) {
diff --git a/src/server.c b/src/server.c
index 56feb09a3..996e0f5d2 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1593,6 +1593,28 @@ int clientsCronTrackExpansiveClients(client *c) {
     return 0; /* This function never terminates the client. */
 }
 
+/* Iterating all the clients in getMemoryOverheadData() is too slow and
+ * in turn would make the INFO command too slow. So we perform this
+ * computation incrementally and track the (not instantaneous but updated
+ * to the second) total memory used by clients using clinetsCron() in
+ * a more incremental way (depending on server.hz). */
+int clientsCronTrackClientsMemUsage(client *c) {
+    size_t mem = 0;
+    int type = getClientType(c);
+    mem += getClientOutputBufferMemoryUsage(c);
+    mem += sdsAllocSize(c->querybuf);
+    mem += sizeof(client);
+    /* Now that we have the memory used by the client, remove the old
+     * value from the old categoty, and add it back. */
+    server.stat_clients_type_memory[c->client_cron_last_memory_type] -=
+        c->client_cron_last_memory_usage;
+    server.stat_clients_type_memory[type] += mem;
+    /* Remember what we added and where, to remove it next time. */
+    c->client_cron_last_memory_usage = mem;
+    c->client_cron_last_memory_type = type;
+    return 0;
+}
+
 /* Return the max samples in the memory usage of clients tracked by
  * the function clientsCronTrackExpansiveClients(). */
 void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) {
@@ -1653,6 +1675,7 @@ void clientsCron(void) {
         if (clientsCronHandleTimeout(c,now)) continue;
         if (clientsCronResizeQueryBuffer(c)) continue;
         if (clientsCronTrackExpansiveClients(c)) continue;
+        if (clientsCronTrackClientsMemUsage(c)) continue;
     }
 }
 
@@ -2792,6 +2815,8 @@ void initServer(void) {
     server.stat_rdb_cow_bytes = 0;
     server.stat_aof_cow_bytes = 0;
     server.stat_module_cow_bytes = 0;
+    for (int j = 0; j < CLIENT_TYPE_COUNT; j++)
+        server.stat_clients_type_memory[j] = 0;
     server.cron_malloc_stats.zmalloc_used = 0;
     server.cron_malloc_stats.process_rss = 0;
     server.cron_malloc_stats.allocator_allocated = 0;
diff --git a/src/server.h b/src/server.h
index 9b77f55ac..cf4c285f8 100644
--- a/src/server.h
+++ b/src/server.h
@@ -274,6 +274,7 @@ typedef long long ustime_t; /* microsecond time type. */
 #define CLIENT_TYPE_SLAVE 1  /* Slaves. */
 #define CLIENT_TYPE_PUBSUB 2 /* Clients subscribed to PubSub channels. */
 #define CLIENT_TYPE_MASTER 3 /* Master. */
+#define CLIENT_TYPE_COUNT 4  /* Total number of client types. */
 #define CLIENT_TYPE_OBUF_COUNT 3 /* Number of clients to expose to output
                                     buffer configuration. Just the first
                                     three: normal, slave, pubsub. */
@@ -820,10 +821,10 @@ typedef struct client {
                                                * when the authenticated user
                                                * changes. */
     void *auth_callback_privdata; /* Private data that is passed when the auth
-                                   * changed callback is executed. Opaque for 
+                                   * changed callback is executed. Opaque for
                                    * Redis Core. */
     void *auth_module;      /* The module that owns the callback, which is used
-                             * to disconnect the client if the module is 
+                             * to disconnect the client if the module is
                              * unloaded for cleanup. Opaque for Redis Core.*/
 
     /* If this client is in tracking mode and this field is non zero,
@@ -833,6 +834,13 @@ typedef struct client {
     rax *client_tracking_prefixes; /* A dictionary of prefixes we are already
                                       subscribed to in BCAST mode, in the
                                       context of client side caching. */
+    /* In clientsCronTrackClientsMemUsage() we track the memory usage of
+     * each client and add it to the sum of all the clients of a given type,
+     * however we need to remember what was the old contribution of each
+     * client, and in which categoty the client was, in order to remove it
+     * before adding it the new value. */
+    uint64_t client_cron_last_memory_usage;
+    int      client_cron_last_memory_type;
     /* Response buffer */
     int bufpos;
     char buf[PROTO_REPLY_CHUNK_BYTES];
@@ -1129,6 +1137,7 @@ struct redisServer {
     size_t stat_rdb_cow_bytes;      /* Copy on write bytes during RDB saving. */
     size_t stat_aof_cow_bytes;      /* Copy on write bytes during AOF rewrite. */
     size_t stat_module_cow_bytes;   /* Copy on write bytes during module fork. */
+    uint64_t stat_clients_type_memory[CLIENT_TYPE_COUNT];/* Mem usage by type */
     long long stat_unexpected_error_replies; /* Number of unexpected (aof-loading, replica to master, etc.) error replies */
     /* The following two are used to track instantaneous metrics, like
      * number of operations per second, network traffic. */

From 96688aa6462f330dfd4780d222ce4806d766ff33 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 8 Apr 2020 10:54:18 +0200
Subject: [PATCH 0255/1098] Fix ACL HELP table missing comma.

---
 src/acl.c | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 27f4bdb84..733988013 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1847,18 +1847,18 @@ void aclCommand(client *c) {
         }
     } else if (!strcasecmp(sub,"help")) {
         const char *help[] = {
-"LOAD                              -- Reload users from the ACL file.",
-"SAVE                              -- Save the current config to the ACL file."
-"LIST                              -- Show user details in config file format.",
-"USERS                             -- List all the registered usernames.",
-"SETUSER <username> [attribs ...]  -- Create or modify a user.",
-"GETUSER <username>                -- Get the user details.",
-"DELUSER <username> [...]          -- Delete a list of users.",
-"CAT                               -- List available categories.",
-"CAT <category>                    -- List commands inside category.",
-"GENPASS                           -- Generate a secure user password.",
-"WHOAMI                            -- Return the current connection username.",
-"LOG [<count> | RESET]             -- Show the ACL log entries.",
+"LOAD                             -- Reload users from the ACL file.",
+"SAVE                             -- Save the current config to the ACL file.",
+"LIST                             -- Show user details in config file format.",
+"USERS                            -- List all the registered usernames.",
+"SETUSER <username> [attribs ...] -- Create or modify a user.",
+"GETUSER <username>               -- Get the user details.",
+"DELUSER <username> [...]         -- Delete a list of users.",
+"CAT                              -- List available categories.",
+"CAT <category>                   -- List commands inside category.",
+"GENPASS                          -- Generate a secure user password.",
+"WHOAMI                           -- Return the current connection username.",
+"LOG [<count> | RESET]            -- Show the ACL log entries.",
 NULL
         };
         addReplyHelp(c,help);

From 51b9de037dc64911430a333c277ac90064c9b834 Mon Sep 17 00:00:00 2001
From: hayleeliu <liuhuailing_123@163.com>
Date: Wed, 8 Apr 2020 18:20:32 +0800
Subject: [PATCH 0256/1098] fix spelling mistake in bitops.c

---
 src/bitops.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/bitops.c b/src/bitops.c
index f78e4fd34..496d78b66 100644
--- a/src/bitops.c
+++ b/src/bitops.c
@@ -269,7 +269,7 @@ int64_t getSignedBitfield(unsigned char *p, uint64_t offset, uint64_t bits) {
  * then zero is returned, otherwise in case of overflow, 1 is returned,
  * otherwise in case of underflow, -1 is returned.
  *
- * When non-zero is returned (oferflow or underflow), if not NULL, *limit is
+ * When non-zero is returned (overflow or underflow), if not NULL, *limit is
  * set to the value the operation should result when an overflow happens,
  * depending on the specified overflow semantics:
  *

From 96a54866ab4694cf338af0441f28aa69e9643376 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 8 Apr 2020 12:55:57 +0200
Subject: [PATCH 0257/1098] Speedup: unblock clients on keys in O(1).

See #7071.
---
 src/adlist.c  | 20 +++++++++++++++++---
 src/adlist.h  |  3 ++-
 src/blocked.c | 48 ++++++++++++++++++++++++++++++------------------
 src/server.c  |  2 +-
 4 files changed, 50 insertions(+), 23 deletions(-)

diff --git a/src/adlist.c b/src/adlist.c
index ec5f8bbf4..0fedc0729 100644
--- a/src/adlist.c
+++ b/src/adlist.c
@@ -327,12 +327,11 @@ listNode *listIndex(list *list, long index) {
 }
 
 /* Rotate the list removing the tail node and inserting it to the head. */
-void listRotate(list *list) {
-    listNode *tail = list->tail;
-
+void listRotateTailToHead(list *list) {
     if (listLength(list) <= 1) return;
 
     /* Detach current tail */
+    listNode *tail = list->tail;
     list->tail = tail->prev;
     list->tail->next = NULL;
     /* Move it as head */
@@ -342,6 +341,21 @@ void listRotate(list *list) {
     list->head = tail;
 }
 
+/* Rotate the list removing the head node and inserting it to the tail. */
+void listRotateHeadToTail(list *list) {
+    if (listLength(list) <= 1) return;
+
+    listNode *head = list->head;
+    /* Detach current head */
+    list->head = head->next;
+    list->head->prev = NULL;
+    /* Move it as tail */
+    list->tail->next = head;
+    head->next = NULL;
+    head->prev = list->tail;
+    list->tail = head;
+}
+
 /* Add all the elements of the list 'o' at the end of the
  * list 'l'. The list 'other' remains empty but otherwise valid. */
 void listJoin(list *l, list *o) {
diff --git a/src/adlist.h b/src/adlist.h
index 28b9016ce..dd8a8d693 100644
--- a/src/adlist.h
+++ b/src/adlist.h
@@ -85,7 +85,8 @@ listNode *listSearchKey(list *list, void *key);
 listNode *listIndex(list *list, long index);
 void listRewind(list *list, listIter *li);
 void listRewindTail(list *list, listIter *li);
-void listRotate(list *list);
+void listRotateTailToHead(list *list);
+void listRotateHeadToTail(list *list);
 void listJoin(list *l, list *o);
 
 /* Directions for iterators */
diff --git a/src/blocked.c b/src/blocked.c
index e3a803ae3..00cc798d5 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -64,6 +64,21 @@
 
 int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int where);
 
+/* This structure represents the blocked key information that we store
+ * in the client structure. Each client blocked on keys, has a
+ * client->bpop.keys hash table. The keys of the hash table are Redis
+ * keys pointers to 'robj' structures. The value is this structure.
+ * The structure has two goals: firstly we store the list node that this
+ * client uses to be listed in the database "blocked clients for this key"
+ * list, so we can later unblock in O(1) without a list scan.
+ * Secondly for certain blocking types, we have additional info. Right now
+ * the only use for additional info we have is when clients are blocked
+ * on streams, as we have to remember the ID it blocked for. */
+typedef struct bkinfo {
+    listNode *listnode;     /* List node for db->blocking_keys[key] list. */
+    streamID stream_id;     /* Stream ID if we blocked in a stream. */
+} bkinfo;
+
 /* Block a client for the specific operation type. Once the CLIENT_BLOCKED
  * flag is set client query buffer is not longer processed, but accumulated,
  * and will be processed when the client is unblocked. */
@@ -211,8 +226,7 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) {
             if (receiver->btype != BLOCKED_LIST) {
                 /* Put at the tail, so that at the next call
                  * we'll not run into it again. */
-                listDelNode(clients,clientnode);
-                listAddNodeTail(clients,receiver);
+                listRotateHeadToTail(clients);
                 continue;
             }
 
@@ -273,8 +287,7 @@ void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) {
             if (receiver->btype != BLOCKED_ZSET) {
                 /* Put at the tail, so that at the next call
                  * we'll not run into it again. */
-                listDelNode(clients,clientnode);
-                listAddNodeTail(clients,receiver);
+                listRotateHeadToTail(clients);
                 continue;
             }
 
@@ -320,8 +333,8 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) {
         while((ln = listNext(&li))) {
             client *receiver = listNodeValue(ln);
             if (receiver->btype != BLOCKED_STREAM) continue;
-            streamID *gt = dictFetchValue(receiver->bpop.keys,
-                                          rl->key);
+            bkinfo *bki = dictFetchValue(receiver->bpop.keys,rl->key);
+            streamID *gt = &bki->stream_id;
 
             /* If we blocked in the context of a consumer
              * group, we need to resolve the group and update the
@@ -419,8 +432,7 @@ void serveClientsBlockedOnKeyByModule(readyList *rl) {
              * ready to be served, so they'll remain in the list
              * sometimes. We want also be able to skip clients that are
              * not blocked for the MODULE type safely. */
-            listDelNode(clients,clientnode);
-            listAddNodeTail(clients,receiver);
+            listRotateHeadToTail(clients);
 
             if (receiver->btype != BLOCKED_MODULE) continue;
 
@@ -551,17 +563,15 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo
     if (target != NULL) incrRefCount(target);
 
     for (j = 0; j < numkeys; j++) {
-        /* The value associated with the key name in the bpop.keys dictionary
-         * is NULL for lists and sorted sets, or the stream ID for streams. */
-        void *key_data = NULL;
-        if (btype == BLOCKED_STREAM) {
-            key_data = zmalloc(sizeof(streamID));
-            memcpy(key_data,ids+j,sizeof(streamID));
-        }
+        /* Allocate our bkinfo structure, associated to each key the client
+         * is blocked for. */
+        bkinfo *bki = zmalloc(sizeof(*bki));
+        if (btype == BLOCKED_STREAM)
+            bki->stream_id = ids[j];
 
         /* If the key already exists in the dictionary ignore it. */
-        if (dictAdd(c->bpop.keys,keys[j],key_data) != DICT_OK) {
-            zfree(key_data);
+        if (dictAdd(c->bpop.keys,keys[j],bki) != DICT_OK) {
+            zfree(bki);
             continue;
         }
         incrRefCount(keys[j]);
@@ -580,6 +590,7 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo
             l = dictGetVal(de);
         }
         listAddNodeTail(l,c);
+        bki->listnode = listLast(l);
     }
     blockClient(c,btype);
 }
@@ -596,11 +607,12 @@ void unblockClientWaitingData(client *c) {
     /* The client may wait for multiple keys, so unblock it for every key. */
     while((de = dictNext(di)) != NULL) {
         robj *key = dictGetKey(de);
+        bkinfo *bki = dictGetVal(de);
 
         /* Remove this client from the list of clients waiting for this key. */
         l = dictFetchValue(c->db->blocking_keys,key);
         serverAssertWithInfo(c,key,l != NULL);
-        listDelNode(l,listSearchKey(l,c));
+        listDelNode(l,bki->listnode);
         /* If the list is empty we need to remove it to avoid wasting memory */
         if (listLength(l) == 0)
             dictDelete(c->db->blocking_keys,key);
diff --git a/src/server.c b/src/server.c
index 996e0f5d2..0afd67514 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1666,7 +1666,7 @@ void clientsCron(void) {
         /* Rotate the list, take the current head, process.
          * This way if the client must be removed from the list it's the
          * first element and we don't incur into O(N) computation. */
-        listRotate(server.clients);
+        listRotateTailToHead(server.clients);
         head = listFirst(server.clients);
         c = listNodeValue(head);
         /* The following functions do different service checks on the client.

From 30adc62232b2f8fe7dc31b850d8df3979f443234 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 9 Apr 2020 10:24:10 +0200
Subject: [PATCH 0258/1098] RDB: load files faster avoiding useless
 free+realloc.

Reloading of the RDB generated by

    DEBUG POPULATE 5000000
    SAVE

is now 25% faster.

This commit also prepares the ability to have more flexibility when
loading stuff from the RDB, since we no longer use dbAdd() but can
control exactly how things are added in the database.
---
 src/cluster.c         |  2 +-
 src/db.c              | 16 ++++++++--------
 src/lazyfree.c        |  2 +-
 src/rdb.c             | 40 ++++++++++++++++++++++++++--------------
 src/rdb.h             |  2 +-
 src/redis-check-rdb.c |  2 +-
 src/server.h          |  4 ++--
 7 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 385ff5763..2377b386b 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4966,7 +4966,7 @@ void restoreCommand(client *c) {
 
     rioInitWithBuffer(&payload,c->argv[3]->ptr);
     if (((type = rdbLoadObjectType(&payload)) == -1) ||
-        ((obj = rdbLoadObject(type,&payload,c->argv[1])) == NULL))
+        ((obj = rdbLoadObject(type,&payload,c->argv[1]->ptr)) == NULL))
     {
         addReplyError(c,"Bad data format");
         return;
diff --git a/src/db.c b/src/db.c
index 9b5d62f29..d393a5fdd 100644
--- a/src/db.c
+++ b/src/db.c
@@ -185,7 +185,7 @@ void dbAdd(redisDb *db, robj *key, robj *val) {
         val->type == OBJ_ZSET ||
         val->type == OBJ_STREAM)
         signalKeyAsReady(db, key);
-    if (server.cluster_enabled) slotToKeyAdd(key);
+    if (server.cluster_enabled) slotToKeyAdd(key->ptr);
 }
 
 /* Overwrite an existing key with a new value. Incrementing the reference
@@ -288,7 +288,7 @@ int dbSyncDelete(redisDb *db, robj *key) {
      * the key, because it is shared with the main dictionary. */
     if (dictSize(db->expires) > 0) dictDelete(db->expires,key->ptr);
     if (dictDelete(db->dict,key->ptr) == DICT_OK) {
-        if (server.cluster_enabled) slotToKeyDel(key);
+        if (server.cluster_enabled) slotToKeyDel(key->ptr);
         return 1;
     } else {
         return 0;
@@ -1647,17 +1647,17 @@ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys)
  * a fast way a key that belongs to a specified hash slot. This is useful
  * while rehashing the cluster and in other conditions when we need to
  * understand if we have keys for a given hash slot. */
-void slotToKeyUpdateKey(robj *key, int add) {
-    unsigned int hashslot = keyHashSlot(key->ptr,sdslen(key->ptr));
+void slotToKeyUpdateKey(sds key, int add) {
+    size_t keylen = sdslen(key);
+    unsigned int hashslot = keyHashSlot(key,keylen);
     unsigned char buf[64];
     unsigned char *indexed = buf;
-    size_t keylen = sdslen(key->ptr);
 
     server.cluster->slots_keys_count[hashslot] += add ? 1 : -1;
     if (keylen+2 > 64) indexed = zmalloc(keylen+2);
     indexed[0] = (hashslot >> 8) & 0xff;
     indexed[1] = hashslot & 0xff;
-    memcpy(indexed+2,key->ptr,keylen);
+    memcpy(indexed+2,key,keylen);
     if (add) {
         raxInsert(server.cluster->slots_to_keys,indexed,keylen+2,NULL,NULL);
     } else {
@@ -1666,11 +1666,11 @@ void slotToKeyUpdateKey(robj *key, int add) {
     if (indexed != buf) zfree(indexed);
 }
 
-void slotToKeyAdd(robj *key) {
+void slotToKeyAdd(sds key) {
     slotToKeyUpdateKey(key,1);
 }
 
-void slotToKeyDel(robj *key) {
+void slotToKeyDel(sds key) {
     slotToKeyUpdateKey(key,0);
 }
 
diff --git a/src/lazyfree.c b/src/lazyfree.c
index 3d3159c90..f01504e70 100644
--- a/src/lazyfree.c
+++ b/src/lazyfree.c
@@ -83,7 +83,7 @@ int dbAsyncDelete(redisDb *db, robj *key) {
      * field to NULL in order to lazy free it later. */
     if (de) {
         dictFreeUnlinkedEntry(db->dict,de);
-        if (server.cluster_enabled) slotToKeyDel(key);
+        if (server.cluster_enabled) slotToKeyDel(key->ptr);
         return 1;
     } else {
         return 0;
diff --git a/src/rdb.c b/src/rdb.c
index 78ec83cce..d3104ffbf 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1422,7 +1422,7 @@ robj *rdbLoadCheckModuleValue(rio *rdb, char *modulename) {
 
 /* Load a Redis object of the specified type from the specified file.
  * On success a newly allocated object is returned, otherwise NULL. */
-robj *rdbLoadObject(int rdbtype, rio *rdb, robj *key) {
+robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
     robj *o = NULL, *ele, *dec;
     uint64_t len;
     unsigned int i;
@@ -1886,7 +1886,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, robj *key) {
             exit(1);
         }
         RedisModuleIO io;
-        moduleInitIOContext(io,mt,rdb,key);
+        robj keyobj;
+        initStaticStringObject(keyobj,key);
+        moduleInitIOContext(io,mt,rdb,&keyobj);
         io.ver = (rdbtype == RDB_TYPE_MODULE) ? 1 : 2;
         /* Call the rdb_load method of the module providing the 10 bit
          * encoding version in the lower 10 bits of the module ID. */
@@ -2044,7 +2046,8 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
     long long lru_clock = LRU_CLOCK();
 
     while(1) {
-        robj *key, *val;
+        sds key;
+        robj *val;
 
         /* Read type. */
         if ((type = rdbLoadType(rdb)) == -1) goto eoferr;
@@ -2216,10 +2219,11 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
         }
 
         /* Read key */
-        if ((key = rdbLoadStringObject(rdb)) == NULL) goto eoferr;
+        if ((key = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL)
+            goto eoferr;
         /* Read value */
         if ((val = rdbLoadObject(type,rdb,key)) == NULL) {
-            decrRefCount(key);
+            sdsfree(key);
             goto eoferr;
         }
 
@@ -2229,24 +2233,32 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
          * responsible for key expiry. If we would expire keys here, the
          * snapshot taken by the master may not be reflected on the slave. */
         if (iAmMaster() && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now) {
-            decrRefCount(key);
+            sdsfree(key);
             decrRefCount(val);
         } else {
             /* Add the new object in the hash table */
-            dbAdd(db,key,val);
+            int retval = dictAdd(db->dict, key, val);
+            if (retval != DICT_OK) {
+                serverLog(LL_WARNING,
+                    "RDB has duplicated key '%s' in DB %d",key,db->id);
+                serverPanic("Duplicated key found in RDB file");
+            }
+            if (server.cluster_enabled) slotToKeyAdd(key);
 
             /* Set the expire time if needed */
-            if (expiretime != -1) setExpire(NULL,db,key,expiretime);
+            if (expiretime != -1) {
+                robj keyobj;
+                initStaticStringObject(keyobj,key);
+                setExpire(NULL,db,&keyobj,expiretime);
+            }
 
             /* Set usage information (for eviction). */
             objectSetLRUOrLFU(val,lfu_freq,lru_idle,lru_clock,1000);
-
-            /* Decrement the key refcount since dbAdd() will take its
-             * own reference. */
-            decrRefCount(key);
         }
-        if (server.key_load_delay)
-            usleep(server.key_load_delay);
+
+        /* Loading the database more slowly is useful in order to test
+         * certain edge cases. */
+        if (server.key_load_delay) usleep(server.key_load_delay);
 
         /* Reset the state that is key-specified and is populated by
          * opcodes before the key, so that we start from scratch again. */
diff --git a/src/rdb.h b/src/rdb.h
index b276a978b..526ed116b 100644
--- a/src/rdb.h
+++ b/src/rdb.h
@@ -144,7 +144,7 @@ void rdbRemoveTempFile(pid_t childpid);
 int rdbSave(char *filename, rdbSaveInfo *rsi);
 ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key);
 size_t rdbSavedObjectLen(robj *o, robj *key);
-robj *rdbLoadObject(int type, rio *rdb, robj *key);
+robj *rdbLoadObject(int type, rio *rdb, sds key);
 void backgroundSaveDoneHandler(int exitcode, int bysignal);
 int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime);
 ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt);
diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c
index 1210d49b4..17ec656ce 100644
--- a/src/redis-check-rdb.c
+++ b/src/redis-check-rdb.c
@@ -287,7 +287,7 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) {
         rdbstate.keys++;
         /* Read value */
         rdbstate.doing = RDB_CHECK_DOING_READ_OBJECT_VALUE;
-        if ((val = rdbLoadObject(type,&rdb,key)) == NULL) goto eoferr;
+        if ((val = rdbLoadObject(type,&rdb,key->ptr)) == NULL) goto eoferr;
         /* Check if the key already expired. */
         if (expiretime != -1 && expiretime < now)
             rdbstate.already_expired++;
diff --git a/src/server.h b/src/server.h
index cf4c285f8..b8c46153d 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2093,8 +2093,8 @@ unsigned int delKeysInSlot(unsigned int hashslot);
 int verifyClusterConfigWithData(void);
 void scanGenericCommand(client *c, robj *o, unsigned long cursor);
 int parseScanCursorOrReply(client *c, robj *o, unsigned long *cursor);
-void slotToKeyAdd(robj *key);
-void slotToKeyDel(robj *key);
+void slotToKeyAdd(sds key);
+void slotToKeyDel(sds key);
 void slotToKeyFlush(void);
 int dbAsyncDelete(redisDb *db, robj *key);
 void emptyDbAsync(redisDb *db);

From 3d5b2d41b52906269778c42fec40d3c85535b9bf Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 9 Apr 2020 11:09:40 +0200
Subject: [PATCH 0259/1098] RDB: clarify a condition in rdbLoadRio().

---
 src/rdb.c | 10 ++++++++--
 src/rdb.h |  1 +
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/rdb.c b/src/rdb.c
index d3104ffbf..313830c25 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2231,8 +2231,14 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
          * an RDB file from disk, either at startup, or when an RDB was
          * received from the master. In the latter case, the master is
          * responsible for key expiry. If we would expire keys here, the
-         * snapshot taken by the master may not be reflected on the slave. */
-        if (iAmMaster() && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now) {
+         * snapshot taken by the master may not be reflected on the slave.
+         * Similarly if the RDB is the preamble of an AOF file, we want to
+         * load all the keys as they are, since the log of operations later
+         * assume to work in an exact keyspace state. */
+        if (iAmMaster() &&
+            !(rdbflags&RDBFLAGS_AOF_PREAMBLE) &&
+            expiretime != -1 && expiretime < now)
+        {
             sdsfree(key);
             decrRefCount(val);
         } else {
diff --git a/src/rdb.h b/src/rdb.h
index 526ed116b..9dfcae7a5 100644
--- a/src/rdb.h
+++ b/src/rdb.h
@@ -125,6 +125,7 @@
 #define RDBFLAGS_NONE 0
 #define RDBFLAGS_AOF_PREAMBLE (1<<0)
 #define RDBFLAGS_REPLICATION (1<<1)
+#define RDBFLAGS_ALLOW_DUP (1<<2)
 
 int rdbSaveType(rio *rdb, unsigned char type);
 int rdbLoadType(rio *rdb);

From 3fbfa1885c082b1043970f9c273a9413bce05a83 Mon Sep 17 00:00:00 2001
From: liumiuyong <lmy@meitu.com>
Date: Thu, 9 Apr 2020 17:48:29 +0800
Subject: [PATCH 0260/1098] FIX: truncate max/min longitude,latitude related
 geo_point (ex:  {180, 85.05112878} )

---
 src/geohash.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/geohash.c b/src/geohash.c
index db5ae025a..de9620b7a 100644
--- a/src/geohash.c
+++ b/src/geohash.c
@@ -206,7 +206,11 @@ int geohashDecodeWGS84(const GeoHashBits hash, GeoHashArea *area) {
 int geohashDecodeAreaToLongLat(const GeoHashArea *area, double *xy) {
     if (!xy) return 0;
     xy[0] = (area->longitude.min + area->longitude.max) / 2;
+    if (xy[0] > GEO_LONG_MAX) xy[0] = GEO_LONG_MAX;
+    if (xy[0] < GEO_LONG_MIN) xy[0] = GEO_LONG_MIN;
     xy[1] = (area->latitude.min + area->latitude.max) / 2;
+    if (xy[1] > GEO_LAT_MAX) xy[1] = GEO_LAT_MAX;
+    if (xy[1] < GEO_LAT_MIN) xy[1] = GEO_LAT_MIN;
     return 1;
 }
 

From 451872527cec0b369211284b48e48867f5d168a9 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 9 Apr 2020 12:02:27 +0200
Subject: [PATCH 0261/1098] More powerful DEBUG RELOAD.

Related to #3243.
---
 src/debug.c | 45 +++++++++++++++++++++++++++++++++++++--------
 src/rdb.c   | 18 ++++++++++++++----
 src/rdb.h   |  8 ++++----
 3 files changed, 55 insertions(+), 16 deletions(-)

diff --git a/src/debug.c b/src/debug.c
index baaaa2424..1351b2536 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -366,7 +366,7 @@ void debugCommand(client *c) {
 "OOM -- Crash the server simulating an out-of-memory error.",
 "PANIC -- Crash the server simulating a panic.",
 "POPULATE <count> [prefix] [size] -- Create <count> string keys named key:<num>. If a prefix is specified is used instead of the 'key' prefix.",
-"RELOAD -- Save the RDB on disk and reload it back in memory.",
+"RELOAD [MERGE] [NOFLUSH] [NOSAVE] -- Save the RDB on disk and reload it back in memory. By default it will save the RDB file and load it back. With the NOFLUSH option the current database is not removed before loading the new one, but conficts in keys will kill the server with an exception. When MERGE is used, conflicting keys will be loaded (the key in the loaded RDB file will win). When NOSAVE is used, the server will not save the current dataset in the RDB file before loading. Use DEBUG RELOAD NOSAVE when you want just to load the RDB file you placed in the Redis working directory in order to replace the current dataset in memory. Use DEBUG RELOAD NOSAVE NOFLUSH MERGE when you want to add what is in the current RDB file placed in the Redis current directory, with the current memory content. Use DEBUG RELOAD when you want to verify Redis is able to persist the current dataset in the RDB file, flush the memory content, and load it back.",
 "RESTART -- Graceful restart: save config, db, restart.",
 "SDSLEN <key> -- Show low level SDS string info representing key and value.",
 "SEGFAULT -- Crash the server with sigsegv.",
@@ -411,15 +411,44 @@ NULL
         serverLog(LL_WARNING, "DEBUG LOG: %s", (char*)c->argv[2]->ptr);
         addReply(c,shared.ok);
     } else if (!strcasecmp(c->argv[1]->ptr,"reload")) {
-        rdbSaveInfo rsi, *rsiptr;
-        rsiptr = rdbPopulateSaveInfo(&rsi);
-        if (rdbSave(server.rdb_filename,rsiptr) != C_OK) {
-            addReply(c,shared.err);
-            return;
+        int flush = 1, save = 1;
+        int flags = RDBFLAGS_NONE;
+
+        /* Parse the additional options that modify the RELOAD
+         * behavior. */
+        for (int j = 2; j < c->argc; j++) {
+            char *opt = c->argv[j]->ptr;
+            if (!strcasecmp(opt,"MERGE")) {
+                flags |= RDBFLAGS_ALLOW_DUP;
+            } else if (!strcasecmp(opt,"NOFLUSH")) {
+                flush = 0;
+            } else if (!strcasecmp(opt,"NOSAVE")) {
+                save = 0;
+            } else {
+                addReplyError(c,"DEBUG RELOAD only supports the "
+                                "MERGE, NOFLUSH and NOSAVE options.");
+                return;
+            }
         }
-        emptyDb(-1,EMPTYDB_NO_FLAGS,NULL);
+
+        /* The default beahvior is to save the RDB file before loading
+         * it back. */
+        if (save) {
+            rdbSaveInfo rsi, *rsiptr;
+            rsiptr = rdbPopulateSaveInfo(&rsi);
+            if (rdbSave(server.rdb_filename,rsiptr) != C_OK) {
+                addReply(c,shared.err);
+                return;
+            }
+        }
+
+        /* The default behavior is to remove the current dataset from
+         * memory before loading the RDB file, however when MERGE is
+         * used together with NOFLUSH, we are able to merge two datasets. */
+        if (flush) emptyDb(-1,EMPTYDB_NO_FLAGS,NULL);
+
         protectClient(c);
-        int ret = rdbLoad(server.rdb_filename,NULL,RDBFLAGS_NONE);
+        int ret = rdbLoad(server.rdb_filename,NULL,flags);
         unprotectClient(c);
         if (ret != C_OK) {
             addReplyError(c,"Error trying to load the RDB dump");
diff --git a/src/rdb.c b/src/rdb.c
index 313830c25..3f25535ab 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2242,18 +2242,28 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
             sdsfree(key);
             decrRefCount(val);
         } else {
+            robj keyobj;
+
             /* Add the new object in the hash table */
             int retval = dictAdd(db->dict, key, val);
             if (retval != DICT_OK) {
-                serverLog(LL_WARNING,
-                    "RDB has duplicated key '%s' in DB %d",key,db->id);
-                serverPanic("Duplicated key found in RDB file");
+                if (rdbflags & RDBFLAGS_ALLOW_DUP) {
+                    /* This flag is useful for DEBUG RELOAD special modes.
+                     * When it's set we allow new keys to replace the current
+                     * keys with the same name. */
+                    initStaticStringObject(keyobj,key);
+                    dbSyncDelete(db,&keyobj);
+                    dictAdd(db->dict, key, val);
+                } else {
+                    serverLog(LL_WARNING,
+                        "RDB has duplicated key '%s' in DB %d",key,db->id);
+                    serverPanic("Duplicated key found in RDB file");
+                }
             }
             if (server.cluster_enabled) slotToKeyAdd(key);
 
             /* Set the expire time if needed */
             if (expiretime != -1) {
-                robj keyobj;
                 initStaticStringObject(keyobj,key);
                 setExpire(NULL,db,&keyobj,expiretime);
             }
diff --git a/src/rdb.h b/src/rdb.h
index 9dfcae7a5..aae682dbc 100644
--- a/src/rdb.h
+++ b/src/rdb.h
@@ -122,10 +122,10 @@
 #define RDB_LOAD_SDS    (1<<2)
 
 /* flags on the purpose of rdb save or load */
-#define RDBFLAGS_NONE 0
-#define RDBFLAGS_AOF_PREAMBLE (1<<0)
-#define RDBFLAGS_REPLICATION (1<<1)
-#define RDBFLAGS_ALLOW_DUP (1<<2)
+#define RDBFLAGS_NONE 0                 /* No special RDB loading. */
+#define RDBFLAGS_AOF_PREAMBLE (1<<0)    /* Load/save the RDB as AOF preamble. */
+#define RDBFLAGS_REPLICATION (1<<1)     /* Load/save for SYNC. */
+#define RDBFLAGS_ALLOW_DUP (1<<2)       /* Allow duplicated keys when loading.*/
 
 int rdbSaveType(rio *rdb, unsigned char type);
 int rdbLoadType(rio *rdb);

From 399a6b2b471cac481b00673e317fd585bd13d41f Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 9 Apr 2020 16:20:41 +0200
Subject: [PATCH 0262/1098] incrRefCount(): abort on statically allocated
 object.

---
 src/object.c | 10 +++++++++-
 src/server.h |  4 +++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/object.c b/src/object.c
index 52d5b11f5..1bc400e85 100644
--- a/src/object.c
+++ b/src/object.c
@@ -347,7 +347,15 @@ void freeStreamObject(robj *o) {
 }
 
 void incrRefCount(robj *o) {
-    if (o->refcount != OBJ_SHARED_REFCOUNT) o->refcount++;
+    if (o->refcount < OBJ_FIRST_SPECIAL_REFCOUNT) {
+        o->refcount++;
+    } else {
+        if (o->refcount == OBJ_SHARED_REFCOUNT) {
+            /* Nothing to do: this refcount is immutable. */
+        } else if (o->refcount == OBJ_STATIC_REFCOUNT) {
+            serverPanic("You tried to retain an object allocated in the stack");
+        }
+    }
 }
 
 void decrRefCount(robj *o) {
diff --git a/src/server.h b/src/server.h
index b8c46153d..9691381c3 100644
--- a/src/server.h
+++ b/src/server.h
@@ -597,7 +597,9 @@ typedef struct RedisModuleDigest {
 #define LRU_CLOCK_MAX ((1<<LRU_BITS)-1) /* Max value of obj->lru */
 #define LRU_CLOCK_RESOLUTION 1000 /* LRU clock resolution in ms */
 
-#define OBJ_SHARED_REFCOUNT INT_MAX
+#define OBJ_SHARED_REFCOUNT INT_MAX     /* Global object never destroyed. */
+#define OBJ_STATIC_REFCOUNT (INT_MAX-1) /* Object allocated in the stack. */
+#define OBJ_FIRST_SPECIAL_REFCOUNT OBJ_STATIC_REFCOUNT
 typedef struct redisObject {
     unsigned type:4;
     unsigned encoding:4;

From d88f52ee7d97c37a1845cf38d259b7900a940c71 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 9 Apr 2020 16:21:48 +0200
Subject: [PATCH 0263/1098] RDB: refactor some RDB loading code into
 dbAddRDBLoad().

---
 src/db.c     | 18 ++++++++++++++++++
 src/rdb.c    |  7 +++----
 src/server.h |  1 +
 3 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/src/db.c b/src/db.c
index d393a5fdd..59f0cc7a0 100644
--- a/src/db.c
+++ b/src/db.c
@@ -188,6 +188,24 @@ void dbAdd(redisDb *db, robj *key, robj *val) {
     if (server.cluster_enabled) slotToKeyAdd(key->ptr);
 }
 
+/* This is a special version of dbAdd() that is used only when loading
+ * keys from the RDB file: the key is passed as an SDS string that is
+ * retained by the function (and not freed by the caller).
+ *
+ * Moreover this function will not abort if the key is already busy, to
+ * give more control to the caller, nor will signal the key as ready
+ * since it is not useful in this context.
+ *
+ * The function returns 1 if the key was added to the database, taking
+ * ownership of the SDS string, otherwise 0 is returned, and is up to the
+ * caller to free the SDS string. */
+int dbAddRDBLoad(redisDb *db, sds key, robj *val) {
+    int retval = dictAdd(db->dict, key, val);
+    if (retval != DICT_OK) return 0;
+    if (server.cluster_enabled) slotToKeyAdd(key);
+    return 1;
+}
+
 /* Overwrite an existing key with a new value. Incrementing the reference
  * count of the new value is up to the caller.
  * This function does not modify the expire time of the existing key.
diff --git a/src/rdb.c b/src/rdb.c
index 3f25535ab..143b6c325 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2245,22 +2245,21 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
             robj keyobj;
 
             /* Add the new object in the hash table */
-            int retval = dictAdd(db->dict, key, val);
-            if (retval != DICT_OK) {
+            int added = dbAddRDBLoad(db,key,val);
+            if (!added) {
                 if (rdbflags & RDBFLAGS_ALLOW_DUP) {
                     /* This flag is useful for DEBUG RELOAD special modes.
                      * When it's set we allow new keys to replace the current
                      * keys with the same name. */
                     initStaticStringObject(keyobj,key);
                     dbSyncDelete(db,&keyobj);
-                    dictAdd(db->dict, key, val);
+                    dbAddRDBLoad(db,key,val);
                 } else {
                     serverLog(LL_WARNING,
                         "RDB has duplicated key '%s' in DB %d",key,db->id);
                     serverPanic("Duplicated key found in RDB file");
                 }
             }
-            if (server.cluster_enabled) slotToKeyAdd(key);
 
             /* Set the expire time if needed */
             if (expiretime != -1) {
diff --git a/src/server.h b/src/server.h
index 9691381c3..c268f2bee 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2069,6 +2069,7 @@ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
 #define LOOKUP_NONE 0
 #define LOOKUP_NOTOUCH (1<<0)
 void dbAdd(redisDb *db, robj *key, robj *val);
+int dbAddRDBLoad(redisDb *db, sds key, robj *val);
 void dbOverwrite(redisDb *db, robj *key, robj *val);
 void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal);
 void setKey(redisDb *db, robj *key, robj *val);

From 6ad44b76b9238eac119c9168994c9171057f3d47 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 9 Apr 2020 16:25:30 +0200
Subject: [PATCH 0264/1098] Use the special static refcount for stack objects.

---
 src/server.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.h b/src/server.h
index c268f2bee..d0d5ff154 100644
--- a/src/server.h
+++ b/src/server.h
@@ -620,7 +620,7 @@ char *getObjectTypeName(robj*);
  * we'll update it when the structure is changed, to avoid bugs like
  * bug #85 introduced exactly in this way. */
 #define initStaticStringObject(_var,_ptr) do { \
-    _var.refcount = 1; \
+    _var.refcount = OBJ_STATIC_REFCOUNT; \
     _var.type = OBJ_STRING; \
     _var.encoding = OBJ_ENCODING_RAW; \
     _var.ptr = _ptr; \

From 839bbcb039bddc3e49f00633879be5317ebdf829 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 10 Apr 2020 10:12:26 +0200
Subject: [PATCH 0265/1098] RESP3: change streams items from maps to arrays.

Streams items are similar to dictionaries, however they preserve both
the order, and allow for duplicated field names. So a map is not a
semantically sounding way to deal with this.

https://twitter.com/antirez/status/1248261087553880069
---
 src/t_stream.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index e0af87f97..4ce3a9b25 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -963,7 +963,7 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
         addReplyArrayLen(c,2);
         addReplyStreamID(c,&id);
 
-        addReplyMapLen(c,numfields);
+        addReplyArrayLen(c,numfields*2);
 
         /* Emit the field-value pairs. */
         while(numfields--) {

From 29760b2eb74132566b6eb7ac341c54c50a96ae6a Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Sat, 11 Apr 2020 15:05:01 +0300
Subject: [PATCH 0266/1098] Typo in getTimeoutFromObjectOrReply's error reply

---
 src/timeout.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/timeout.c b/src/timeout.c
index bb5999418..7787a049f 100644
--- a/src/timeout.c
+++ b/src/timeout.c
@@ -166,7 +166,7 @@ int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int
 
     if (unit == UNIT_SECONDS) {
         if (getLongDoubleFromObjectOrReply(c,object,&ftval,
-            "timeout is not an float or out of range") != C_OK)
+            "timeout is not a float or out of range") != C_OK)
             return C_ERR;
         tval = (long long) (ftval * 1000.0);
     } else {

From 17bf8dc99c34ac99fc2b74d03f8b7e6230f452e8 Mon Sep 17 00:00:00 2001
From: Jamie Scott <jamie@redislabs.com>
Date: Sun, 12 Apr 2020 00:10:19 -0700
Subject: [PATCH 0267/1098] Adding acllog-max-len to Redis.conf

While playing with ACLs I noticed that acllog-max-len wasn't in the redis.conf, but was a supported config.

This PR documents and adds the directive to the redis.conf file.
---
 redis.conf | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/redis.conf b/redis.conf
index 5baeae65f..fab0a5898 100644
--- a/redis.conf
+++ b/redis.conf
@@ -737,6 +737,15 @@ replica-priority 100
 # For more information about ACL configuration please refer to
 # the Redis web site at https://redis.io/topics/acl
 
+# ACL LOG
+#
+# The ACL Log tracks failed commands and authentication events associated
+# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked 
+# by ACLs. The ACL Log is stored in and consumes memory. There is no limit
+# to its length.You can reclaim memory with ACL LOG RESET or set a maximum
+# length below.
+acllog-max-len 128
+
 # Using an external ACL file
 #
 # Instead of configuring users here in this file, it is possible to use

From 4f3b15e6ea2d72debeb2f0533e2ba11884257021 Mon Sep 17 00:00:00 2001
From: Jamie Scott <jamie@redislabs.com>
Date: Sun, 12 Apr 2020 17:56:58 -0700
Subject: [PATCH 0268/1098] minor fix

---
 redis.conf | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/redis.conf b/redis.conf
index fab0a5898..01a2d801f 100644
--- a/redis.conf
+++ b/redis.conf
@@ -741,9 +741,8 @@ replica-priority 100
 #
 # The ACL Log tracks failed commands and authentication events associated
 # with ACLs. The ACL Log is useful to troubleshoot failed commands blocked 
-# by ACLs. The ACL Log is stored in and consumes memory. There is no limit
-# to its length.You can reclaim memory with ACL LOG RESET or set a maximum
-# length below.
+# by ACLs. The ACL Log is stored in memory. You can reclaim memory with 
+# ACL LOG RESET. Define the maximum entry length of the ACL Log below.
 acllog-max-len 128
 
 # Using an external ACL file

From a063c8c188427ee21082efae624c13a830d7b7f5 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Mon, 13 Apr 2020 17:28:11 +0300
Subject: [PATCH 0269/1098] Prevents default save configuration being reset...

...when using any command line argument
---
 src/server.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index 0afd67514..b07049851 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5035,7 +5035,6 @@ int main(int argc, char **argv) {
                 "Sentinel needs config file on disk to save state.  Exiting...");
             exit(1);
         }
-        resetServerSaveParams();
         loadServerConfig(configfile,options);
         sdsfree(options);
     }

From 1323afed8b42db83961a559754928d051f7bc2e3 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Tue, 14 Apr 2020 00:16:29 -0400
Subject: [PATCH 0270/1098] fix spelling in acl.c

---
 src/acl.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 733988013..6847130ad 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1783,8 +1783,8 @@ void aclCommand(client *c) {
         long count = 10; /* Number of entries to emit by default. */
 
         /* Parse the only argument that LOG may have: it could be either
-         * the number of entires the user wants to display, or alternatively
-         * the "RESET" command in order to flush the old entires. */
+         * the number of entries the user wants to display, or alternatively
+         * the "RESET" command in order to flush the old entries. */
         if (c->argc == 3) {
             if (!strcasecmp(c->argv[2]->ptr,"reset")) {
                 listSetFreeMethod(ACLLog,ACLFreeLogEntry);

From 49ccd2a8e1b0cdd6cad6cdcdaa52dab43aeb9345 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 14 Apr 2020 10:52:40 +0200
Subject: [PATCH 0271/1098] Fix function names in zslDeleteNode() top comment.

---
 src/t_zset.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/t_zset.c b/src/t_zset.c
index ea6f4b848..147313d53 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -186,7 +186,8 @@ zskiplistNode *zslInsert(zskiplist *zsl, double score, sds ele) {
     return x;
 }
 
-/* Internal function used by zslDelete, zslDeleteByScore and zslDeleteByRank */
+/* Internal function used by zslDelete, zslDeleteRangeByScore and
+ * zslDeleteRangeByRank. */
 void zslDeleteNode(zskiplist *zsl, zskiplistNode *x, zskiplistNode **update) {
     int i;
     for (i = 0; i < zsl->level; i++) {

From 086c1db2d998192d92d923ae2d2fba4f145e152f Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 14 Apr 2020 11:23:44 +0200
Subject: [PATCH 0272/1098] Fix zsetAdd() top comment spelling.

---
 src/t_zset.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/t_zset.c b/src/t_zset.c
index 147313d53..5c000e76f 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -1301,14 +1301,14 @@ int zsetScore(robj *zobj, sds member, double *score) {
  * none could be set if we re-added an element using the same score it used
  * to have, or in the case a zero increment is used).
  *
- * The function returns 0 on erorr, currently only when the increment
+ * The function returns 0 on error, currently only when the increment
  * produces a NAN condition, or when the 'score' value is NAN since the
  * start.
  *
- * The commad as a side effect of adding a new element may convert the sorted
+ * The command as a side effect of adding a new element may convert the sorted
  * set internal encoding from ziplist to hashtable+skiplist.
  *
- * Memory managemnet of 'ele':
+ * Memory management of 'ele':
  *
  * The function does not take ownership of the 'ele' SDS string, but copies
  * it if needed. */

From 9cf500a3f67e4e2ce51414c354e3472faf095d5b Mon Sep 17 00:00:00 2001
From: ShooterIT <wangyuancode@163.com>
Date: Tue, 14 Apr 2020 23:56:34 +0800
Subject: [PATCH 0273/1098] Implements sendfile for redis.

---
 src/config.h      |  6 ++++++
 src/replication.c | 51 +++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 55 insertions(+), 2 deletions(-)

diff --git a/src/config.h b/src/config.h
index efa9d11f2..058d73932 100644
--- a/src/config.h
+++ b/src/config.h
@@ -133,6 +133,12 @@ void setproctitle(const char *fmt, ...);
 /* Byte ordering detection */
 #include <sys/types.h> /* This will likely define BYTE_ORDER */
 
+/* Define redis_sendfile. */
+#if defined(__linux__) || (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_5))
+#define HAVE_SENDFILE 1
+ssize_t redis_sendfile(int out_fd, int in_fd, off_t offset, size_t count);
+#endif
+
 #ifndef BYTE_ORDER
 #if (BSD >= 199103)
 # include <machine/endian.h>
diff --git a/src/replication.c b/src/replication.c
index 3e9910374..4a3bbbdcf 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -972,10 +972,41 @@ void removeRDBUsedToSyncReplicas(void) {
     }
 }
 
+#if HAVE_SENDFILE
+/* Implements redis_sendfile to transfer data between file descriptors and
+ * avoid transferring data to and from user space.
+ * 
+ * The function prototype is just like sendfile(2) on Linux. in_fd is a file
+ * descriptor opened for reading and out_fd is a descriptor opened for writing.
+ * offset specifies where to start reading data from in_fd. count is the number
+ * of bytes to copy between the file descriptors.
+ * 
+ * The return value is the number of bytes written to out_fd, if the transfer
+ * was successful. On error, -1 is returned, and errno is set appropriately. */
+ssize_t redis_sendfile(int out_fd, int in_fd, off_t offset, size_t count) {
+#if defined(__linux__)
+    #include <sys/sendfile.h>
+    return sendfile(out_fd, in_fd, &offset, count);
+
+#elif defined(__APPLE__)
+    off_t len = count;
+    /* Notice that it may return -1 and errno is set to EAGAIN even if some
+     * bytes have been sent successfully and the len argument is set correctly
+     * when using a socket marked for non-blocking I/O. */
+    if (sendfile(in_fd, out_fd, offset, &len, NULL, 0) == -1 &&
+        errno != EAGAIN) return -1;
+    else
+        return (ssize_t)len;
+
+#endif
+    errno = ENOSYS;
+    return -1;
+}
+#endif
+
 void sendBulkToSlave(connection *conn) {
     client *slave = connGetPrivateData(conn);
-    char buf[PROTO_IOBUF_LEN];
-    ssize_t nwritten, buflen;
+    ssize_t nwritten;
 
     /* Before sending the RDB file, we send the preamble as configured by the
      * replication process. Currently the preamble is just the bulk count of
@@ -1001,6 +1032,21 @@ void sendBulkToSlave(connection *conn) {
     }
 
     /* If the preamble was already transferred, send the RDB bulk data. */
+#if HAVE_SENDFILE
+    if ((nwritten = redis_sendfile(conn->fd,slave->repldbfd,
+        slave->repldboff,PROTO_IOBUF_LEN)) == -1)
+    {
+        if (errno != EAGAIN) {
+            serverLog(LL_WARNING,"Sendfile error sending DB to replica: %s",
+                strerror(errno));
+            freeClient(slave);
+        }
+        return;
+    }
+#else
+    ssize_t buflen;
+    char buf[PROTO_IOBUF_LEN];
+
     lseek(slave->repldbfd,slave->repldboff,SEEK_SET);
     buflen = read(slave->repldbfd,buf,PROTO_IOBUF_LEN);
     if (buflen <= 0) {
@@ -1017,6 +1063,7 @@ void sendBulkToSlave(connection *conn) {
         }
         return;
     }
+#endif
     slave->repldboff += nwritten;
     server.stat_net_output_bytes += nwritten;
     if (slave->repldboff == slave->repldbsize) {

From 7f794a122f0bd5f0babf504a80484a533742e8e9 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 15 Apr 2020 15:59:52 +0200
Subject: [PATCH 0274/1098] Fix HELLO reply in Sentinel mode, see #6160.

---
 src/networking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index 654fda517..3dd6bf5da 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2460,7 +2460,7 @@ void helloCommand(client *c) {
 
     addReplyBulkCString(c,"mode");
     if (server.sentinel_mode) addReplyBulkCString(c,"sentinel");
-    if (server.cluster_enabled) addReplyBulkCString(c,"cluster");
+    else if (server.cluster_enabled) addReplyBulkCString(c,"cluster");
     else addReplyBulkCString(c,"standalone");
 
     if (!server.sentinel_mode) {

From 3519a5a026be50022fb4e103ddc602ffd59daf42 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 15 Apr 2020 16:12:06 +0200
Subject: [PATCH 0275/1098] Don't allow empty spaces in ACL key patterns.

Fixes issue #6418.
---
 src/acl.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/acl.c b/src/acl.c
index 6847130ad..a5e35c4d1 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -30,6 +30,7 @@
 #include "server.h"
 #include "sha256.h"
 #include <fcntl.h>
+#include <ctype.h>
 
 /* =============================================================================
  * Global state for ACLs
@@ -690,7 +691,8 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) {
  *
  * When an error is returned, errno is set to the following values:
  *
- * EINVAL: The specified opcode is not understood.
+ * EINVAL: The specified opcode is not understood or the key pattern is
+ *         invalid (contains non allowed characters).
  * ENOENT: The command name or command category provided with + or - is not
  *         known.
  * EBUSY:  The subcommand you want to add is about a command that is currently
@@ -789,6 +791,15 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
             errno = EEXIST;
             return C_ERR;
         }
+        /* Validate the pattern: no spaces nor null characters
+         * are allowed, for simpler rewriting of the ACLs without
+         * using quoting. */
+        for (int i = 1; i < oplen; i++) {
+            if (isspace(op[i]) || op[i] == 0) {
+                errno = EINVAL;
+                return C_ERR;
+            }
+        }
         sds newpat = sdsnewlen(op+1,oplen-1);
         listNode *ln = listSearchKey(u->patterns,newpat);
         /* Avoid re-adding the same pattern multiple times. */

From 503a5a24fb6cf3bd95590a53d14fce82086be52c Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 15 Apr 2020 16:39:42 +0200
Subject: [PATCH 0276/1098] Don't allow empty spaces in ACL usernames.

Fixes issue #6418.
---
 src/acl.c | 44 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index a5e35c4d1..75b954c5e 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -170,6 +170,18 @@ sds ACLHashPassword(unsigned char *cleartext, size_t len) {
  * Low level ACL API
  * ==========================================================================*/
 
+/* Return 1 if the specified string contains spaces or null characters.
+ * We do this for usernames and key patterns for simpler rewriting of
+ * ACL rules, presentation on ACL list, and to avoid subtle security bugs
+ * that may arise from parsing the rules in presence of escapes.
+ * The function returns 0 if the string has no spaces. */
+int ACLStringHasSpaces(const char *s, size_t len) {
+    for (size_t i = 0; i < len; i++) {
+        if (isspace(s[i]) || s[i] == 0) return 1;
+    }
+    return 0;
+}
+
 /* Given the category name the command returns the corresponding flag, or
  * zero if there is no match. */
 uint64_t ACLGetCommandCategoryFlagByName(const char *name) {
@@ -791,14 +803,9 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
             errno = EEXIST;
             return C_ERR;
         }
-        /* Validate the pattern: no spaces nor null characters
-         * are allowed, for simpler rewriting of the ACLs without
-         * using quoting. */
-        for (int i = 1; i < oplen; i++) {
-            if (isspace(op[i]) || op[i] == 0) {
-                errno = EINVAL;
-                return C_ERR;
-            }
+        if (ACLStringHasSpaces(op+1,oplen-1)) {
+            errno = EINVAL;
+            return C_ERR;
         }
         sds newpat = sdsnewlen(op+1,oplen-1);
         listNode *ln = listSearchKey(u->patterns,newpat);
@@ -1175,6 +1182,12 @@ int ACLLoadConfiguredUsers(void) {
     while ((ln = listNext(&li)) != NULL) {
         sds *aclrules = listNodeValue(ln);
         sds username = aclrules[0];
+
+        if (ACLStringHasSpaces(aclrules[0],sdslen(aclrules[0]))) {
+            serverLog(LL_WARNING,"Spaces not allowed in ACL usernames");
+            return C_ERR;
+        }
+
         user *u = ACLCreateUser(username,sdslen(username));
         if (!u) {
             u = ACLGetUserByName(username,sdslen(username));
@@ -1300,6 +1313,14 @@ sds ACLLoadFromFile(const char *filename) {
             continue;
         }
 
+        /* Spaces are not allowed in usernames. */
+        if (ACLStringHasSpaces(argv[1],sdslen(argv[1]))) {
+            errors = sdscatprintf(errors,
+                     "'%s:%d: username '%s' contains invalid characters. ",
+                     server.acl_filename, linenum, argv[1]);
+            continue;
+        }
+
         /* Try to process the line using the fake user to validate iif
          * the rules are able to apply cleanly. */
         ACLSetUser(fakeuser,"reset",-1);
@@ -1609,6 +1630,13 @@ void aclCommand(client *c) {
     char *sub = c->argv[1]->ptr;
     if (!strcasecmp(sub,"setuser") && c->argc >= 3) {
         sds username = c->argv[2]->ptr;
+        /* Check username validity. */
+        if (ACLStringHasSpaces(username,sdslen(username))) {
+            addReplyErrorFormat(c,
+                "Usernames can't contain spaces or null characters");
+            return;
+        }
+
         /* Create a temporary user to validate and stage all changes against
          * before applying to an existing user or creating a new user. If all
          * arguments are valid the user parameters will all be applied together.

From 12b9fd4f03657e2e22e76f3fda851a874bb4dab7 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Wed, 15 Apr 2020 22:00:36 -0400
Subject: [PATCH 0277/1098] Fix not used marco in cluster.c

---
 src/cluster.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cluster.c b/src/cluster.c
index 2377b386b..eac5ebc54 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -2106,7 +2106,7 @@ int clusterProcessPacket(clusterLink *link) {
         resetManualFailover();
         server.cluster->mf_end = mstime() + CLUSTER_MF_TIMEOUT;
         server.cluster->mf_slave = sender;
-        pauseClients(mstime()+(CLUSTER_MF_TIMEOUT*2));
+        pauseClients(mstime()+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT));
         serverLog(LL_WARNING,"Manual failover requested by replica %.40s.",
             sender->name);
     } else if (type == CLUSTERMSG_TYPE_UPDATE) {

From c9388ecc73c64491b0c5aef2c6a65068ddd6b2c7 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 16 Apr 2020 11:21:52 +0200
Subject: [PATCH 0278/1098] RESP3: fix HELLO map len in Sentinel mode.

See #6160.
---
 src/networking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index 3dd6bf5da..8f3d79170 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2444,7 +2444,7 @@ void helloCommand(client *c) {
 
     /* Let's switch to the specified RESP mode. */
     c->resp = ver;
-    addReplyMapLen(c,7);
+    addReplyMapLen(c,6 + !server.sentinel_mode);
 
     addReplyBulkCString(c,"server");
     addReplyBulkCString(c,"redis");

From 37b135a4000ae2a83afd5d0d59022c6b868cd001 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 16 Apr 2020 16:08:37 +0200
Subject: [PATCH 0279/1098] Update SDS to latest version.

---
 src/sds.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sds.c b/src/sds.c
index 98bd2e77f..118971621 100644
--- a/src/sds.c
+++ b/src/sds.c
@@ -97,11 +97,11 @@ sds sdsnewlen(const void *init, size_t initlen) {
     unsigned char *fp; /* flags pointer. */
 
     sh = s_malloc(hdrlen+initlen+1);
+    if (sh == NULL) return NULL;
     if (init==SDS_NOINIT)
         init = NULL;
     else if (!init)
         memset(sh, 0, hdrlen+initlen+1);
-    if (sh == NULL) return NULL;
     s = (char*)sh+hdrlen;
     fp = ((unsigned char*)s)-1;
     switch(type) {

From b9fa42a197702c0da0baa2edb184d984dffa2933 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 16 Apr 2020 11:05:03 +0300
Subject: [PATCH 0280/1098] testsuite run the defrag latency test solo

this test is time sensitive and it sometimes fail to pass below the
latency threshold, even on strong machines.

this test was the reson we're running just 2 parallel tests in the
github actions CI, revering this.
---
 .github/workflows/ci.yml     |  4 ++--
 tests/test_helper.tcl        | 38 ++++++++++++++++++++++++++++++++++++
 tests/unit/memefficiency.tcl |  2 ++
 3 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3a81d1a08..551fb2d91 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,9 +12,9 @@ jobs:
     - name: test
       run: |
         sudo apt-get install tcl8.5
-        ./runtest --clients 2 --verbose
+        ./runtest --verbose
     - name: module api test
-      run: ./runtest-moduleapi --clients 2 --verbose
+      run: ./runtest-moduleapi --verbose
 
   build-ubuntu-old:
     runs-on: ubuntu-16.04
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index d80cb6907..11a804bdc 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -82,6 +82,7 @@ set ::skiptests {}
 set ::allowtags {}
 set ::only_tests {}
 set ::single_tests {}
+set ::run_solo_tests {}
 set ::skip_till ""
 set ::external 0; # If "1" this means, we are running against external instance
 set ::file ""; # If set, runs only the tests in this comma separated list
@@ -112,6 +113,11 @@ proc execute_tests name {
     send_data_packet $::test_server_fd done "$name"
 }
 
+proc execute_code {name code} {
+    eval $code
+    send_data_packet $::test_server_fd done "$name"
+}
+
 # Setup a list to hold a stack of server configs. When calls to start_server
 # are nested, use "srv 0 pid" to get the pid of the inner server. To access
 # outer servers, use "srv -1 pid" etcetera.
@@ -188,6 +194,18 @@ proc s {args} {
     status [srv $level "client"] [lindex $args 0]
 }
 
+# Test wrapped into run_solo are sent back from the client to the
+# test server, so that the test server will send them again to
+# clients once the clients are idle.
+proc run_solo {name code} {
+    if {$::numclients == 1 || $::loop || $::external} {
+        # run_solo is not supported in these scenarios, just run the code.
+        eval $code
+        return
+    }
+    send_data_packet $::test_server_fd run_solo [list $name $code]
+}
+
 proc cleanup {} {
     if {$::dont_clean} {
         return
@@ -337,6 +355,8 @@ proc read_from_test_client fd {
     } elseif {$status eq {server-killed}} {
         set ::active_servers [lsearch -all -inline -not -exact $::active_servers $data]
         set ::active_clients_task($fd) "(KILLED SERVER) pid:$data"
+    } elseif {$status eq {run_solo}} {
+        lappend ::run_solo_tests $data
     } else {
         if {!$::quiet} {
             puts "\[$status\]: $data"
@@ -369,6 +389,13 @@ proc force_kill_all_servers {} {
     }
 }
 
+proc lpop {listVar {count 1}} {
+    upvar 1 $listVar l
+    set ele [lindex $l 0]
+    set l [lrange $l 1 end]
+    set ele
+}
+
 # A new client is idle. Remove it from the list of active clients and
 # if there are still test units to run, launch them.
 proc signal_idle_client fd {
@@ -389,6 +416,14 @@ proc signal_idle_client fd {
         if {$::loop && $::next_test == [llength $::all_tests]} {
             set ::next_test 0
         }
+    } elseif {[llength $::run_solo_tests] != 0 && [llength $::active_clients] == 0} {
+        if {!$::quiet} {
+            puts [colorstr bold-white "Testing solo test"]
+            set ::active_clients_task($fd) "ASSIGNED: $fd solo test"
+        }
+        set ::clients_start_time($fd) [clock seconds]
+        send_data_packet $fd run_code [lpop ::run_solo_tests]
+        lappend ::active_clients $fd
     } else {
         lappend ::idle_clients $fd
         set ::active_clients_task($fd) "SLEEPING, no more units to assign"
@@ -433,6 +468,9 @@ proc test_client_main server_port {
         foreach {cmd data} $payload break
         if {$cmd eq {run}} {
             execute_tests $data
+        } elseif {$cmd eq {run_code}} {
+            foreach {name code} $data break
+            execute_code $name $code
         } else {
             error "Unknown test client command: $cmd"
         }
diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl
index 06b0e07d7..777693fdf 100644
--- a/tests/unit/memefficiency.tcl
+++ b/tests/unit/memefficiency.tcl
@@ -36,6 +36,7 @@ start_server {tags {"memefficiency"}} {
     }
 }
 
+run_solo {defrag} {
 start_server {tags {"defrag"}} {
     if {[string match {*jemalloc*} [s mem_allocator]]} {
         test "Active defrag" {
@@ -328,3 +329,4 @@ start_server {tags {"defrag"}} {
         } {1}
     }
 }
+} ;# run_solo

From 002052f8dee82b206e69b828de6cd63ee466dfec Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 17 Apr 2020 10:51:12 +0200
Subject: [PATCH 0281/1098] A few comments and name changes for #7103.

---
 tests/test_helper.tcl | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 11a804bdc..3eee1aeb7 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -106,14 +106,23 @@ set ::tlsdir "tests/tls"
 set ::client 0
 set ::numclients 16
 
-proc execute_tests name {
+# This function is called by one of the test clients when it receives
+# a "run" command from the server, with a filename as data.
+# It will run the specified test source file and signal it to the
+# test server when finished.
+proc execute_test_file name {
     set path "tests/$name.tcl"
     set ::curfile $path
     source $path
     send_data_packet $::test_server_fd done "$name"
 }
 
-proc execute_code {name code} {
+# This function is called by one of the test clients when it receives
+# a "run_code" command from the server, with a verbatim test source code
+# as argument, and an associated name.
+# It will run the specified code and signal it to the test server when
+# finished.
+proc execute_test_code {name code} {
     eval $code
     send_data_packet $::test_server_fd done "$name"
 }
@@ -467,10 +476,10 @@ proc test_client_main server_port {
         set payload [read $::test_server_fd $bytes]
         foreach {cmd data} $payload break
         if {$cmd eq {run}} {
-            execute_tests $data
+            execute_test_file $data
         } elseif {$cmd eq {run_code}} {
             foreach {name code} $data break
-            execute_code $name $code
+            execute_test_code $name $code
         } else {
             error "Unknown test client command: $cmd"
         }

From c479eace4512193bcfe3dcab3ab238486f6f9405 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 17 Apr 2020 12:38:12 +0200
Subject: [PATCH 0282/1098] Fix XCLAIM propagation in AOF/replicas for blocking
 XREADGROUP.

See issue #7105.
---
 src/server.c   | 9 +++++++--
 src/t_stream.c | 2 +-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/server.c b/src/server.c
index 0afd67514..fc9b87aae 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3082,8 +3082,13 @@ struct redisCommand *lookupCommandOrOriginal(sds name) {
  * + PROPAGATE_AOF (propagate into the AOF file if is enabled)
  * + PROPAGATE_REPL (propagate into the replication link)
  *
- * This should not be used inside commands implementation. Use instead
- * alsoPropagate(), preventCommandPropagation(), forceCommandPropagation().
+ * This should not be used inside commands implementation since it will not
+ * wrap the resulting commands in MULTI/EXEC. Use instead alsoPropagate(),
+ * preventCommandPropagation(), forceCommandPropagation().
+ *
+ * However for functions that need to (also) propagate out of the context of a
+ * command execution, for example when serving a blocked client, you
+ * want to use propagate().
  */
 void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc,
                int flags)
diff --git a/src/t_stream.c b/src/t_stream.c
index 4ce3a9b25..155167af9 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -848,7 +848,7 @@ void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupnam
     argv[11] = createStringObject("JUSTID",6);
     argv[12] = createStringObject("LASTID",6);
     argv[13] = createObjectFromStreamID(&group->last_id);
-    alsoPropagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL);
+    propagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL);
     decrRefCount(argv[0]);
     decrRefCount(argv[3]);
     decrRefCount(argv[4]);

From 9d27e00ddb7f49d9af104ab66e87c2a626d68ab8 Mon Sep 17 00:00:00 2001
From: omg-by <504094596@qq.com>
Date: Sat, 18 Apr 2020 00:49:16 +0800
Subject: [PATCH 0283/1098] fix(sentinel): sentinel.running_scripts will always
 increase more times and not reset

when trigger a always fail scripts, sentinel.running_scripts will increase ten times, however it
only decrease one times onretry the maximum. and it will't reset, when it become
SENTINEL_SCRIPT_MAX_RUNNING, sentinel don't trigger scripts.
---
 src/sentinel.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/sentinel.c b/src/sentinel.c
index 10c003d03..7a25e5ae4 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -859,6 +859,7 @@ void sentinelCollectTerminatedScripts(void) {
             sj->pid = 0;
             sj->start_time = mstime() +
                              sentinelScriptRetryDelay(sj->retry_num);
+            sentinel.running_scripts--;
         } else {
             /* Otherwise let's remove the script, but log the event if the
              * execution did not terminated in the best of the ways. */

From 5010da6ab068cab4ee19591ebaf7112b9ec5f375 Mon Sep 17 00:00:00 2001
From: zhenwei pi <pizhenwei@bytedance.com>
Date: Mon, 13 Apr 2020 09:58:35 +0800
Subject: [PATCH 0284/1098] Threaded IO: set thread name for redis-server

Set thread name for each thread of redis-server, this helps us to
monitor the utilization and optimise the performance.

And suggested-by Salvatore, implement this feature for multi
platforms. Currently support linux and bsd, ignore other OS.

An exmaple on Linux:
 # top -d 5 -p `pidof redis-server ` -H

    PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
3682671 root      20   0  227744   8248   3836 R 99.2  0.0   0:19.53 redis-server
3682677 root      20   0  227744   8248   3836 S 26.4  0.0   0:04.15 io_thd_3
3682675 root      20   0  227744   8248   3836 S 23.6  0.0   0:03.98 io_thd_1
3682676 root      20   0  227744   8248   3836 S 23.6  0.0   0:03.97 io_thd_2
3682672 root      20   0  227744   8248   3836 S  0.2  0.0   0:00.02 bio_close_file
3682673 root      20   0  227744   8248   3836 S  0.2  0.0   0:00.02 bio_aof_fsync
3682674 root      20   0  227744   8248   3836 S  0.0  0.0   0:00.00 bio_lazy_free
3682678 root      20   0  227744   8248   3836 S  0.0  0.0   0:00.00 jemalloc_bg_thd
3682682 root      20   0  227744   8248   3836 S  0.0  0.0   0:00.00 jemalloc_bg_thd
3682683 root      20   0  227744   8248   3836 S  0.0  0.0   0:00.00 jemalloc_bg_thd
3682684 root      20   0  227744   8248   3836 S  0.0  0.0   0:00.00 jemalloc_bg_thd
3682685 root      20   0  227744   8248   3836 S  0.0  0.0   0:00.00 jemalloc_bg_thd
3682687 root      20   0  227744   8248   3836 S  0.0  0.0   0:00.00 jemalloc_bg_thd

Another exmaple on FreeBSD-12.1:
  PID USERNAME    PRI NICE   SIZE    RES STATE    C   TIME    WCPU COMMAND
 5212 root        100    0    48M  7280K CPU2     2   0:26  99.52% redis-server{redis-server}
 5212 root         38    0    48M  7280K umtxn    4   0:06  26.94% redis-server{io_thd_3}
 5212 root         36    0    48M  7280K umtxn    6   0:06  26.84% redis-server{io_thd_1}
 5212 root         39    0    48M  7280K umtxn    1   0:06  25.30% redis-server{io_thd_2}
 5212 root         20    0    48M  7280K uwait    3   0:00   0.00% redis-server{redis-server}
 5212 root         21    0    48M  7280K uwait    2   0:00   0.00% redis-server{bio_close_file}
 5212 root         21    0    48M  7280K uwait    3   0:00   0.00% redis-server{bio_aof_fsync}
 5212 root         21    0    48M  7280K uwait    0   0:00   0.00% redis-server{bio_lazy_free}

Signed-off-by: zhenwei pi <pizhenwei@bytedance.com>
---
 src/bio.c        | 12 ++++++++++++
 src/config.h     | 12 ++++++++++++
 src/networking.c |  4 ++++
 3 files changed, 28 insertions(+)

diff --git a/src/bio.c b/src/bio.c
index 2af684570..0662c8c4c 100644
--- a/src/bio.c
+++ b/src/bio.c
@@ -154,6 +154,18 @@ void *bioProcessBackgroundJobs(void *arg) {
         return NULL;
     }
 
+    switch (type) {
+    case BIO_CLOSE_FILE:
+        redis_set_thread_title("bio_close_file");
+        break;
+    case BIO_AOF_FSYNC:
+        redis_set_thread_title("bio_aof_fsync");
+        break;
+    case BIO_LAZY_FREE:
+        redis_set_thread_title("bio_lazy_free");
+        break;
+    }
+
     /* Make the thread killable at any time, so that bioKillThreads()
      * can work reliably. */
     pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
diff --git a/src/config.h b/src/config.h
index efa9d11f2..82dc201d3 100644
--- a/src/config.h
+++ b/src/config.h
@@ -226,4 +226,16 @@ void setproctitle(const char *fmt, ...);
 #define USE_ALIGNED_ACCESS
 #endif
 
+/* Define for redis_set_thread_title */
+#ifdef __linux__
+#define redis_set_thread_title(name) pthread_setname_np(pthread_self(), name)
+#else
+#if (defined __NetBSD__ || defined __FreeBSD__ || defined __OpenBSD__)
+#include <pthread_np.h>
+#define redis_set_thread_title(name) pthread_set_name_np(pthread_self(), name)
+#else
+#define redis_set_thread_title(name)
+#endif
+#endif
+
 #endif
diff --git a/src/networking.c b/src/networking.c
index 8f3d79170..1f5d0bd5d 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2821,6 +2821,10 @@ void *IOThreadMain(void *myid) {
     /* The ID is the thread number (from 0 to server.iothreads_num-1), and is
      * used by the thread to just manipulate a single sub-array of clients. */
     long id = (unsigned long)myid;
+    char thdname[16];
+
+    snprintf(thdname, sizeof(thdname), "io_thd_%ld", id);
+    redis_set_thread_title(thdname);
 
     while(1) {
         /* Wait for start */

From 1bc557c9c5985b63f71c122e1219878dced7cb67 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Sun, 19 Apr 2020 15:59:58 +0300
Subject: [PATCH 0285/1098] Add the stream tag to XSETID tests

---
 tests/unit/type/stream.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl
index 5de9f0571..c2b524d7f 100644
--- a/tests/unit/type/stream.tcl
+++ b/tests/unit/type/stream.tcl
@@ -414,7 +414,7 @@ start_server {tags {"stream"} overrides {appendonly yes stream-node-max-entries
     }
 }
 
-start_server {tags {"xsetid"}} {
+start_server {tags {"stream xsetid"}} {
     test {XADD can CREATE an empty stream} {
         r XADD mystream MAXLEN 0 * a b
         assert {[dict get [r xinfo stream mystream] length] == 0}

From 23219392184ef605cab26323a94ae031273bc6e7 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 20 Apr 2020 11:52:29 +0200
Subject: [PATCH 0286/1098] Sentinel: small refactoring of
 sentinelCollectTerminatedScripts().

Related to #7113.
---
 src/sentinel.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/sentinel.c b/src/sentinel.c
index 204a8b317..fb504ae4d 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -860,7 +860,6 @@ void sentinelCollectTerminatedScripts(void) {
             sj->pid = 0;
             sj->start_time = mstime() +
                              sentinelScriptRetryDelay(sj->retry_num);
-            sentinel.running_scripts--;
         } else {
             /* Otherwise let's remove the script, but log the event if the
              * execution did not terminated in the best of the ways. */
@@ -870,8 +869,8 @@ void sentinelCollectTerminatedScripts(void) {
             }
             listDelNode(sentinel.scripts_queue,ln);
             sentinelReleaseScriptJob(sj);
-            sentinel.running_scripts--;
         }
+        sentinel.running_scripts--;
     }
 }
 

From c7db333abb0e45ff8974ef2d1fc4f9ae1e7be1e2 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 20 Apr 2020 12:17:11 +0200
Subject: [PATCH 0287/1098] Implement redis_set_thread_title for MacOS.

Strange enough, pthread_setname_np() produces a warning for not defined
function even if pthread is included. Moreover the MacOS documentation
claims the return value for the function is void, but actually is int.

Related to #7089.
---
 src/config.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/config.h b/src/config.h
index 82dc201d3..40dc683ce 100644
--- a/src/config.h
+++ b/src/config.h
@@ -234,8 +234,14 @@ void setproctitle(const char *fmt, ...);
 #include <pthread_np.h>
 #define redis_set_thread_title(name) pthread_set_name_np(pthread_self(), name)
 #else
+#if (defined __APPLE__ && defined(MAC_OS_X_VERSION_10_7))
+int pthread_setname_np(const char *name);
+#include <pthread.h>
+#define redis_set_thread_title(name) pthread_setname_np(name)
+#else
 #define redis_set_thread_title(name)
 #endif
 #endif
+#endif
 
 #endif

From c8a720d17dd81ccbcb513c59806c788479f8bb4a Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 20 Apr 2020 13:34:37 +0300
Subject: [PATCH 0288/1098] TLS: Fix build on older verisons of OpenSSL.

---
 src/tls.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/tls.c b/src/tls.c
index 5fac6902b..ea0c34469 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -168,7 +168,9 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
 
     SSL_CTX_set_mode(ctx, SSL_MODE_ENABLE_PARTIAL_WRITE|SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
     SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);
+#if defined(SSL_CTX_set_ecdh_auto)
     SSL_CTX_set_ecdh_auto(ctx, 1);
+#endif
 
     if (SSL_CTX_use_certificate_file(ctx, ctx_config->cert_file, SSL_FILETYPE_PEM) <= 0) {
         ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf));

From 071197fb64ed618fdff1c6974b2094b9d9fdce10 Mon Sep 17 00:00:00 2001
From: Dave-in-lafayette <64047859+Dave-in-lafayette@users.noreply.github.com>
Date: Mon, 20 Apr 2020 16:34:36 -0700
Subject: [PATCH 0289/1098] fix for unintended crash during panic response

If redis crashes early, before lua is set up (like, if File Descriptor 0 is closed before exec), it will crash again trying to print memory statistics.
---
 src/server.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index fc9b87aae..ebd8c61c1 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4037,7 +4037,7 @@ sds genRedisInfoString(const char *section) {
         size_t zmalloc_used = zmalloc_used_memory();
         size_t total_system_mem = server.system_memory_size;
         const char *evict_policy = evictPolicyToString();
-        long long memory_lua = (long long)lua_gc(server.lua,LUA_GCCOUNT,0)*1024;
+        long long memory_lua = server.lua ? (long long)lua_gc(server.lua,LUA_GCCOUNT,0)*1024 : 0;
         struct redisMemOverhead *mh = getMemoryOverheadData();
 
         /* Peak memory is updated from time to time by serverCron() so it

From 80b6f9b0cb46666a14d471248cc3747930a28dd4 Mon Sep 17 00:00:00 2001
From: Dave-in-lafayette <64047859+Dave-in-lafayette@users.noreply.github.com>
Date: Mon, 20 Apr 2020 16:38:06 -0700
Subject: [PATCH 0290/1098] fix for crash during panic before all threads are
 up

If there's a panic before all threads have been started (say, if file descriptor 0 is closed at exec), the panic response will crash here again.
---
 src/bio.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/bio.c b/src/bio.c
index 0662c8c4c..85f681185 100644
--- a/src/bio.c
+++ b/src/bio.c
@@ -266,7 +266,7 @@ void bioKillThreads(void) {
     int err, j;
 
     for (j = 0; j < BIO_NUM_OPS; j++) {
-        if (pthread_cancel(bio_threads[j]) == 0) {
+        if (bio_threads[j] && pthread_cancel(bio_threads[j]) == 0) {
             if ((err = pthread_join(bio_threads[j],NULL)) != 0) {
                 serverLog(LL_WARNING,
                     "Bio thread for job type #%d can be joined: %s",

From 94f2e7f9f9f7e6eca8f8bd7ae412c34806e68351 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 21 Apr 2020 10:51:46 +0200
Subject: [PATCH 0291/1098] Tracking: NOLOOP internals implementation.

---
 src/bitops.c      |   8 ++--
 src/cluster.c     |   4 +-
 src/db.c          |  30 +++++++-----
 src/debug.c       |   2 +-
 src/expire.c      |   6 +--
 src/geo.c         |   4 +-
 src/hyperloglog.c |   6 +--
 src/module.c      |  10 ++--
 src/server.h      |  12 +++--
 src/sort.c        |   4 +-
 src/t_hash.c      |  10 ++--
 src/t_list.c      |  20 ++++----
 src/t_set.c       |  20 ++++----
 src/t_stream.c    |   6 +--
 src/t_string.c    |  14 +++---
 src/t_zset.c      |  12 ++---
 src/tracking.c    | 118 ++++++++++++++++++++++++++++++++++------------
 17 files changed, 174 insertions(+), 112 deletions(-)

diff --git a/src/bitops.c b/src/bitops.c
index 496d78b66..f506a881b 100644
--- a/src/bitops.c
+++ b/src/bitops.c
@@ -554,7 +554,7 @@ void setbitCommand(client *c) {
     byteval &= ~(1 << bit);
     byteval |= ((on & 0x1) << bit);
     ((uint8_t*)o->ptr)[byte] = byteval;
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id);
     server.dirty++;
     addReply(c, bitval ? shared.cone : shared.czero);
@@ -754,11 +754,11 @@ void bitopCommand(client *c) {
     /* Store the computed value into the target key */
     if (maxlen) {
         o = createObject(OBJ_STRING,res);
-        setKey(c->db,targetkey,o);
+        setKey(c,c->db,targetkey,o);
         notifyKeyspaceEvent(NOTIFY_STRING,"set",targetkey,c->db->id);
         decrRefCount(o);
     } else if (dbDelete(c->db,targetkey)) {
-        signalModifiedKey(c->db,targetkey);
+        signalModifiedKey(c,c->db,targetkey);
         notifyKeyspaceEvent(NOTIFY_GENERIC,"del",targetkey,c->db->id);
     }
     server.dirty++;
@@ -1135,7 +1135,7 @@ void bitfieldGeneric(client *c, int flags) {
     }
 
     if (changes) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id);
         server.dirty += changes;
     }
diff --git a/src/cluster.c b/src/cluster.c
index 2377b386b..b103e2fe1 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4982,7 +4982,7 @@ void restoreCommand(client *c) {
         setExpire(c,c->db,c->argv[1],ttl);
     }
     objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock,1000);
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_GENERIC,"restore",c->argv[1],c->db->id);
     addReply(c,shared.ok);
     server.dirty++;
@@ -5329,7 +5329,7 @@ try_again:
             if (!copy) {
                 /* No COPY option: remove the local key, signal the change. */
                 dbDelete(c->db,kv[j]);
-                signalModifiedKey(c->db,kv[j]);
+                signalModifiedKey(c,c->db,kv[j]);
                 notifyKeyspaceEvent(NOTIFY_GENERIC,"del",kv[j],c->db->id);
                 server.dirty++;
 
diff --git a/src/db.c b/src/db.c
index 59f0cc7a0..dc4a0b63e 100644
--- a/src/db.c
+++ b/src/db.c
@@ -238,8 +238,10 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) {
  * 3) The expire time of the key is reset (the key is made persistent),
  *    unless 'keepttl' is true.
  *
- * All the new keys in the database should be created via this interface. */
-void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal) {
+ * All the new keys in the database should be created via this interface.
+ * The client 'c' argument may be set to NULL if the operation is performed
+ * in a context where there is no clear client performing the operation. */
+void genericSetKey(client *c, redisDb *db, robj *key, robj *val, int keepttl, int signal) {
     if (lookupKeyWrite(db,key) == NULL) {
         dbAdd(db,key,val);
     } else {
@@ -247,12 +249,12 @@ void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal) {
     }
     incrRefCount(val);
     if (!keepttl) removeExpire(db,key);
-    if (signal) signalModifiedKey(db,key);
+    if (signal) signalModifiedKey(c,db,key);
 }
 
 /* Common case for genericSetKey() where the TTL is not retained. */
-void setKey(redisDb *db, robj *key, robj *val) {
-    genericSetKey(db,key,val,0,1);
+void setKey(client *c, redisDb *db, robj *key, robj *val) {
+    genericSetKey(c,db,key,val,0,1);
 }
 
 /* Return true if the specified key exists in the specified database.
@@ -467,9 +469,11 @@ long long dbTotalServerKeyCount() {
  * Every time a DB is flushed the function signalFlushDb() is called.
  *----------------------------------------------------------------------------*/
 
-void signalModifiedKey(redisDb *db, robj *key) {
+/* Note that the 'c' argument may be NULL if the key was modified out of
+ * a context of a client. */
+void signalModifiedKey(client *c, redisDb *db, robj *key) {
     touchWatchedKey(db,key);
-    trackingInvalidateKey(key);
+    trackingInvalidateKey(c,key);
 }
 
 void signalFlushedDb(int dbid) {
@@ -563,7 +567,7 @@ void delGenericCommand(client *c, int lazy) {
         int deleted  = lazy ? dbAsyncDelete(c->db,c->argv[j]) :
                               dbSyncDelete(c->db,c->argv[j]);
         if (deleted) {
-            signalModifiedKey(c->db,c->argv[j]);
+            signalModifiedKey(c,c->db,c->argv[j]);
             notifyKeyspaceEvent(NOTIFY_GENERIC,
                 "del",c->argv[j],c->db->id);
             server.dirty++;
@@ -1003,8 +1007,8 @@ void renameGenericCommand(client *c, int nx) {
     dbAdd(c->db,c->argv[2],o);
     if (expire != -1) setExpire(c,c->db,c->argv[2],expire);
     dbDelete(c->db,c->argv[1]);
-    signalModifiedKey(c->db,c->argv[1]);
-    signalModifiedKey(c->db,c->argv[2]);
+    signalModifiedKey(c,c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[2]);
     notifyKeyspaceEvent(NOTIFY_GENERIC,"rename_from",
         c->argv[1],c->db->id);
     notifyKeyspaceEvent(NOTIFY_GENERIC,"rename_to",
@@ -1072,8 +1076,8 @@ void moveCommand(client *c) {
 
     /* OK! key moved, free the entry in the source DB */
     dbDelete(src,c->argv[1]);
-    signalModifiedKey(src,c->argv[1]);
-    signalModifiedKey(dst,c->argv[1]);
+    signalModifiedKey(c,src,c->argv[1]);
+    signalModifiedKey(c,dst,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_GENERIC,
                 "move_from",c->argv[1],src->id);
     notifyKeyspaceEvent(NOTIFY_GENERIC,
@@ -1317,7 +1321,7 @@ int expireIfNeeded(redisDb *db, robj *key) {
         "expired",key,db->id);
     int retval = server.lazyfree_lazy_expire ? dbAsyncDelete(db,key) :
                                                dbSyncDelete(db,key);
-    if (retval) signalModifiedKey(db,key);
+    if (retval) signalModifiedKey(NULL,db,key);
     return retval;
 }
 
diff --git a/src/debug.c b/src/debug.c
index 1351b2536..cbb56cb71 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -588,7 +588,7 @@ NULL
                 memcpy(val->ptr, buf, valsize<=buflen? valsize: buflen);
             }
             dbAdd(c->db,key,val);
-            signalModifiedKey(c->db,key);
+            signalModifiedKey(c,c->db,key);
             decrRefCount(key);
         }
         addReply(c,shared.ok);
diff --git a/src/expire.c b/src/expire.c
index c102a01ff..30a27193d 100644
--- a/src/expire.c
+++ b/src/expire.c
@@ -64,7 +64,7 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) {
             dbSyncDelete(db,keyobj);
         notifyKeyspaceEvent(NOTIFY_EXPIRED,
             "expired",keyobj,db->id);
-        trackingInvalidateKey(keyobj);
+        trackingInvalidateKey(NULL,keyobj);
         decrRefCount(keyobj);
         server.stat_expiredkeys++;
         return 1;
@@ -519,14 +519,14 @@ void expireGenericCommand(client *c, long long basetime, int unit) {
         /* Replicate/AOF this as an explicit DEL or UNLINK. */
         aux = server.lazyfree_lazy_expire ? shared.unlink : shared.del;
         rewriteClientCommandVector(c,2,aux,key);
-        signalModifiedKey(c->db,key);
+        signalModifiedKey(c,c->db,key);
         notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
         addReply(c, shared.cone);
         return;
     } else {
         setExpire(c,c->db,key,when);
         addReply(c,shared.cone);
-        signalModifiedKey(c->db,key);
+        signalModifiedKey(c,c->db,key);
         notifyKeyspaceEvent(NOTIFY_GENERIC,"expire",key,c->db->id);
         server.dirty++;
         return;
diff --git a/src/geo.c b/src/geo.c
index f7920a2e2..3e5d5f606 100644
--- a/src/geo.c
+++ b/src/geo.c
@@ -657,13 +657,13 @@ void georadiusGeneric(client *c, int flags) {
 
         if (returned_items) {
             zsetConvertToZiplistIfNeeded(zobj,maxelelen);
-            setKey(c->db,storekey,zobj);
+            setKey(c,c->db,storekey,zobj);
             decrRefCount(zobj);
             notifyKeyspaceEvent(NOTIFY_ZSET,"georadiusstore",storekey,
                                 c->db->id);
             server.dirty += returned_items;
         } else if (dbDelete(c->db,storekey)) {
-            signalModifiedKey(c->db,storekey);
+            signalModifiedKey(c,c->db,storekey);
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",storekey,c->db->id);
             server.dirty++;
         }
diff --git a/src/hyperloglog.c b/src/hyperloglog.c
index facd99743..721f492a1 100644
--- a/src/hyperloglog.c
+++ b/src/hyperloglog.c
@@ -1209,7 +1209,7 @@ void pfaddCommand(client *c) {
     }
     hdr = o->ptr;
     if (updated) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_STRING,"pfadd",c->argv[1],c->db->id);
         server.dirty++;
         HLL_INVALIDATE_CACHE(hdr);
@@ -1300,7 +1300,7 @@ void pfcountCommand(client *c) {
              * data structure is not modified, since the cached value
              * may be modified and given that the HLL is a Redis string
              * we need to propagate the change. */
-            signalModifiedKey(c->db,c->argv[1]);
+            signalModifiedKey(c,c->db,c->argv[1]);
             server.dirty++;
         }
         addReplyLongLong(c,card);
@@ -1373,7 +1373,7 @@ void pfmergeCommand(client *c) {
                      last hllSparseSet() call. */
     HLL_INVALIDATE_CACHE(hdr);
 
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     /* We generate a PFADD event for PFMERGE for semantical simplicity
      * since in theory this is a mass-add of elements. */
     notifyKeyspaceEvent(NOTIFY_STRING,"pfadd",c->argv[1],c->db->id);
diff --git a/src/module.c b/src/module.c
index 4d3d9e1af..e3a338dad 100644
--- a/src/module.c
+++ b/src/module.c
@@ -896,7 +896,7 @@ void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) {
 /* Signals that the key is modified from user's perspective (i.e. invalidate WATCH
  * and client side caching). */
 int RM_SignalModifiedKey(RedisModuleCtx *ctx, RedisModuleString *keyname) {
-    signalModifiedKey(ctx->client->db,keyname);
+    signalModifiedKey(ctx->client,ctx->client->db,keyname);
     return REDISMODULE_OK;
 }
 
@@ -2016,7 +2016,7 @@ void *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) {
 static void moduleCloseKey(RedisModuleKey *key) {
     int signal = SHOULD_SIGNAL_MODIFIED_KEYS(key->ctx);
     if ((key->mode & REDISMODULE_WRITE) && signal)
-        signalModifiedKey(key->db,key->key);
+        signalModifiedKey(key->ctx->client,key->db,key->key);
     /* TODO: if (key->iter) RM_KeyIteratorStop(kp); */
     RM_ZsetRangeStop(key);
     decrRefCount(key->key);
@@ -2157,7 +2157,7 @@ RedisModuleString *RM_RandomKey(RedisModuleCtx *ctx) {
 int RM_StringSet(RedisModuleKey *key, RedisModuleString *str) {
     if (!(key->mode & REDISMODULE_WRITE) || key->iter) return REDISMODULE_ERR;
     RM_DeleteKey(key);
-    genericSetKey(key->db,key->key,str,0,0);
+    genericSetKey(key->ctx->client,key->db,key->key,str,0,0);
     key->value = str;
     return REDISMODULE_OK;
 }
@@ -2237,7 +2237,7 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) {
     if (key->value == NULL) {
         /* Empty key: create it with the new size. */
         robj *o = createObject(OBJ_STRING,sdsnewlen(NULL, newlen));
-        genericSetKey(key->db,key->key,o,0,0);
+        genericSetKey(key->ctx->client,key->db,key->key,o,0,0);
         key->value = o;
         decrRefCount(o);
     } else {
@@ -3625,7 +3625,7 @@ int RM_ModuleTypeSetValue(RedisModuleKey *key, moduleType *mt, void *value) {
     if (!(key->mode & REDISMODULE_WRITE) || key->iter) return REDISMODULE_ERR;
     RM_DeleteKey(key);
     robj *o = createModuleObject(mt,value);
-    genericSetKey(key->db,key->key,o,0,0);
+    genericSetKey(key->ctx->client,key->db,key->key,o,0,0);
     decrRefCount(o);
     key->value = o;
     return REDISMODULE_OK;
diff --git a/src/server.h b/src/server.h
index d0d5ff154..d39359dce 100644
--- a/src/server.h
+++ b/src/server.h
@@ -252,7 +252,9 @@ typedef long long ustime_t; /* microsecond time type. */
 #define CLIENT_TRACKING_OPTOUT (1ULL<<35) /* Tracking in opt-out mode. */
 #define CLIENT_TRACKING_CACHING (1ULL<<36) /* CACHING yes/no was given,
                                               depending on optin/optout mode. */
-#define CLIENT_IN_TO_TABLE (1ULL<<37) /* This client is in the timeout table. */
+#define CLIENT_TRACKING_NOLOOP (1ULL<<37) /* Don't send invalidation messages
+                                             about writes performed by myself.*/
+#define CLIENT_IN_TO_TABLE (1ULL<<38) /* This client is in the timeout table. */
 
 /* Client block type (btype field in client structure)
  * if CLIENT_BLOCKED flag is set. */
@@ -1683,7 +1685,7 @@ void addReplyStatusFormat(client *c, const char *fmt, ...);
 void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **prefix, size_t numprefix);
 void disableTracking(client *c);
 void trackingRememberKeys(client *c);
-void trackingInvalidateKey(robj *keyobj);
+void trackingInvalidateKey(client *c, robj *keyobj);
 void trackingInvalidateKeysOnFlush(int dbid);
 void trackingLimitUsedSlots(void);
 uint64_t trackingGetTotalItems(void);
@@ -2071,8 +2073,8 @@ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
 void dbAdd(redisDb *db, robj *key, robj *val);
 int dbAddRDBLoad(redisDb *db, sds key, robj *val);
 void dbOverwrite(redisDb *db, robj *key, robj *val);
-void genericSetKey(redisDb *db, robj *key, robj *val, int keepttl, int signal);
-void setKey(redisDb *db, robj *key, robj *val);
+void genericSetKey(client *c, redisDb *db, robj *key, robj *val, int keepttl, int signal);
+void setKey(client *c, redisDb *db, robj *key, robj *val);
 int dbExists(redisDb *db, robj *key);
 robj *dbRandomKey(redisDb *db);
 int dbSyncDelete(redisDb *db, robj *key);
@@ -2088,7 +2090,7 @@ void flushAllDataAndResetRDB(int flags);
 long long dbTotalServerKeyCount();
 
 int selectDb(client *c, int id);
-void signalModifiedKey(redisDb *db, robj *key);
+void signalModifiedKey(client *c, redisDb *db, robj *key);
 void signalFlushedDb(int dbid);
 unsigned int getKeysInSlot(unsigned int hashslot, robj **keys, unsigned int count);
 unsigned int countKeysInSlot(unsigned int hashslot);
diff --git a/src/sort.c b/src/sort.c
index db26da158..f269a7731 100644
--- a/src/sort.c
+++ b/src/sort.c
@@ -570,12 +570,12 @@ void sortCommand(client *c) {
             }
         }
         if (outputlen) {
-            setKey(c->db,storekey,sobj);
+            setKey(c,c->db,storekey,sobj);
             notifyKeyspaceEvent(NOTIFY_LIST,"sortstore",storekey,
                                 c->db->id);
             server.dirty += outputlen;
         } else if (dbDelete(c->db,storekey)) {
-            signalModifiedKey(c->db,storekey);
+            signalModifiedKey(c,c->db,storekey);
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",storekey,c->db->id);
             server.dirty++;
         }
diff --git a/src/t_hash.c b/src/t_hash.c
index b9f0db7fc..866bcd25b 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -521,7 +521,7 @@ void hsetnxCommand(client *c) {
     } else {
         hashTypeSet(o,c->argv[2]->ptr,c->argv[3]->ptr,HASH_SET_COPY);
         addReply(c, shared.cone);
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id);
         server.dirty++;
     }
@@ -551,7 +551,7 @@ void hsetCommand(client *c) {
         /* HMSET */
         addReply(c, shared.ok);
     }
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id);
     server.dirty++;
 }
@@ -586,7 +586,7 @@ void hincrbyCommand(client *c) {
     new = sdsfromlonglong(value);
     hashTypeSet(o,c->argv[2]->ptr,new,HASH_SET_TAKE_VALUE);
     addReplyLongLong(c,value);
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_HASH,"hincrby",c->argv[1],c->db->id);
     server.dirty++;
 }
@@ -625,7 +625,7 @@ void hincrbyfloatCommand(client *c) {
     new = sdsnewlen(buf,len);
     hashTypeSet(o,c->argv[2]->ptr,new,HASH_SET_TAKE_VALUE);
     addReplyBulkCBuffer(c,buf,len);
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_HASH,"hincrbyfloat",c->argv[1],c->db->id);
     server.dirty++;
 
@@ -721,7 +721,7 @@ void hdelCommand(client *c) {
         }
     }
     if (deleted) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_HASH,"hdel",c->argv[1],c->db->id);
         if (keyremoved)
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],
diff --git a/src/t_list.c b/src/t_list.c
index eaeaa8e48..4770a2272 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -217,7 +217,7 @@ void pushGenericCommand(client *c, int where) {
     if (pushed) {
         char *event = (where == LIST_HEAD) ? "lpush" : "rpush";
 
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_LIST,event,c->argv[1],c->db->id);
     }
     server.dirty += pushed;
@@ -247,7 +247,7 @@ void pushxGenericCommand(client *c, int where) {
 
     if (pushed) {
         char *event = (where == LIST_HEAD) ? "lpush" : "rpush";
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_LIST,event,c->argv[1],c->db->id);
     }
     server.dirty += pushed;
@@ -292,7 +292,7 @@ void linsertCommand(client *c) {
     listTypeReleaseIterator(iter);
 
     if (inserted) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_LIST,"linsert",
                             c->argv[1],c->db->id);
         server.dirty++;
@@ -355,7 +355,7 @@ void lsetCommand(client *c) {
             addReply(c,shared.outofrangeerr);
         } else {
             addReply(c,shared.ok);
-            signalModifiedKey(c->db,c->argv[1]);
+            signalModifiedKey(c,c->db,c->argv[1]);
             notifyKeyspaceEvent(NOTIFY_LIST,"lset",c->argv[1],c->db->id);
             server.dirty++;
         }
@@ -382,7 +382,7 @@ void popGenericCommand(client *c, int where) {
                                 c->argv[1],c->db->id);
             dbDelete(c->db,c->argv[1]);
         }
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         server.dirty++;
     }
 }
@@ -482,7 +482,7 @@ void ltrimCommand(client *c) {
         dbDelete(c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id);
     }
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     server.dirty++;
     addReply(c,shared.ok);
 }
@@ -519,7 +519,7 @@ void lremCommand(client *c) {
     listTypeReleaseIterator(li);
 
     if (removed) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_LIST,"lrem",c->argv[1],c->db->id);
     }
 
@@ -555,7 +555,7 @@ void rpoplpushHandlePush(client *c, robj *dstkey, robj *dstobj, robj *value) {
                             server.list_compress_depth);
         dbAdd(c->db,dstkey,dstobj);
     }
-    signalModifiedKey(c->db,dstkey);
+    signalModifiedKey(c,c->db,dstkey);
     listTypePush(dstobj,value,LIST_HEAD);
     notifyKeyspaceEvent(NOTIFY_LIST,"lpush",dstkey,c->db->id);
     /* Always send the pushed value to the client. */
@@ -593,7 +593,7 @@ void rpoplpushCommand(client *c) {
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
                                 touchedkey,c->db->id);
         }
-        signalModifiedKey(c->db,touchedkey);
+        signalModifiedKey(c,c->db,touchedkey);
         decrRefCount(touchedkey);
         server.dirty++;
         if (c->cmd->proc == brpoplpushCommand) {
@@ -708,7 +708,7 @@ void blockingPopGenericCommand(client *c, int where) {
                         notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
                                             c->argv[j],c->db->id);
                     }
-                    signalModifiedKey(c->db,c->argv[j]);
+                    signalModifiedKey(c,c->db,c->argv[j]);
                     server.dirty++;
 
                     /* Replicate it as an [LR]POP instead of B[LR]POP. */
diff --git a/src/t_set.c b/src/t_set.c
index 60cf22d8c..c2e73a6e6 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -280,7 +280,7 @@ void saddCommand(client *c) {
         if (setTypeAdd(set,c->argv[j]->ptr)) added++;
     }
     if (added) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_SET,"sadd",c->argv[1],c->db->id);
     }
     server.dirty += added;
@@ -305,7 +305,7 @@ void sremCommand(client *c) {
         }
     }
     if (deleted) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_SET,"srem",c->argv[1],c->db->id);
         if (keyremoved)
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],
@@ -358,8 +358,8 @@ void smoveCommand(client *c) {
         dbAdd(c->db,c->argv[2],dstset);
     }
 
-    signalModifiedKey(c->db,c->argv[1]);
-    signalModifiedKey(c->db,c->argv[2]);
+    signalModifiedKey(c,c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[2]);
     server.dirty++;
 
     /* An extra key has changed when ele was successfully added to dstset */
@@ -444,7 +444,7 @@ void spopWithCountCommand(client *c) {
 
         /* Propagate this command as an DEL operation */
         rewriteClientCommandVector(c,2,shared.del,c->argv[1]);
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         server.dirty++;
         return;
     }
@@ -546,7 +546,7 @@ void spopWithCountCommand(client *c) {
      * the alsoPropagate() API. */
     decrRefCount(propargv[0]);
     preventCommandPropagation(c);
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     server.dirty++;
 }
 
@@ -599,7 +599,7 @@ void spopCommand(client *c) {
     }
 
     /* Set has been modified */
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     server.dirty++;
 }
 
@@ -808,7 +808,7 @@ void sinterGenericCommand(client *c, robj **setkeys,
             zfree(sets);
             if (dstkey) {
                 if (dbDelete(c->db,dstkey)) {
-                    signalModifiedKey(c->db,dstkey);
+                    signalModifiedKey(c,c->db,dstkey);
                     server.dirty++;
                 }
                 addReply(c,shared.czero);
@@ -908,7 +908,7 @@ void sinterGenericCommand(client *c, robj **setkeys,
                 notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
                     dstkey,c->db->id);
         }
-        signalModifiedKey(c->db,dstkey);
+        signalModifiedKey(c,c->db,dstkey);
         server.dirty++;
     } else {
         setDeferredSetLen(c,replylen,cardinality);
@@ -1083,7 +1083,7 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
                 notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
                     dstkey,c->db->id);
         }
-        signalModifiedKey(c->db,dstkey);
+        signalModifiedKey(c,c->db,dstkey);
         server.dirty++;
     }
     zfree(sets);
diff --git a/src/t_stream.c b/src/t_stream.c
index 155167af9..3efaa4509 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1262,7 +1262,7 @@ void xaddCommand(client *c) {
     }
     addReplyStreamID(c,&id);
 
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_STREAM,"xadd",c->argv[1],c->db->id);
     server.dirty++;
 
@@ -2390,7 +2390,7 @@ void xdelCommand(client *c) {
 
     /* Propagate the write if needed. */
     if (deleted) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_STREAM,"xdel",c->argv[1],c->db->id);
         server.dirty += deleted;
     }
@@ -2467,7 +2467,7 @@ void xtrimCommand(client *c) {
 
     /* Propagate the write if needed. */
     if (deleted) {
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id);
         server.dirty += deleted;
         if (approx_maxlen) streamRewriteApproxMaxlen(c,s,maxlen_arg_idx);
diff --git a/src/t_string.c b/src/t_string.c
index 335bda404..ef382bb0c 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -84,7 +84,7 @@ void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire,
         addReply(c, abort_reply ? abort_reply : shared.null[c->resp]);
         return;
     }
-    genericSetKey(c->db,key,val,flags & OBJ_SET_KEEPTTL,1);
+    genericSetKey(c,c->db,key,val,flags & OBJ_SET_KEEPTTL,1);
     server.dirty++;
     if (expire) setExpire(c,c->db,key,mstime()+milliseconds);
     notifyKeyspaceEvent(NOTIFY_STRING,"set",key,c->db->id);
@@ -183,7 +183,7 @@ void getCommand(client *c) {
 void getsetCommand(client *c) {
     if (getGenericCommand(c) == C_ERR) return;
     c->argv[2] = tryObjectEncoding(c->argv[2]);
-    setKey(c->db,c->argv[1],c->argv[2]);
+    setKey(c,c->db,c->argv[1],c->argv[2]);
     notifyKeyspaceEvent(NOTIFY_STRING,"set",c->argv[1],c->db->id);
     server.dirty++;
 }
@@ -240,7 +240,7 @@ void setrangeCommand(client *c) {
     if (sdslen(value) > 0) {
         o->ptr = sdsgrowzero(o->ptr,offset+sdslen(value));
         memcpy((char*)o->ptr+offset,value,sdslen(value));
-        signalModifiedKey(c->db,c->argv[1]);
+        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_STRING,
             "setrange",c->argv[1],c->db->id);
         server.dirty++;
@@ -328,7 +328,7 @@ void msetGenericCommand(client *c, int nx) {
 
     for (j = 1; j < c->argc; j += 2) {
         c->argv[j+1] = tryObjectEncoding(c->argv[j+1]);
-        setKey(c->db,c->argv[j],c->argv[j+1]);
+        setKey(c,c->db,c->argv[j],c->argv[j+1]);
         notifyKeyspaceEvent(NOTIFY_STRING,"set",c->argv[j],c->db->id);
     }
     server.dirty += (c->argc-1)/2;
@@ -373,7 +373,7 @@ void incrDecrCommand(client *c, long long incr) {
             dbAdd(c->db,c->argv[1],new);
         }
     }
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_STRING,"incrby",c->argv[1],c->db->id);
     server.dirty++;
     addReply(c,shared.colon);
@@ -423,7 +423,7 @@ void incrbyfloatCommand(client *c) {
         dbOverwrite(c->db,c->argv[1],new);
     else
         dbAdd(c->db,c->argv[1],new);
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_STRING,"incrbyfloat",c->argv[1],c->db->id);
     server.dirty++;
     addReplyBulk(c,new);
@@ -467,7 +467,7 @@ void appendCommand(client *c) {
         o->ptr = sdscatlen(o->ptr,append->ptr,sdslen(append->ptr));
         totlen = sdslen(o->ptr);
     }
-    signalModifiedKey(c->db,c->argv[1]);
+    signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_STRING,"append",c->argv[1],c->db->id);
     server.dirty++;
     addReplyLongLong(c,totlen);
diff --git a/src/t_zset.c b/src/t_zset.c
index 5c000e76f..9c409cd96 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -1646,7 +1646,7 @@ reply_to_client:
 cleanup:
     zfree(scores);
     if (added || updated) {
-        signalModifiedKey(c->db,key);
+        signalModifiedKey(c,c->db,key);
         notifyKeyspaceEvent(NOTIFY_ZSET,
             incr ? "zincr" : "zadd", key, c->db->id);
     }
@@ -1681,7 +1681,7 @@ void zremCommand(client *c) {
         notifyKeyspaceEvent(NOTIFY_ZSET,"zrem",key,c->db->id);
         if (keyremoved)
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
-        signalModifiedKey(c->db,key);
+        signalModifiedKey(c,c->db,key);
         server.dirty += deleted;
     }
     addReplyLongLong(c,deleted);
@@ -1779,7 +1779,7 @@ void zremrangeGenericCommand(client *c, int rangetype) {
     /* Step 4: Notifications and reply. */
     if (deleted) {
         char *event[3] = {"zremrangebyrank","zremrangebyscore","zremrangebylex"};
-        signalModifiedKey(c->db,key);
+        signalModifiedKey(c,c->db,key);
         notifyKeyspaceEvent(NOTIFY_ZSET,event[rangetype],key,c->db->id);
         if (keyremoved)
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
@@ -2383,7 +2383,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
         zsetConvertToZiplistIfNeeded(dstobj,maxelelen);
         dbAdd(c->db,dstkey,dstobj);
         addReplyLongLong(c,zsetLength(dstobj));
-        signalModifiedKey(c->db,dstkey);
+        signalModifiedKey(c,c->db,dstkey);
         notifyKeyspaceEvent(NOTIFY_ZSET,
             (op == SET_OP_UNION) ? "zunionstore" : "zinterstore",
             dstkey,c->db->id);
@@ -2392,7 +2392,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
         decrRefCount(dstobj);
         addReply(c,shared.czero);
         if (touched) {
-            signalModifiedKey(c->db,dstkey);
+            signalModifiedKey(c,c->db,dstkey);
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",dstkey,c->db->id);
             server.dirty++;
         }
@@ -3216,7 +3216,7 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey
         if (arraylen == 0) { /* Do this only for the first iteration. */
             char *events[2] = {"zpopmin","zpopmax"};
             notifyKeyspaceEvent(NOTIFY_ZSET,events[where],key,c->db->id);
-            signalModifiedKey(c->db,key);
+            signalModifiedKey(c,c->db,key);
         }
 
         addReplyBulkCBuffer(c,ele,sdslen(ele));
diff --git a/src/tracking.c b/src/tracking.c
index 6f7929430..434e086b5 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -245,7 +245,7 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen, int proto) {
  * matches one or more prefixes in the prefix table. Later when we
  * return to the event loop, we'll send invalidation messages to the
  * clients subscribed to each prefix. */
-void trackingRememberKeyToBroadcast(char *keyname, size_t keylen) {
+void trackingRememberKeyToBroadcast(client *c, char *keyname, size_t keylen) {
     raxIterator ri;
     raxStart(&ri,PrefixTable);
     raxSeek(&ri,"^",NULL,0);
@@ -254,7 +254,11 @@ void trackingRememberKeyToBroadcast(char *keyname, size_t keylen) {
         if (ri.key_len != 0 && memcmp(ri.key,keyname,ri.key_len) != 0)
             continue;
         bcastState *bs = ri.data;
-        raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,NULL,NULL);
+        /* We insert the client pointer as associated value in the radix
+         * tree. This way we know who was the client that did the last
+         * change to the key, and can avoid sending the notification in the
+         * case the client is in NOLOOP mode. */
+        raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,c,NULL);
     }
     raxStop(&ri);
 }
@@ -262,13 +266,17 @@ void trackingRememberKeyToBroadcast(char *keyname, size_t keylen) {
 /* This function is called from signalModifiedKey() or other places in Redis
  * when a key changes value. In the context of keys tracking, our task here is
  * to send a notification to every client that may have keys about such caching
- * slot. */
-void trackingInvalidateKey(robj *keyobj) {
+ * slot.
+ *
+ * Note that 'c' may be NULL in case the operation was performed outside the
+ * context of a client modifying the database (for instance when we delete a
+ * key because of expire). */
+void trackingInvalidateKey(client *c, robj *keyobj) {
     if (TrackingTable == NULL) return;
     sds sdskey = keyobj->ptr;
 
     if (raxSize(PrefixTable) > 0)
-        trackingRememberKeyToBroadcast(sdskey,sdslen(sdskey));
+        trackingRememberKeyToBroadcast(c,sdskey,sdslen(sdskey));
 
     rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey));
     if (ids == raxNotFound) return;
@@ -279,19 +287,28 @@ void trackingInvalidateKey(robj *keyobj) {
     while(raxNext(&ri)) {
         uint64_t id;
         memcpy(&id,ri.key,sizeof(id));
-        client *c = lookupClientByID(id);
+        client *target = lookupClientByID(id);
         /* Note that if the client is in BCAST mode, we don't want to
          * send invalidation messages that were pending in the case
          * previously the client was not in BCAST mode. This can happen if
          * TRACKING is enabled normally, and then the client switches to
          * BCAST mode. */
-        if (c == NULL ||
-            !(c->flags & CLIENT_TRACKING)||
-            c->flags & CLIENT_TRACKING_BCAST)
+        if (target == NULL ||
+            !(target->flags & CLIENT_TRACKING)||
+            target->flags & CLIENT_TRACKING_BCAST)
         {
             continue;
         }
-        sendTrackingMessage(c,sdskey,sdslen(sdskey),0);
+
+        /* If the client enabled the NOLOOP mode, don't send notifications
+         * about keys changed by the client itself. */
+        if (target->flags & CLIENT_TRACKING_NOLOOP &&
+            target == c)
+        {
+            continue;
+        }
+
+        sendTrackingMessage(target,sdskey,sdslen(sdskey),0);
     }
     raxStop(&ri);
 
@@ -383,6 +400,54 @@ void trackingLimitUsedSlots(void) {
     timeout_counter++;
 }
 
+/* Generate Redis protocol for an array containing all the key names
+ * in the 'keys' radix tree. If the client is not NULL, the list will not
+ * include keys that were modified the last time by this client, in order
+ * to implement the NOLOOP option.
+ *
+ * If the resultin array would be empty, NULL is returned instead. */
+sds trackingBuildBroadcastReply(client *c, rax *keys) {
+    raxIterator ri;
+    uint64_t count;
+
+    if (c == NULL) {
+        count = raxSize(keys);
+    } else {
+        count = 0;
+        raxStart(&ri,keys);
+        raxSeek(&ri,"^",NULL,0);
+        while(raxNext(&ri)) {
+            if (ri.data != c) count++;
+        }
+        raxStop(&ri);
+
+        if (count == 0) return NULL;
+    }
+
+    /* Create the array reply with the list of keys once, then send
+    * it to all the clients subscribed to this prefix. */
+    char buf[32];
+    size_t len = ll2string(buf,sizeof(buf),count);
+    sds proto = sdsempty();
+    proto = sdsMakeRoomFor(proto,count*15);
+    proto = sdscatlen(proto,"*",1);
+    proto = sdscatlen(proto,buf,len);
+    proto = sdscatlen(proto,"\r\n",2);
+    raxStart(&ri,keys);
+    raxSeek(&ri,"^",NULL,0);
+    while(raxNext(&ri)) {
+        if (c && ri.data == c) continue;
+        len = ll2string(buf,sizeof(buf),ri.key_len);
+        proto = sdscatlen(proto,"$",1);
+        proto = sdscatlen(proto,buf,len);
+        proto = sdscatlen(proto,"\r\n",2);
+        proto = sdscatlen(proto,ri.key,ri.key_len);
+        proto = sdscatlen(proto,"\r\n",2);
+    }
+    raxStop(&ri);
+    return proto;
+}
+
 /* This function will run the prefixes of clients in BCAST mode and
  * keys that were modified about each prefix, and will send the
  * notifications to each client in each prefix. */
@@ -397,26 +462,10 @@ void trackingBroadcastInvalidationMessages(void) {
     while(raxNext(&ri)) {
         bcastState *bs = ri.data;
         if (raxSize(bs->keys)) {
-            /* Create the array reply with the list of keys once, then send
-            * it to all the clients subscribed to this prefix. */
-            char buf[32];
-            size_t len = ll2string(buf,sizeof(buf),raxSize(bs->keys));
-            sds proto = sdsempty();
-            proto = sdsMakeRoomFor(proto,raxSize(bs->keys)*15);
-            proto = sdscatlen(proto,"*",1);
-            proto = sdscatlen(proto,buf,len);
-            proto = sdscatlen(proto,"\r\n",2);
-            raxStart(&ri2,bs->keys);
-            raxSeek(&ri2,"^",NULL,0);
-            while(raxNext(&ri2)) {
-                len = ll2string(buf,sizeof(buf),ri2.key_len);
-                proto = sdscatlen(proto,"$",1);
-                proto = sdscatlen(proto,buf,len);
-                proto = sdscatlen(proto,"\r\n",2);
-                proto = sdscatlen(proto,ri2.key,ri2.key_len);
-                proto = sdscatlen(proto,"\r\n",2);
-            }
-            raxStop(&ri2);
+
+            /* Generate the common protocol for all the clients that are
+             * not using the NOLOOP option. */
+            sds proto = trackingBuildBroadcastReply(NULL,bs->keys);
 
             /* Send this array of keys to every client in the list. */
             raxStart(&ri2,bs->clients);
@@ -424,7 +473,14 @@ void trackingBroadcastInvalidationMessages(void) {
             while(raxNext(&ri2)) {
                 client *c;
                 memcpy(&c,ri2.key,sizeof(c));
-                sendTrackingMessage(c,proto,sdslen(proto),1);
+                if (c->flags & CLIENT_TRACKING_NOLOOP) {
+                    /* This client may have certain keys excluded. */
+                    sds adhoc = trackingBuildBroadcastReply(c,bs->keys);
+                    sendTrackingMessage(c,adhoc,sdslen(adhoc),1);
+                    sdsfree(adhoc);
+                } else {
+                    sendTrackingMessage(c,proto,sdslen(proto),1);
+                }
             }
             raxStop(&ri2);
 

From 92974f9a98d22c96ec3bbd68002dacfaeb3c7960 Mon Sep 17 00:00:00 2001
From: yanhui13 <yanhui13@meituan.com>
Date: Tue, 21 Apr 2020 16:55:05 +0800
Subject: [PATCH 0292/1098] optimize the output of cluster slots

---
 src/cluster.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 2377b386b..4edc068ba 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4191,11 +4191,17 @@ void clusterReplyMultiBulkSlots(client *c) {
     while((de = dictNext(di)) != NULL) {
         clusterNode *node = dictGetVal(de);
         int j = 0, start = -1;
+        int i, nested_elements = 0;
 
         /* Skip slaves (that are iterated when producing the output of their
          * master) and  masters not serving any slot. */
         if (!nodeIsMaster(node) || node->numslots == 0) continue;
 
+        for(i = 0; i < node->numslaves; i++) {
+            if (nodeFailed(node->slaves[i])) continue;
+            nested_elements++;
+        }
+
         for (j = 0; j < CLUSTER_SLOTS; j++) {
             int bit, i;
 
@@ -4203,8 +4209,7 @@ void clusterReplyMultiBulkSlots(client *c) {
                 if (start == -1) start = j;
             }
             if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) {
-                int nested_elements = 3; /* slots (2) + master addr (1). */
-                void *nested_replylen = addReplyDeferredLen(c);
+                addReplyArrayLen(c, nested_elements + 3); /* slots (2) + master addr (1). */
 
                 if (bit && j == CLUSTER_SLOTS-1) j++;
 
@@ -4234,9 +4239,7 @@ void clusterReplyMultiBulkSlots(client *c) {
                     addReplyBulkCString(c, node->slaves[i]->ip);
                     addReplyLongLong(c, node->slaves[i]->port);
                     addReplyBulkCBuffer(c, node->slaves[i]->name, CLUSTER_NAMELEN);
-                    nested_elements++;
                 }
-                setDeferredArrayLen(c, nested_replylen, nested_elements);
                 num_masters++;
             }
         }

From 6b547c395692a2f124e573e3e79b0866c024a28d Mon Sep 17 00:00:00 2001
From: yanhui13 <yanhui13@meituan.com>
Date: Tue, 21 Apr 2020 16:56:10 +0800
Subject: [PATCH 0293/1098] add tcl test for cluster slots

---
 tests/cluster/tests/15-cluster-slots.tcl | 44 ++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 tests/cluster/tests/15-cluster-slots.tcl

diff --git a/tests/cluster/tests/15-cluster-slots.tcl b/tests/cluster/tests/15-cluster-slots.tcl
new file mode 100644
index 000000000..dc9938ef6
--- /dev/null
+++ b/tests/cluster/tests/15-cluster-slots.tcl
@@ -0,0 +1,44 @@
+source "../tests/includes/init-tests.tcl"
+
+proc cluster_allocate_mixedSlots {n} {
+    set slot 16383
+    while {$slot >= 0} {
+        set node [expr {$slot % $n}]
+        lappend slots_$node $slot
+        incr slot -1
+    }
+    for {set j 0} {$j < $n} {incr j} {
+        R $j cluster addslots {*}[set slots_${j}]
+    }
+}
+
+proc create_cluster_with_mixedSlot {masters slaves} {
+    cluster_allocate_mixedSlots $masters
+    if {$slaves} {
+        cluster_allocate_slaves $masters $slaves
+    }
+    assert_cluster_state ok
+}
+
+test "Create a 5 nodes cluster" {
+    create_cluster_with_mixedSlot 5 15
+}
+
+test "Cluster is up" {
+    assert_cluster_state ok
+}
+
+test "Cluster is writable" {
+    cluster_write_test 0
+}
+
+test "Instance #5 is a slave" {
+    assert {[RI 5 role] eq {slave}}
+}
+
+test "client do not break when cluster slot" {
+    R 0 config set client-output-buffer-limit "normal 33554432 16777216 60"
+    if { [catch {R 0 cluster slots}] } {
+        fail "output overflow when cluster slots"
+    }
+}
\ No newline at end of file

From c6ccc2f6901890ef749ec0f2738404bd8a507861 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 21 Apr 2020 17:29:18 +0200
Subject: [PATCH 0294/1098] Tracking: NOLOOP further implementation and fixes.

---
 src/networking.c |  2 ++
 src/tracking.c   | 25 +++++++++++++++++++------
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 1f5d0bd5d..744979d16 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2289,6 +2289,8 @@ NULL
                 options |= CLIENT_TRACKING_OPTIN;
             } else if (!strcasecmp(c->argv[j]->ptr,"optout")) {
                 options |= CLIENT_TRACKING_OPTOUT;
+            } else if (!strcasecmp(c->argv[j]->ptr,"noloop")) {
+                options |= CLIENT_TRACKING_NOLOOP;
             } else if (!strcasecmp(c->argv[j]->ptr,"prefix") && moreargs) {
                 j++;
                 prefix = zrealloc(prefix,sizeof(robj*)*(numprefix+1));
diff --git a/src/tracking.c b/src/tracking.c
index 434e086b5..48d231627 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -94,7 +94,8 @@ void disableTracking(client *c) {
         server.tracking_clients--;
         c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR|
                       CLIENT_TRACKING_BCAST|CLIENT_TRACKING_OPTIN|
-                      CLIENT_TRACKING_OPTOUT|CLIENT_TRACKING_CACHING);
+                      CLIENT_TRACKING_OPTOUT|CLIENT_TRACKING_CACHING|
+                      CLIENT_TRACKING_NOLOOP);
     }
 }
 
@@ -129,14 +130,19 @@ void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **pr
     if (!(c->flags & CLIENT_TRACKING)) server.tracking_clients++;
     c->flags |= CLIENT_TRACKING;
     c->flags &= ~(CLIENT_TRACKING_BROKEN_REDIR|CLIENT_TRACKING_BCAST|
-                  CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT);
+                  CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT|
+                  CLIENT_TRACKING_NOLOOP);
     c->client_tracking_redirection = redirect_to;
+
+    /* This may be the first client we ever enable. Crete the tracking
+     * table if it does not exist. */
     if (TrackingTable == NULL) {
         TrackingTable = raxNew();
         PrefixTable = raxNew();
         TrackingChannelName = createStringObject("__redis__:invalidate",20);
     }
 
+    /* For broadcasting, set the list of prefixes in the client. */
     if (options & CLIENT_TRACKING_BCAST) {
         c->flags |= CLIENT_TRACKING_BCAST;
         if (numprefix == 0) enableBcastTrackingForPrefix(c,"",0);
@@ -145,7 +151,10 @@ void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **pr
             enableBcastTrackingForPrefix(c,sdsprefix,sdslen(sdsprefix));
         }
     }
-    c->flags |= options & (CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT);
+
+    /* Set the remaining flags that don't need any special handling. */
+    c->flags |= options & (CLIENT_TRACKING_OPTIN|CLIENT_TRACKING_OPTOUT|
+                           CLIENT_TRACKING_NOLOOP);
 }
 
 /* This function is called after the execution of a readonly command in the
@@ -459,10 +468,12 @@ void trackingBroadcastInvalidationMessages(void) {
 
     raxStart(&ri,PrefixTable);
     raxSeek(&ri,"^",NULL,0);
+
+    /* For each prefix... */
     while(raxNext(&ri)) {
         bcastState *bs = ri.data;
-        if (raxSize(bs->keys)) {
 
+        if (raxSize(bs->keys)) {
             /* Generate the common protocol for all the clients that are
              * not using the NOLOOP option. */
             sds proto = trackingBuildBroadcastReply(NULL,bs->keys);
@@ -476,8 +487,10 @@ void trackingBroadcastInvalidationMessages(void) {
                 if (c->flags & CLIENT_TRACKING_NOLOOP) {
                     /* This client may have certain keys excluded. */
                     sds adhoc = trackingBuildBroadcastReply(c,bs->keys);
-                    sendTrackingMessage(c,adhoc,sdslen(adhoc),1);
-                    sdsfree(adhoc);
+                    if (adhoc) {
+                        sendTrackingMessage(c,adhoc,sdslen(adhoc),1);
+                        sdsfree(adhoc);
+                    }
                 } else {
                     sendTrackingMessage(c,proto,sdslen(proto),1);
                 }

From 04f1a280e09734150ce78d40b8632035c38f17bc Mon Sep 17 00:00:00 2001
From: Theo Buehler <theo.buehler@cysec.systems>
Date: Wed, 22 Apr 2020 09:43:01 +0200
Subject: [PATCH 0295/1098] TLS: Fix build with SSL_OP_NO_CLIENT_RENEGOTIATION

There is no ssl in this scope, so the build breaks.
All the other options are set directly on the ctx.
---
 src/tls.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tls.c b/src/tls.c
index ea0c34469..c18aafebe 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -160,7 +160,7 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
 #endif
 
 #ifdef SSL_OP_NO_CLIENT_RENEGOTIATION
-    SSL_CTX_set_options(ssl->ctx, SSL_OP_NO_CLIENT_RENEGOTIATION);
+    SSL_CTX_set_options(ctx, SSL_OP_NO_CLIENT_RENEGOTIATION);
 #endif
 
     if (ctx_config->prefer_server_ciphers)

From 2d1968f841277e363c14d1a2f56dd295dc49906c Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 22 Apr 2020 10:49:17 +0200
Subject: [PATCH 0296/1098] Tracking: signal key as modified when evicting.

---
 src/evict.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/evict.c b/src/evict.c
index 71260c040..a829f1f99 100644
--- a/src/evict.c
+++ b/src/evict.c
@@ -569,6 +569,7 @@ int freeMemoryIfNeeded(void) {
                 dbAsyncDelete(db,keyobj);
             else
                 dbSyncDelete(db,keyobj);
+            signalModifiedKey(NULL,db,keyobj);
             latencyEndMonitor(eviction_latency);
             latencyAddSampleIfNeeded("eviction-del",eviction_latency);
             latencyRemoveNestedEvent(latency,eviction_latency);

From 58d61dd639382fc05a2516a45656ff83ebcb18ee Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 22 Apr 2020 11:24:19 +0200
Subject: [PATCH 0297/1098] Tracking: NOLOOP tests.

---
 tests/unit/tracking.tcl | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index 2058319f7..dc6cd671a 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -7,6 +7,9 @@ start_server {tags {"tracking"}} {
     $rd1 subscribe __redis__:invalidate
     $rd1 read ; # Consume the SUBSCRIBE reply.
 
+    # Create another client as well in order to test NOLOOP
+    set rd2 [redis_deferring_client]
+
     test {Clients are able to enable tracking and redirect it} {
         r CLIENT TRACKING on REDIRECT $redir
     } {*OK}
@@ -62,5 +65,34 @@ start_server {tags {"tracking"}} {
         assert {$keys eq {c:1234}}
     }
 
+    test {Tracking NOLOOP mode in standard mode works} {
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on REDIRECT $redir NOLOOP
+        r MGET otherkey1 loopkey otherkey2
+        $rd2 SET otherkey1 1; # We should get this
+        r SET loopkey 1 ; # We should not get this
+        $rd2 SET otherkey2 1; # We should get this
+        # Because of the internals, we know we are going to receive
+        # two separated notifications for the two different prefixes.
+        set keys1 [lsort [lindex [$rd1 read] 2]]
+        set keys2 [lsort [lindex [$rd1 read] 2]]
+        set keys [lsort [list {*}$keys1 {*}$keys2]]
+        assert {$keys eq {otherkey1 otherkey2}}
+    }
+
+    test {Tracking NOLOOP mode in BCAST mode works} {
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on BCAST REDIRECT $redir NOLOOP
+        $rd2 SET otherkey1 1; # We should get this
+        r SET loopkey 1 ; # We should not get this
+        $rd2 SET otherkey2 1; # We should get this
+        # Because of the internals, we know we are going to receive
+        # two separated notifications for the two different prefixes.
+        set keys1 [lsort [lindex [$rd1 read] 2]]
+        set keys2 [lsort [lindex [$rd1 read] 2]]
+        set keys [lsort [list {*}$keys1 {*}$keys2]]
+        assert {$keys eq {otherkey1 otherkey2}}
+    }
+
     $rd1 close
 }

From 8d67211450ef71c1eac505a44958998d852cbfda Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 22 Apr 2020 11:45:34 +0200
Subject: [PATCH 0298/1098] Tracking: test expired keys notifications.

---
 tests/unit/tracking.tcl | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index dc6cd671a..43bb5f864 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -94,5 +94,18 @@ start_server {tags {"tracking"}} {
         assert {$keys eq {otherkey1 otherkey2}}
     }
 
+    test {Tracking gets notification of expired keys} {
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on BCAST REDIRECT $redir NOLOOP
+        r SET mykey myval px 1
+        r SET mykeyotherkey myval ; # We should not get it
+        after 1000
+        # Because of the internals, we know we are going to receive
+        # two separated notifications for the two different prefixes.
+        set keys1 [lsort [lindex [$rd1 read] 2]]
+        set keys [lsort [list {*}$keys1]]
+        assert {$keys eq {mykey}}
+    }
+
     $rd1 close
 }

From 6fd2d7cfee05c7493a0c50e9ef462b8183035ca1 Mon Sep 17 00:00:00 2001
From: Valentino Geron <valentino@redislabs.com>
Date: Tue, 21 Apr 2020 20:55:43 +0300
Subject: [PATCH 0299/1098] XREADGROUP with NOACK should propagate only one
 XGROUP SETID command

---
 src/t_stream.c | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 155167af9..2dffdbcdf 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -935,6 +935,8 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
     streamIterator si;
     int64_t numfields;
     streamID id;
+    int propagate_last_id = 0;
+    int noack = flags & STREAM_RWR_NOACK;
 
     /* If the client is asking for some history, we serve it using a
      * different function, so that we return entries *solely* from its
@@ -950,12 +952,14 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
         arraylen_ptr = addReplyDeferredLen(c);
     streamIteratorStart(&si,s,start,end,rev);
     while(streamIteratorGetID(&si,&id,&numfields)) {
-        int propagate_last_id = 0;
-
         /* Update the group last_id if needed. */
         if (group && streamCompareID(&id,&group->last_id) > 0) {
             group->last_id = id;
-            propagate_last_id = 1;
+            /* Group last id should be propagated only if NOACK was
+             * specified, otherwise the last id would be included
+             * in XCLAIM. */
+            if (noack)
+                propagate_last_id = 1;
         }
 
         /* Emit a two elements array for each item. The first is
@@ -983,7 +987,7 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
          * XGROUP SETID command. So if we find that there is already
          * a NACK for the entry, we need to associate it to the new
          * consumer. */
-        if (group && !(flags & STREAM_RWR_NOACK)) {
+        if (group && !noack) {
             unsigned char buf[sizeof(streamID)];
             streamEncodeID(buf,&id);
 
@@ -1020,14 +1024,16 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
                 streamPropagateXCLAIM(c,spi->keyname,group,spi->groupname,idarg,nack);
                 decrRefCount(idarg);
             }
-        } else {
-            if (propagate_last_id)
-                streamPropagateGroupID(c,spi->keyname,group,spi->groupname);
         }
 
         arraylen++;
         if (count && count == arraylen) break;
     }
+
+    if (spi && propagate_last_id) {
+        streamPropagateGroupID(c,spi->keyname,group,spi->groupname);
+    }
+
     streamIteratorStop(&si);
     if (arraylen_ptr) setDeferredArrayLen(c,arraylen_ptr,arraylen);
     return arraylen;

From 889eaba2b70d7652accd6d34c305023eea490fbc Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 22 Apr 2020 17:14:15 +0200
Subject: [PATCH 0300/1098] ACL: deny commands execution of disabled users.

---
 src/acl.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/acl.c b/src/acl.c
index 75b954c5e..9e2ed6af7 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1055,6 +1055,10 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) {
     /* If there is no associated user, the connection can run anything. */
     if (u == NULL) return ACL_OK;
 
+    /* If the user is disabled we don't allow the execution of any
+     * command. */
+    if (!(u->flags & USER_FLAG_ENABLED)) return ACL_DENIED_CMD;
+
     /* Check if the user can execute this command. */
     if (!(u->flags & USER_FLAG_ALLCOMMANDS) &&
         c->cmd->proc != authCommand)

From dc1bc0f4233f471c3bef35552260de4ae600a9fb Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 23 Apr 2020 10:39:53 +0200
Subject: [PATCH 0301/1098] ACL GENPASS: emit 256 bits instead of 128.

---
 src/acl.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/acl.c b/src/acl.c
index 9e2ed6af7..228811cba 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1819,7 +1819,7 @@ void aclCommand(client *c) {
         dictReleaseIterator(di);
         setDeferredArrayLen(c,dl,arraylen);
     } else if (!strcasecmp(sub,"genpass") && c->argc == 2) {
-        char pass[32]; /* 128 bits of actual pseudo random data. */
+        char pass[64]; /* 256 bits of actual pseudo random data. */
         getRandomHexChars(pass,sizeof(pass));
         addReplyBulkCBuffer(c,pass,sizeof(pass));
     } else if (!strcasecmp(sub,"log") && (c->argc == 2 || c->argc ==3)) {

From a95a086be7b6c8eda5b0d181b6874ac88502aae5 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 23 Apr 2020 10:53:21 +0200
Subject: [PATCH 0302/1098] ACL GENPASS: take number of bits as argument.

---
 src/acl.c | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 228811cba..bf5dd18f1 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1626,7 +1626,7 @@ void addACLLogEntry(client *c, int reason, int keypos, sds username) {
  * ACL SETUSER <username> ... acl rules ...
  * ACL DELUSER <username> [...]
  * ACL GETUSER <username>
- * ACL GENPASS
+ * ACL GENPASS [<bits>]
  * ACL WHOAMI
  * ACL LOG [<count> | RESET]
  */
@@ -1818,10 +1818,25 @@ void aclCommand(client *c) {
         }
         dictReleaseIterator(di);
         setDeferredArrayLen(c,dl,arraylen);
-    } else if (!strcasecmp(sub,"genpass") && c->argc == 2) {
-        char pass[64]; /* 256 bits of actual pseudo random data. */
-        getRandomHexChars(pass,sizeof(pass));
-        addReplyBulkCBuffer(c,pass,sizeof(pass));
+    } else if (!strcasecmp(sub,"genpass") && (c->argc == 2 || c->argc == 3)) {
+        #define GENPASS_MAX_BITS 4096
+        char pass[GENPASS_MAX_BITS/8*2]; /* Hex representation. */
+        long bits = 256; /* By default generate 256 bits passwords. */
+
+        if (c->argc == 3 && getLongFromObjectOrReply(c,c->argv[2],&bits,NULL)
+            != C_OK) return;
+
+        if (bits <= 0 || bits > GENPASS_MAX_BITS) {
+            addReplyErrorFormat(c,
+                "ACL GENPASS argument must be the number of "
+                "bits for the output password, a positive number "
+                "up to %d",GENPASS_MAX_BITS);
+            return;
+        }
+
+        long chars = (bits+3)/4; /* Round to number of characters to emit. */
+        getRandomHexChars(pass,chars);
+        addReplyBulkCBuffer(c,pass,chars);
     } else if (!strcasecmp(sub,"log") && (c->argc == 2 || c->argc ==3)) {
         long count = 10; /* Number of entries to emit by default. */
 
@@ -1899,7 +1914,7 @@ void aclCommand(client *c) {
 "DELUSER <username> [...]         -- Delete a list of users.",
 "CAT                              -- List available categories.",
 "CAT <category>                   -- List commands inside category.",
-"GENPASS                          -- Generate a secure user password.",
+"GENPASS [<bits>]                 -- Generate a secure user password.",
 "WHOAMI                           -- Return the current connection username.",
 "LOG [<count> | RESET]            -- Show the ACL log entries.",
 NULL

From 9ae8254e20c151bb153519a868933cbd13d4c164 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 23 Apr 2020 11:17:42 +0200
Subject: [PATCH 0303/1098] getRandomBytes(): use HMAC-SHA256.

Now that we have an interface to use this API directly, via ACL GENPASS,
we are no longer sure what people could do with it. So why don't make it
a strong primitive exported by Redis in order to create unique IDs and
so forth?

The implementation was tested against the test vectors that can
be found in RFC4231.
---
 src/util.c | 38 +++++++++++++++++++++++++++++---------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/src/util.c b/src/util.c
index bd8f0fb98..d173f776f 100644
--- a/src/util.c
+++ b/src/util.c
@@ -42,7 +42,7 @@
 #include <time.h>
 
 #include "util.h"
-#include "sha1.h"
+#include "sha256.h"
 
 /* Glob-style pattern matching. */
 int stringmatchlen(const char *pattern, int patternLen,
@@ -622,7 +622,7 @@ int ld2string(char *buf, size_t len, long double value, ld2string_mode mode) {
 void getRandomBytes(unsigned char *p, size_t len) {
     /* Global state. */
     static int seed_initialized = 0;
-    static unsigned char seed[20]; /* The SHA1 seed, from /dev/urandom. */
+    static unsigned char seed[64]; /* 512 bit internal block size. */
     static uint64_t counter = 0; /* The counter we hash with the seed. */
 
     if (!seed_initialized) {
@@ -647,14 +647,34 @@ void getRandomBytes(unsigned char *p, size_t len) {
     }
 
     while(len) {
-        unsigned char digest[20];
-        SHA1_CTX ctx;
-        unsigned int copylen = len > 20 ? 20 : len;
+        /* This implements SHA256-HMAC. */
+        unsigned char digest[SHA256_BLOCK_SIZE];
+        unsigned char kxor[64];
+        unsigned int copylen =
+            len > SHA256_BLOCK_SIZE ? SHA256_BLOCK_SIZE : len;
 
-        SHA1Init(&ctx);
-        SHA1Update(&ctx, seed, sizeof(seed));
-        SHA1Update(&ctx, (unsigned char*)&counter,sizeof(counter));
-        SHA1Final(digest, &ctx);
+        /* IKEY: key xored with 0x36. */
+        memcpy(kxor,seed,sizeof(kxor));
+        for (unsigned int i = 0; i < sizeof(kxor); i++) kxor[i] ^= 0x36;
+
+        /* Obtain HASH(IKEY||MESSAGE). */
+        SHA256_CTX ctx;
+        sha256_init(&ctx);
+        sha256_update(&ctx,kxor,sizeof(kxor));
+        sha256_update(&ctx,(unsigned char*)&counter,sizeof(counter));
+        sha256_final(&ctx,digest);
+
+        /* OKEY: key xored with 0x5c. */
+        memcpy(kxor,seed,sizeof(kxor));
+        for (unsigned int i = 0; i < sizeof(kxor); i++) kxor[i] ^= 0x5C;
+
+        /* Obtain HASH(OKEY || HASH(IKEY||MESSAGE)). */
+        sha256_init(&ctx);
+        sha256_update(&ctx,kxor,sizeof(kxor));
+        sha256_update(&ctx,digest,SHA256_BLOCK_SIZE);
+        sha256_final(&ctx,digest);
+
+        /* Increment the counter for the next iteration. */
         counter++;
 
         memcpy(p,digest,copylen);

From 96ebfc3b361b7c45f87e86cf1d4025e62da062e6 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 23 Apr 2020 11:56:36 +0200
Subject: [PATCH 0304/1098] ACL: re-enable command execution of disabled users.

After all I changed idea again: enabled/disabled should have a more
clear meaning, and it only means: you can't authenticate with such user
with new connections, however old connections continue to work as
expected.
---
 src/acl.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index bf5dd18f1..3194feb5b 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1055,10 +1055,6 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) {
     /* If there is no associated user, the connection can run anything. */
     if (u == NULL) return ACL_OK;
 
-    /* If the user is disabled we don't allow the execution of any
-     * command. */
-    if (!(u->flags & USER_FLAG_ENABLED)) return ACL_DENIED_CMD;
-
     /* Check if the user can execute this command. */
     if (!(u->flags & USER_FLAG_ALLCOMMANDS) &&
         c->cmd->proc != authCommand)

From 12cd67b7cac9cfbcef4e339fd51027b1afbff86b Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 23 Apr 2020 16:13:45 +0200
Subject: [PATCH 0305/1098] Minor aesthetic changes to #7135.

---
 src/t_stream.c | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 8abbee02f..596d8ad0d 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -955,11 +955,10 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
         /* Update the group last_id if needed. */
         if (group && streamCompareID(&id,&group->last_id) > 0) {
             group->last_id = id;
-            /* Group last id should be propagated only if NOACK was
-             * specified, otherwise the last id would be included
-             * in XCLAIM. */
-            if (noack)
-                propagate_last_id = 1;
+            /* Group last ID should be propagated only if NOACK was
+             * specified, otherwise the last id will be included
+             * in the propagation of XCLAIM itself. */
+            if (noack) propagate_last_id = 1;
         }
 
         /* Emit a two elements array for each item. The first is
@@ -1030,9 +1029,8 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
         if (count && count == arraylen) break;
     }
 
-    if (spi && propagate_last_id) {
+    if (spi && propagate_last_id)
         streamPropagateGroupID(c,spi->keyname,group,spi->groupname);
-    }
 
     streamIteratorStop(&si);
     if (arraylen_ptr) setDeferredArrayLen(c,arraylen_ptr,arraylen);

From 57a0c9c98d3bb966d6191fd1657df010cfe060fb Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 24 Apr 2020 10:13:20 +0200
Subject: [PATCH 0306/1098] Also use propagate() in streamPropagateGroupID().

---
 src/t_stream.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 596d8ad0d..758db637e 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -848,6 +848,11 @@ void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupnam
     argv[11] = createStringObject("JUSTID",6);
     argv[12] = createStringObject("LASTID",6);
     argv[13] = createObjectFromStreamID(&group->last_id);
+
+    /* We use progagate() because this code path is not always called from
+     * the command execution context. Moreover this will just alter the
+     * consumer group state, and we don't need MULTI/EXEC wrapping because
+     * there is no message state cross-message atomicity required. */
     propagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL);
     decrRefCount(argv[0]);
     decrRefCount(argv[3]);
@@ -875,7 +880,12 @@ void streamPropagateGroupID(client *c, robj *key, streamCG *group, robj *groupna
     argv[2] = key;
     argv[3] = groupname;
     argv[4] = createObjectFromStreamID(&group->last_id);
-    alsoPropagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL);
+
+    /* We use progagate() because this code path is not always called from
+     * the command execution context. Moreover this will just alter the
+     * consumer group state, and we don't need MULTI/EXEC wrapping because
+     * there is no message state cross-message atomicity required. */
+    propagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL);
     decrRefCount(argv[0]);
     decrRefCount(argv[1]);
     decrRefCount(argv[4]);

From fb732f7a944a4d4c90bb7375cb6030e88211f5aa Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 24 Apr 2020 17:20:28 +0300
Subject: [PATCH 0307/1098] optimize memory usage of deferred replies

When deffered reply is added the previous reply node cannot be used so
all the extra space we allocated in it is wasted. in case someone uses
deffered replies in a loop, each time adding a small reply, each of
these reply nodes (the small string reply) would have consumed a 16k
block.
now when we add anther diferred reply node, we trim the unused portion
of the previous reply block.

see #7123
---
 src/networking.c | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index 744979d16..a4247a8a6 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -436,6 +436,36 @@ void addReplyStatusFormat(client *c, const char *fmt, ...) {
     sdsfree(s);
 }
 
+/* Sometimes we are forced to create a new reply node, and we can't append to
+ * the previous one, when that happens, we wanna try to trim the unused space
+ * at the end of the last reply node which we won't use anymore. */
+void trimReplyUnusedTailSpace(client *c) {
+    listNode *ln = listLast(c->reply);
+    clientReplyBlock *tail = ln? listNodeValue(ln): NULL;
+
+    /* Note that 'tail' may be NULL even if we have a tail node, becuase when
+     * addDeferredMultiBulkLength() is used */
+    if (!tail) return;
+
+    /* We only try to trim the space is relatively high (more than a 1/4 of the
+     * allocation), otherwise there's a high chance realloc will NOP.
+     * Also, to avoid large memmove which happens as part of realloc, we only do
+     * that if the used part is small.  */
+    if (tail->size - tail->used > tail->size / 4 &&
+        tail->used < PROTO_REPLY_CHUNK_BYTES)
+    {
+        size_t old_size = tail->size;
+        tail = zrealloc(tail, tail->used + sizeof(clientReplyBlock));
+        /* If realloc was a NOP, we got the same value which has internal frag */
+        if (tail == listNodeValue(ln)) return;
+        /* take over the allocation's internal fragmentation (at least for
+         * memory usage tracking) */
+        tail->size = zmalloc_usable(tail) - sizeof(clientReplyBlock);
+        c->reply_bytes += tail->size - old_size;
+        listNodeValue(ln) = tail;
+    }
+}
+
 /* Adds an empty object to the reply list that will contain the multi bulk
  * length, which is not known when this function is called. */
 void *addReplyDeferredLen(client *c) {
@@ -443,6 +473,7 @@ void *addReplyDeferredLen(client *c) {
      * ready to be sent, since we are sure that before returning to the
      * event loop setDeferredAggregateLen() will be called. */
     if (prepareClientToWrite(c) != C_OK) return NULL;
+    trimReplyUnusedTailSpace(c);
     listAddNodeTail(c->reply,NULL); /* NULL is our placeholder. */
     return listLast(c->reply);
 }

From 8a7f255cd005a13aaebb77468ab77dc2c6501d23 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 24 Apr 2020 16:49:27 +0200
Subject: [PATCH 0308/1098] LCS -> STRALGO LCS.

STRALGO should be a container for mostly read-only string
algorithms in Redis. The algorithms should have two main
characteristics:

1. They should be non trivial to compute, and often not part of
programming language standard libraries.
2. They should be fast enough that it is a good idea to have optimized C
implementations.

Next thing I would love to see? A small strings compression algorithm.
---
 src/server.c               |  2 +-
 src/server.h               |  2 +-
 src/t_string.c             | 23 ++++++++++++++++++-----
 tests/unit/type/string.tcl | 16 ++++++++--------
 4 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/src/server.c b/src/server.c
index fc9b87aae..f7af79c3f 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1006,7 +1006,7 @@ struct redisCommand redisCommandTable[] = {
      "admin no-script no-slowlog ok-loading ok-stale",
      0,NULL,0,0,0,0,0,0},
 
-    {"lcs",lcsCommand,-4,
+    {"stralgo",stralgoCommand,-2,
      "write use-memory @string",
      0,lcsGetKeys,0,0,0,0,0,0}
 };
diff --git a/src/server.h b/src/server.h
index d39359dce..9e1e506af 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2389,7 +2389,7 @@ void xdelCommand(client *c);
 void xtrimCommand(client *c);
 void lolwutCommand(client *c);
 void aclCommand(client *c);
-void lcsCommand(client *c);
+void stralgoCommand(client *c);
 
 #if defined(__GNUC__)
 void *calloc(size_t count, size_t size) __attribute__ ((deprecated));
diff --git a/src/t_string.c b/src/t_string.c
index ef382bb0c..d4eb04769 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -480,18 +480,31 @@ void strlenCommand(client *c) {
     addReplyLongLong(c,stringObjectLen(o));
 }
 
-/* LCS -- Longest common subsequence.
+
+/* STRALGO -- Implement complex algorithms on strings.
  *
- * LCS [IDX] [MINMATCHLEN <len>]
- *     STRINGS <string> <string> | KEYS <keya> <keyb> */
-void lcsCommand(client *c) {
+ * STRALGO <algorithm> ... arguments ... */
+void stralgoLCS(client *c);     /* This implements the LCS algorithm. */
+void stralgoCommand(client *c) {
+    /* Select the algorithm. */
+    if (!strcasecmp(c->argv[1]->ptr,"lcs")) {
+        stralgoLCS(c);
+    } else {
+        addReply(c,shared.syntaxerr);
+    }
+}
+
+/* STRALGO <algo> [IDX] [MINMATCHLEN <len>] [WITHMATCHLEN]
+ *     STRINGS <string> <string> | KEYS <keya> <keyb>
+ */
+void stralgoLCS(client *c) {
     uint32_t i, j;
     long long minmatchlen = 0;
     sds a = NULL, b = NULL;
     int getlen = 0, getidx = 0, withmatchlen = 0;
     robj *obja = NULL, *objb = NULL;
 
-    for (j = 1; j < (uint32_t)c->argc; j++) {
+    for (j = 2; j < (uint32_t)c->argc; j++) {
         char *opt = c->argv[j]->ptr;
         int moreargs = (c->argc-1) - j;
 
diff --git a/tests/unit/type/string.tcl b/tests/unit/type/string.tcl
index b9ef9de7a..8126cdee8 100644
--- a/tests/unit/type/string.tcl
+++ b/tests/unit/type/string.tcl
@@ -424,29 +424,29 @@ start_server {tags {"string"}} {
     set rna2 {ATTAAAGGTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGTGTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTT}
     set rnalcs {ACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGTGTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTT}
 
-    test {LCS string output with STRINGS option} {
-        r LCS STRINGS $rna1 $rna2
+    test {STRALGO LCS string output with STRINGS option} {
+        r STRALGO LCS STRINGS $rna1 $rna2
     } $rnalcs
 
-    test {LCS len} {
-        r LCS LEN STRINGS $rna1 $rna2
+    test {STRALGO LCS len} {
+        r STRALGO LCS LEN STRINGS $rna1 $rna2
     } [string length $rnalcs]
 
     test {LCS with KEYS option} {
         r set virus1 $rna1
         r set virus2 $rna2
-        r LCS KEYS virus1 virus2
+        r STRALGO LCS KEYS virus1 virus2
     } $rnalcs
 
     test {LCS indexes} {
-        dict get [r LCS IDX KEYS virus1 virus2] matches
+        dict get [r STRALGO LCS IDX KEYS virus1 virus2] matches
     } {{{238 238} {239 239}} {{236 236} {238 238}} {{229 230} {236 237}} {{224 224} {235 235}} {{1 222} {13 234}}}
 
     test {LCS indexes with match len} {
-        dict get [r LCS IDX KEYS virus1 virus2 WITHMATCHLEN] matches
+        dict get [r STRALGO LCS IDX KEYS virus1 virus2 WITHMATCHLEN] matches
     } {{{238 238} {239 239} 1} {{236 236} {238 238} 1} {{229 230} {236 237} 2} {{224 224} {235 235} 1} {{1 222} {13 234} 222}}
 
     test {LCS indexes with match len and minimum match len} {
-        dict get [r LCS IDX KEYS virus1 virus2 WITHMATCHLEN MINMATCHLEN 5] matches
+        dict get [r STRALGO LCS IDX KEYS virus1 virus2 WITHMATCHLEN MINMATCHLEN 5] matches
     } {{{1 222} {13 234} 222}}
 }

From 486e45ffafe2349e7bd8238d6d3b48caf57accd9 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <matolson@amazon.com>
Date: Fri, 24 Apr 2020 16:59:24 -0700
Subject: [PATCH 0309/1098] Implemented CRC64 based on slice by 4

---
 src/Makefile |   4 +-
 src/crc64.c  | 270 ++++++++++++++++++++++-----------------------------
 src/crc64.h  |   1 +
 src/rdb.c    |   3 +-
 src/server.c |   1 +
 5 files changed, 123 insertions(+), 156 deletions(-)

diff --git a/src/Makefile b/src/Makefile
index 3f982cc8e..f9922afce 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -206,9 +206,9 @@ endif
 
 REDIS_SERVER_NAME=redis-server
 REDIS_SENTINEL_NAME=redis-sentinel
-REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o
+REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o
 REDIS_CLI_NAME=redis-cli
-REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crc64.o siphash.o crc16.o
+REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o
 REDIS_BENCHMARK_NAME=redis-benchmark
 REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o
 REDIS_CHECK_RDB_NAME=redis-check-rdb
diff --git a/src/crc64.c b/src/crc64.c
index f1f764922..165941dea 100644
--- a/src/crc64.c
+++ b/src/crc64.c
@@ -1,16 +1,5 @@
-/* Redis uses the CRC64 variant with "Jones" coefficients and init value of 0.
- *
- * Specification of this CRC64 variant follows:
- * Name: crc-64-jones
- * Width: 64 bites
- * Poly: 0xad93d23594c935a9
- * Reflected In: True
- * Xor_In: 0xffffffffffffffff
- * Reflected_Out: True
- * Xor_Out: 0x0
- * Check("123456789"): 0xe9c6d914c4b8d9ca
- *
- * Copyright (c) 2012, Salvatore Sanfilippo <antirez at gmail dot com>
+/* Copyright (c) 2014, Matt Stancliff <matt@genges.com>
+ * Copyright (c) 2020, Amazon Web Services
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,159 +26,134 @@
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE. */
 
-#include <stdint.h>
+#include "crc64.h"
+#include "crcspeed.h"
+static uint64_t crc64_table[8][256] = {{0}};
 
-static const uint64_t crc64_tab[256] = {
-    UINT64_C(0x0000000000000000), UINT64_C(0x7ad870c830358979),
-    UINT64_C(0xf5b0e190606b12f2), UINT64_C(0x8f689158505e9b8b),
-    UINT64_C(0xc038e5739841b68f), UINT64_C(0xbae095bba8743ff6),
-    UINT64_C(0x358804e3f82aa47d), UINT64_C(0x4f50742bc81f2d04),
-    UINT64_C(0xab28ecb46814fe75), UINT64_C(0xd1f09c7c5821770c),
-    UINT64_C(0x5e980d24087fec87), UINT64_C(0x24407dec384a65fe),
-    UINT64_C(0x6b1009c7f05548fa), UINT64_C(0x11c8790fc060c183),
-    UINT64_C(0x9ea0e857903e5a08), UINT64_C(0xe478989fa00bd371),
-    UINT64_C(0x7d08ff3b88be6f81), UINT64_C(0x07d08ff3b88be6f8),
-    UINT64_C(0x88b81eabe8d57d73), UINT64_C(0xf2606e63d8e0f40a),
-    UINT64_C(0xbd301a4810ffd90e), UINT64_C(0xc7e86a8020ca5077),
-    UINT64_C(0x4880fbd87094cbfc), UINT64_C(0x32588b1040a14285),
-    UINT64_C(0xd620138fe0aa91f4), UINT64_C(0xacf86347d09f188d),
-    UINT64_C(0x2390f21f80c18306), UINT64_C(0x594882d7b0f40a7f),
-    UINT64_C(0x1618f6fc78eb277b), UINT64_C(0x6cc0863448deae02),
-    UINT64_C(0xe3a8176c18803589), UINT64_C(0x997067a428b5bcf0),
-    UINT64_C(0xfa11fe77117cdf02), UINT64_C(0x80c98ebf2149567b),
-    UINT64_C(0x0fa11fe77117cdf0), UINT64_C(0x75796f2f41224489),
-    UINT64_C(0x3a291b04893d698d), UINT64_C(0x40f16bccb908e0f4),
-    UINT64_C(0xcf99fa94e9567b7f), UINT64_C(0xb5418a5cd963f206),
-    UINT64_C(0x513912c379682177), UINT64_C(0x2be1620b495da80e),
-    UINT64_C(0xa489f35319033385), UINT64_C(0xde51839b2936bafc),
-    UINT64_C(0x9101f7b0e12997f8), UINT64_C(0xebd98778d11c1e81),
-    UINT64_C(0x64b116208142850a), UINT64_C(0x1e6966e8b1770c73),
-    UINT64_C(0x8719014c99c2b083), UINT64_C(0xfdc17184a9f739fa),
-    UINT64_C(0x72a9e0dcf9a9a271), UINT64_C(0x08719014c99c2b08),
-    UINT64_C(0x4721e43f0183060c), UINT64_C(0x3df994f731b68f75),
-    UINT64_C(0xb29105af61e814fe), UINT64_C(0xc849756751dd9d87),
-    UINT64_C(0x2c31edf8f1d64ef6), UINT64_C(0x56e99d30c1e3c78f),
-    UINT64_C(0xd9810c6891bd5c04), UINT64_C(0xa3597ca0a188d57d),
-    UINT64_C(0xec09088b6997f879), UINT64_C(0x96d1784359a27100),
-    UINT64_C(0x19b9e91b09fcea8b), UINT64_C(0x636199d339c963f2),
-    UINT64_C(0xdf7adabd7a6e2d6f), UINT64_C(0xa5a2aa754a5ba416),
-    UINT64_C(0x2aca3b2d1a053f9d), UINT64_C(0x50124be52a30b6e4),
-    UINT64_C(0x1f423fcee22f9be0), UINT64_C(0x659a4f06d21a1299),
-    UINT64_C(0xeaf2de5e82448912), UINT64_C(0x902aae96b271006b),
-    UINT64_C(0x74523609127ad31a), UINT64_C(0x0e8a46c1224f5a63),
-    UINT64_C(0x81e2d7997211c1e8), UINT64_C(0xfb3aa75142244891),
-    UINT64_C(0xb46ad37a8a3b6595), UINT64_C(0xceb2a3b2ba0eecec),
-    UINT64_C(0x41da32eaea507767), UINT64_C(0x3b024222da65fe1e),
-    UINT64_C(0xa2722586f2d042ee), UINT64_C(0xd8aa554ec2e5cb97),
-    UINT64_C(0x57c2c41692bb501c), UINT64_C(0x2d1ab4dea28ed965),
-    UINT64_C(0x624ac0f56a91f461), UINT64_C(0x1892b03d5aa47d18),
-    UINT64_C(0x97fa21650afae693), UINT64_C(0xed2251ad3acf6fea),
-    UINT64_C(0x095ac9329ac4bc9b), UINT64_C(0x7382b9faaaf135e2),
-    UINT64_C(0xfcea28a2faafae69), UINT64_C(0x8632586aca9a2710),
-    UINT64_C(0xc9622c4102850a14), UINT64_C(0xb3ba5c8932b0836d),
-    UINT64_C(0x3cd2cdd162ee18e6), UINT64_C(0x460abd1952db919f),
-    UINT64_C(0x256b24ca6b12f26d), UINT64_C(0x5fb354025b277b14),
-    UINT64_C(0xd0dbc55a0b79e09f), UINT64_C(0xaa03b5923b4c69e6),
-    UINT64_C(0xe553c1b9f35344e2), UINT64_C(0x9f8bb171c366cd9b),
-    UINT64_C(0x10e3202993385610), UINT64_C(0x6a3b50e1a30ddf69),
-    UINT64_C(0x8e43c87e03060c18), UINT64_C(0xf49bb8b633338561),
-    UINT64_C(0x7bf329ee636d1eea), UINT64_C(0x012b592653589793),
-    UINT64_C(0x4e7b2d0d9b47ba97), UINT64_C(0x34a35dc5ab7233ee),
-    UINT64_C(0xbbcbcc9dfb2ca865), UINT64_C(0xc113bc55cb19211c),
-    UINT64_C(0x5863dbf1e3ac9dec), UINT64_C(0x22bbab39d3991495),
-    UINT64_C(0xadd33a6183c78f1e), UINT64_C(0xd70b4aa9b3f20667),
-    UINT64_C(0x985b3e827bed2b63), UINT64_C(0xe2834e4a4bd8a21a),
-    UINT64_C(0x6debdf121b863991), UINT64_C(0x1733afda2bb3b0e8),
-    UINT64_C(0xf34b37458bb86399), UINT64_C(0x8993478dbb8deae0),
-    UINT64_C(0x06fbd6d5ebd3716b), UINT64_C(0x7c23a61ddbe6f812),
-    UINT64_C(0x3373d23613f9d516), UINT64_C(0x49aba2fe23cc5c6f),
-    UINT64_C(0xc6c333a67392c7e4), UINT64_C(0xbc1b436e43a74e9d),
-    UINT64_C(0x95ac9329ac4bc9b5), UINT64_C(0xef74e3e19c7e40cc),
-    UINT64_C(0x601c72b9cc20db47), UINT64_C(0x1ac40271fc15523e),
-    UINT64_C(0x5594765a340a7f3a), UINT64_C(0x2f4c0692043ff643),
-    UINT64_C(0xa02497ca54616dc8), UINT64_C(0xdafce7026454e4b1),
-    UINT64_C(0x3e847f9dc45f37c0), UINT64_C(0x445c0f55f46abeb9),
-    UINT64_C(0xcb349e0da4342532), UINT64_C(0xb1eceec59401ac4b),
-    UINT64_C(0xfebc9aee5c1e814f), UINT64_C(0x8464ea266c2b0836),
-    UINT64_C(0x0b0c7b7e3c7593bd), UINT64_C(0x71d40bb60c401ac4),
-    UINT64_C(0xe8a46c1224f5a634), UINT64_C(0x927c1cda14c02f4d),
-    UINT64_C(0x1d148d82449eb4c6), UINT64_C(0x67ccfd4a74ab3dbf),
-    UINT64_C(0x289c8961bcb410bb), UINT64_C(0x5244f9a98c8199c2),
-    UINT64_C(0xdd2c68f1dcdf0249), UINT64_C(0xa7f41839ecea8b30),
-    UINT64_C(0x438c80a64ce15841), UINT64_C(0x3954f06e7cd4d138),
-    UINT64_C(0xb63c61362c8a4ab3), UINT64_C(0xcce411fe1cbfc3ca),
-    UINT64_C(0x83b465d5d4a0eece), UINT64_C(0xf96c151de49567b7),
-    UINT64_C(0x76048445b4cbfc3c), UINT64_C(0x0cdcf48d84fe7545),
-    UINT64_C(0x6fbd6d5ebd3716b7), UINT64_C(0x15651d968d029fce),
-    UINT64_C(0x9a0d8ccedd5c0445), UINT64_C(0xe0d5fc06ed698d3c),
-    UINT64_C(0xaf85882d2576a038), UINT64_C(0xd55df8e515432941),
-    UINT64_C(0x5a3569bd451db2ca), UINT64_C(0x20ed197575283bb3),
-    UINT64_C(0xc49581ead523e8c2), UINT64_C(0xbe4df122e51661bb),
-    UINT64_C(0x3125607ab548fa30), UINT64_C(0x4bfd10b2857d7349),
-    UINT64_C(0x04ad64994d625e4d), UINT64_C(0x7e7514517d57d734),
-    UINT64_C(0xf11d85092d094cbf), UINT64_C(0x8bc5f5c11d3cc5c6),
-    UINT64_C(0x12b5926535897936), UINT64_C(0x686de2ad05bcf04f),
-    UINT64_C(0xe70573f555e26bc4), UINT64_C(0x9ddd033d65d7e2bd),
-    UINT64_C(0xd28d7716adc8cfb9), UINT64_C(0xa85507de9dfd46c0),
-    UINT64_C(0x273d9686cda3dd4b), UINT64_C(0x5de5e64efd965432),
-    UINT64_C(0xb99d7ed15d9d8743), UINT64_C(0xc3450e196da80e3a),
-    UINT64_C(0x4c2d9f413df695b1), UINT64_C(0x36f5ef890dc31cc8),
-    UINT64_C(0x79a59ba2c5dc31cc), UINT64_C(0x037deb6af5e9b8b5),
-    UINT64_C(0x8c157a32a5b7233e), UINT64_C(0xf6cd0afa9582aa47),
-    UINT64_C(0x4ad64994d625e4da), UINT64_C(0x300e395ce6106da3),
-    UINT64_C(0xbf66a804b64ef628), UINT64_C(0xc5bed8cc867b7f51),
-    UINT64_C(0x8aeeace74e645255), UINT64_C(0xf036dc2f7e51db2c),
-    UINT64_C(0x7f5e4d772e0f40a7), UINT64_C(0x05863dbf1e3ac9de),
-    UINT64_C(0xe1fea520be311aaf), UINT64_C(0x9b26d5e88e0493d6),
-    UINT64_C(0x144e44b0de5a085d), UINT64_C(0x6e963478ee6f8124),
-    UINT64_C(0x21c640532670ac20), UINT64_C(0x5b1e309b16452559),
-    UINT64_C(0xd476a1c3461bbed2), UINT64_C(0xaeaed10b762e37ab),
-    UINT64_C(0x37deb6af5e9b8b5b), UINT64_C(0x4d06c6676eae0222),
-    UINT64_C(0xc26e573f3ef099a9), UINT64_C(0xb8b627f70ec510d0),
-    UINT64_C(0xf7e653dcc6da3dd4), UINT64_C(0x8d3e2314f6efb4ad),
-    UINT64_C(0x0256b24ca6b12f26), UINT64_C(0x788ec2849684a65f),
-    UINT64_C(0x9cf65a1b368f752e), UINT64_C(0xe62e2ad306bafc57),
-    UINT64_C(0x6946bb8b56e467dc), UINT64_C(0x139ecb4366d1eea5),
-    UINT64_C(0x5ccebf68aecec3a1), UINT64_C(0x2616cfa09efb4ad8),
-    UINT64_C(0xa97e5ef8cea5d153), UINT64_C(0xd3a62e30fe90582a),
-    UINT64_C(0xb0c7b7e3c7593bd8), UINT64_C(0xca1fc72bf76cb2a1),
-    UINT64_C(0x45775673a732292a), UINT64_C(0x3faf26bb9707a053),
-    UINT64_C(0x70ff52905f188d57), UINT64_C(0x0a2722586f2d042e),
-    UINT64_C(0x854fb3003f739fa5), UINT64_C(0xff97c3c80f4616dc),
-    UINT64_C(0x1bef5b57af4dc5ad), UINT64_C(0x61372b9f9f784cd4),
-    UINT64_C(0xee5fbac7cf26d75f), UINT64_C(0x9487ca0fff135e26),
-    UINT64_C(0xdbd7be24370c7322), UINT64_C(0xa10fceec0739fa5b),
-    UINT64_C(0x2e675fb4576761d0), UINT64_C(0x54bf2f7c6752e8a9),
-    UINT64_C(0xcdcf48d84fe75459), UINT64_C(0xb71738107fd2dd20),
-    UINT64_C(0x387fa9482f8c46ab), UINT64_C(0x42a7d9801fb9cfd2),
-    UINT64_C(0x0df7adabd7a6e2d6), UINT64_C(0x772fdd63e7936baf),
-    UINT64_C(0xf8474c3bb7cdf024), UINT64_C(0x829f3cf387f8795d),
-    UINT64_C(0x66e7a46c27f3aa2c), UINT64_C(0x1c3fd4a417c62355),
-    UINT64_C(0x935745fc4798b8de), UINT64_C(0xe98f353477ad31a7),
-    UINT64_C(0xa6df411fbfb21ca3), UINT64_C(0xdc0731d78f8795da),
-    UINT64_C(0x536fa08fdfd90e51), UINT64_C(0x29b7d047efec8728),
-};
+/******************** BEGIN GENERATED PYCRC FUNCTIONS ********************/
+/**
+ * Generated on Sun Dec 21 14:14:07 2014,
+ * by pycrc v0.8.2, https://www.tty1.net/pycrc/
+ *
+ * LICENSE ON GENERATED CODE:
+ * ==========================
+ * As of version 0.6, pycrc is released under the terms of the MIT licence.
+ * The code generated by pycrc is not considered a substantial portion of the
+ * software, therefore the author of pycrc will not claim any copyright on
+ * the generated code.
+ * ==========================
+ *
+ * CRC configuration:
+ *    Width        = 64
+ *    Poly         = 0xad93d23594c935a9
+ *    XorIn        = 0xffffffffffffffff
+ *    ReflectIn    = True
+ *    XorOut       = 0x0000000000000000
+ *    ReflectOut   = True
+ *    Algorithm    = bit-by-bit-fast
+ *
+ * Modifications after generation (by matt):
+ *   - included finalize step in-line with update for single-call generation
+ *   - re-worked some inner variable architectures
+ *   - adjusted function parameters to match expected prototypes.
+ *****************************************************************************/
 
-uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) {
-    uint64_t j;
+/**
+ * Reflect all bits of a \a data word of \a data_len bytes.
+ *
+ * \param data         The data word to be reflected.
+ * \param data_len     The width of \a data expressed in number of bits.
+ * \return             The reflected data.
+ *****************************************************************************/
+static inline uint_fast64_t crc_reflect(uint_fast64_t data, size_t data_len) {
+    uint_fast64_t ret = data & 0x01;
 
-    for (j = 0; j < l; j++) {
-        uint8_t byte = s[j];
-        crc = crc64_tab[(uint8_t)crc ^ byte] ^ (crc >> 8);
+    for (size_t i = 1; i < data_len; i++) {
+        data >>= 1;
+        ret = (ret << 1) | (data & 0x01);
     }
-    return crc;
+
+    return ret;
+}
+
+/**
+ *  Update the crc value with new data.
+ *
+ * \param crc      The current crc value.
+ * \param data     Pointer to a buffer of \a data_len bytes.
+ * \param data_len Number of bytes in the \a data buffer.
+ * \return         The updated crc value.
+ ******************************************************************************/
+uint64_t _crc64(uint_fast64_t crc, const void *in_data, const uint64_t len) {
+    const uint8_t *data = in_data;
+    unsigned long long bit;
+
+    for (uint64_t offset = 0; offset < len; offset++) {
+        uint8_t c = data[offset];
+        for (uint_fast8_t i = 0x01; i & 0xff; i <<= 1) {
+            bit = crc & 0x8000000000000000;
+            if (c & i) {
+                bit = !bit;
+            }
+
+            crc <<= 1;
+            if (bit) {
+                crc ^= POLY;
+            }
+        }
+
+        crc &= 0xffffffffffffffff;
+    }
+
+    crc = crc & 0xffffffffffffffff;
+    return crc_reflect(crc, 64) ^ 0x0000000000000000;
+}
+
+/******************** END GENERATED PYCRC FUNCTIONS ********************/
+
+/* Initializes the 16KB lookup tables. */
+void crc64_init(void) {
+    crcspeed64native_init(_crc64, crc64_table);
+}
+
+/* Compute crc64 */
+uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) {
+    return crcspeed64native(crc64_table, crc, (void *) s, l);
 }
 
 /* Test main */
-#ifdef REDIS_TEST
+#if defined(REDIS_TEST) || defined(REDIS_TEST_MAIN)
 #include <stdio.h>
 
 #define UNUSED(x) (void)(x)
 int crc64Test(int argc, char *argv[]) {
     UNUSED(argc);
     UNUSED(argv);
-    printf("e9c6d914c4b8d9ca == %016llx\n",
-        (unsigned long long) crc64(0,(unsigned char*)"123456789",9));
+    crc64_init();
+    printf("[calcula]: e9c6d914c4b8d9ca == %016" PRIx64 "\n",
+           (uint64_t)_crc64(0, "123456789", 9));
+    printf("[64speed]: e9c6d914c4b8d9ca == %016" PRIx64 "\n",
+           (uint64_t)crc64speed(0, "123456789", 9));
+    char li[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed "
+                "do eiusmod tempor incididunt ut labore et dolore magna "
+                "aliqua. Ut enim ad minim veniam, quis nostrud exercitation "
+                "ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis "
+                "aute irure dolor in reprehenderit in voluptate velit esse "
+                "cillum dolore eu fugiat nulla pariatur. Excepteur sint "
+                "occaecat cupidatat non proident, sunt in culpa qui officia "
+                "deserunt mollit anim id est laborum.";
+    printf("[calcula]: c7794709e69683b3 == %016" PRIx64 "\n",
+           (uint64_t)_crc64(0, li, sizeof(li)));
+    printf("[64speed]: c7794709e69683b3 == %016" PRIx64 "\n",
+           (uint64_t)crc64(0, li, sizeof(li)));
     return 0;
 }
+
+#endif
+
+#ifdef REDIS_TEST_MAIN
+int main(int argc, char *argv[]) {
+    return crc64Test(argc, argv);
+}
+
 #endif
diff --git a/src/crc64.h b/src/crc64.h
index c9fca519d..60c42345f 100644
--- a/src/crc64.h
+++ b/src/crc64.h
@@ -3,6 +3,7 @@
 
 #include <stdint.h>
 
+void crc64_init(void);
 uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l);
 
 #ifdef REDIS_TEST
diff --git a/src/rdb.c b/src/rdb.c
index 143b6c325..9f6bf13f1 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2291,7 +2291,8 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
             if (cksum == 0) {
                 serverLog(LL_WARNING,"RDB file was saved with checksum disabled: no check performed.");
             } else if (cksum != expected) {
-                serverLog(LL_WARNING,"Wrong RDB checksum. Aborting now.");
+                serverLog(LL_WARNING,"Wrong RDB checksum expected: (%llx) but "
+                    "got (%llx). Aborting now.",expected,cksum);
                 rdbExitReportCorruptRDB("RDB CRC error");
             }
         }
diff --git a/src/server.c b/src/server.c
index f2fe54e1c..7835a261a 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2892,6 +2892,7 @@ void initServer(void) {
     scriptingInit(1);
     slowlogInit();
     latencyMonitorInit();
+    crc64_init();
 }
 
 /* Some steps in server initialization need to be done last (after modules

From 2192a91d62be77987d6ac1691f380300168c31d5 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <matolson@amazon.com>
Date: Fri, 24 Apr 2020 17:05:37 -0700
Subject: [PATCH 0310/1098] Made crc64 test consistent

---
 src/crc64.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/crc64.c b/src/crc64.c
index 165941dea..4cbc019f6 100644
--- a/src/crc64.c
+++ b/src/crc64.c
@@ -30,6 +30,7 @@
 #include "crcspeed.h"
 static uint64_t crc64_table[8][256] = {{0}};
 
+#define POLY UINT64_C(0xad93d23594c935a9)
 /******************** BEGIN GENERATED PYCRC FUNCTIONS ********************/
 /**
  * Generated on Sun Dec 21 14:14:07 2014,
@@ -122,7 +123,7 @@ uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) {
 }
 
 /* Test main */
-#if defined(REDIS_TEST) || defined(REDIS_TEST_MAIN)
+#ifdef REDIS_TEST
 #include <stdio.h>
 
 #define UNUSED(x) (void)(x)
@@ -133,7 +134,7 @@ int crc64Test(int argc, char *argv[]) {
     printf("[calcula]: e9c6d914c4b8d9ca == %016" PRIx64 "\n",
            (uint64_t)_crc64(0, "123456789", 9));
     printf("[64speed]: e9c6d914c4b8d9ca == %016" PRIx64 "\n",
-           (uint64_t)crc64speed(0, "123456789", 9));
+           (uint64_t)crc64(0, "123456789", 9));
     char li[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed "
                 "do eiusmod tempor incididunt ut labore et dolore magna "
                 "aliqua. Ut enim ad minim veniam, quis nostrud exercitation "

From a48dfe98cf460fcd032b62ed776827d791d06abd Mon Sep 17 00:00:00 2001
From: Madelyn Olson <matolson@amazon.com>
Date: Fri, 24 Apr 2020 17:11:21 -0700
Subject: [PATCH 0311/1098] Added crcspeed library

---
 src/crcspeed.c | 281 +++++++++++++++++++++++++++++++++++++++++++++++++
 src/crcspeed.h |  60 +++++++++++
 2 files changed, 341 insertions(+)
 create mode 100644 src/crcspeed.c
 create mode 100644 src/crcspeed.h

diff --git a/src/crcspeed.c b/src/crcspeed.c
new file mode 100644
index 000000000..d2d97a8c7
--- /dev/null
+++ b/src/crcspeed.c
@@ -0,0 +1,281 @@
+/*
+ * Copyright (C) 2013 Mark Adler
+ * Originally by: crc64.c Version 1.4  16 Dec 2013  Mark Adler
+ * Modifications by Matt Stancliff <matt@genges.com>:
+ *   - removed CRC64-specific behavior
+ *   - added generation of lookup tables by parameters
+ *   - removed inversion of CRC input/result
+ *   - removed automatic initialization in favor of explicit initialization
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the author be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+
+  Mark Adler
+  madler@alumni.caltech.edu
+ */
+
+#include "crcspeed.h"
+
+/* Fill in a CRC constants table. */
+void crcspeed64little_init(crcfn64 crcfn, uint64_t table[8][256]) {
+    uint64_t crc;
+
+    /* generate CRCs for all single byte sequences */
+    for (int n = 0; n < 256; n++) {
+        table[0][n] = crcfn(0, &n, 1);
+    }
+
+    /* generate nested CRC table for future slice-by-8 lookup */
+    for (int n = 0; n < 256; n++) {
+        crc = table[0][n];
+        for (int k = 1; k < 8; k++) {
+            crc = table[0][crc & 0xff] ^ (crc >> 8);
+            table[k][n] = crc;
+        }
+    }
+}
+
+void crcspeed16little_init(crcfn16 crcfn, uint16_t table[8][256]) {
+    uint16_t crc;
+
+    /* generate CRCs for all single byte sequences */
+    for (int n = 0; n < 256; n++) {
+        table[0][n] = crcfn(0, &n, 1);
+    }
+
+    /* generate nested CRC table for future slice-by-8 lookup */
+    for (int n = 0; n < 256; n++) {
+        crc = table[0][n];
+        for (int k = 1; k < 8; k++) {
+            crc = table[0][(crc >> 8) & 0xff] ^ (crc << 8);
+            table[k][n] = crc;
+        }
+    }
+}
+
+/* Reverse the bytes in a 64-bit word. */
+static inline uint64_t rev8(uint64_t a) {
+#if defined(__GNUC__) || defined(__clang__)
+    return __builtin_bswap64(a);
+#else
+    uint64_t m;
+
+    m = UINT64_C(0xff00ff00ff00ff);
+    a = ((a >> 8) & m) | (a & m) << 8;
+    m = UINT64_C(0xffff0000ffff);
+    a = ((a >> 16) & m) | (a & m) << 16;
+    return a >> 32 | a << 32;
+#endif
+}
+
+/* This function is called once to initialize the CRC table for use on a
+   big-endian architecture. */
+void crcspeed64big_init(crcfn64 fn, uint64_t big_table[8][256]) {
+    /* Create the little endian table then reverse all the entires. */
+    crcspeed64little_init(fn, big_table);
+    for (int k = 0; k < 8; k++) {
+        for (int n = 0; n < 256; n++) {
+            big_table[k][n] = rev8(big_table[k][n]);
+        }
+    }
+}
+
+void crcspeed16big_init(crcfn16 fn, uint16_t big_table[8][256]) {
+    /* Create the little endian table then reverse all the entires. */
+    crcspeed16little_init(fn, big_table);
+    for (int k = 0; k < 8; k++) {
+        for (int n = 0; n < 256; n++) {
+            big_table[k][n] = rev8(big_table[k][n]);
+        }
+    }
+}
+
+/* Calculate a non-inverted CRC multiple bytes at a time on a little-endian
+ * architecture. If you need inverted CRC, invert *before* calling and invert
+ * *after* calling.
+ * 64 bit crc = process 8 bytes at once;
+ */
+uint64_t crcspeed64little(uint64_t little_table[8][256], uint64_t crc,
+                          void *buf, size_t len) {
+    unsigned char *next = buf;
+
+    /* process individual bytes until we reach an 8-byte aligned pointer */
+    while (len && ((uintptr_t)next & 7) != 0) {
+        crc = little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
+        len--;
+    }
+
+    /* fast middle processing, 8 bytes (aligned!) per loop */
+    while (len >= 8) {
+        crc ^= *(uint64_t *)next;
+        crc = little_table[7][crc & 0xff] ^
+              little_table[6][(crc >> 8) & 0xff] ^
+              little_table[5][(crc >> 16) & 0xff] ^
+              little_table[4][(crc >> 24) & 0xff] ^
+              little_table[3][(crc >> 32) & 0xff] ^
+              little_table[2][(crc >> 40) & 0xff] ^
+              little_table[1][(crc >> 48) & 0xff] ^
+              little_table[0][crc >> 56];
+        next += 8;
+        len -= 8;
+    }
+
+    /* process remaining bytes (can't be larger than 8) */
+    while (len) {
+        crc = little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
+        len--;
+    }
+
+    return crc;
+}
+
+uint16_t crcspeed16little(uint16_t little_table[8][256], uint16_t crc,
+                          void *buf, size_t len) {
+    unsigned char *next = buf;
+
+    /* process individual bytes until we reach an 8-byte aligned pointer */
+    while (len && ((uintptr_t)next & 7) != 0) {
+        crc = little_table[0][((crc >> 8) ^ *next++) & 0xff] ^ (crc << 8);
+        len--;
+    }
+
+    /* fast middle processing, 8 bytes (aligned!) per loop */
+    while (len >= 8) {
+        uint64_t n = *(uint64_t *)next;
+        crc = little_table[7][(n & 0xff) ^ ((crc >> 8) & 0xff)] ^
+              little_table[6][((n >> 8) & 0xff) ^ (crc & 0xff)] ^
+              little_table[5][(n >> 16) & 0xff] ^
+              little_table[4][(n >> 24) & 0xff] ^
+              little_table[3][(n >> 32) & 0xff] ^
+              little_table[2][(n >> 40) & 0xff] ^
+              little_table[1][(n >> 48) & 0xff] ^
+              little_table[0][n >> 56];
+        next += 8;
+        len -= 8;
+    }
+
+    /* process remaining bytes (can't be larger than 8) */
+    while (len) {
+        crc = little_table[0][((crc >> 8) ^ *next++) & 0xff] ^ (crc << 8);
+        len--;
+    }
+
+    return crc;
+}
+
+/* Calculate a non-inverted CRC eight bytes at a time on a big-endian
+ * architecture.
+ */
+uint64_t crcspeed64big(uint64_t big_table[8][256], uint64_t crc, void *buf,
+                       size_t len) {
+    unsigned char *next = buf;
+
+    crc = rev8(crc);
+    while (len && ((uintptr_t)next & 7) != 0) {
+        crc = big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8);
+        len--;
+    }
+
+    while (len >= 8) {
+        crc ^= *(uint64_t *)next;
+        crc = big_table[0][crc & 0xff] ^
+              big_table[1][(crc >> 8) & 0xff] ^
+              big_table[2][(crc >> 16) & 0xff] ^
+              big_table[3][(crc >> 24) & 0xff] ^
+              big_table[4][(crc >> 32) & 0xff] ^
+              big_table[5][(crc >> 40) & 0xff] ^
+              big_table[6][(crc >> 48) & 0xff] ^
+              big_table[7][crc >> 56];
+        next += 8;
+        len -= 8;
+    }
+
+    while (len) {
+        crc = big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8);
+        len--;
+    }
+
+    return rev8(crc);
+}
+
+/* WARNING: Completely untested on big endian architecture.  Possibly broken. */
+uint16_t crcspeed16big(uint16_t big_table[8][256], uint16_t crc_in, void *buf,
+                       size_t len) {
+    unsigned char *next = buf;
+    uint64_t crc = crc_in;
+
+    crc = rev8(crc);
+    while (len && ((uintptr_t)next & 7) != 0) {
+        crc = big_table[0][((crc >> (56 - 8)) ^ *next++) & 0xff] ^ (crc >> 8);
+        len--;
+    }
+
+    while (len >= 8) {
+        uint64_t n = *(uint64_t *)next;
+        crc = big_table[0][(n & 0xff) ^ ((crc >> (56 - 8)) & 0xff)] ^
+              big_table[1][((n >> 8) & 0xff) ^ (crc & 0xff)] ^
+              big_table[2][(n >> 16) & 0xff] ^
+              big_table[3][(n >> 24) & 0xff] ^
+              big_table[4][(n >> 32) & 0xff] ^
+              big_table[5][(n >> 40) & 0xff] ^
+              big_table[6][(n >> 48) & 0xff] ^
+              big_table[7][n >> 56];
+        next += 8;
+        len -= 8;
+    }
+
+    while (len) {
+        crc = big_table[0][((crc >> (56 - 8)) ^ *next++) & 0xff] ^ (crc >> 8);
+        len--;
+    }
+
+    return rev8(crc);
+}
+
+/* Return the CRC of buf[0..len-1] with initial crc, processing eight bytes
+   at a time using passed-in lookup table.
+   This selects one of two routines depending on the endianess of
+   the architecture. */
+uint64_t crcspeed64native(uint64_t table[8][256], uint64_t crc, void *buf,
+                          size_t len) {
+    uint64_t n = 1;
+
+    return *(char *)&n ? crcspeed64little(table, crc, buf, len)
+                       : crcspeed64big(table, crc, buf, len);
+}
+
+uint16_t crcspeed16native(uint16_t table[8][256], uint16_t crc, void *buf,
+                          size_t len) {
+    uint64_t n = 1;
+
+    return *(char *)&n ? crcspeed16little(table, crc, buf, len)
+                       : crcspeed16big(table, crc, buf, len);
+}
+
+/* Initialize CRC lookup table in architecture-dependent manner. */
+void crcspeed64native_init(crcfn64 fn, uint64_t table[8][256]) {
+    uint64_t n = 1;
+
+    *(char *)&n ? crcspeed64little_init(fn, table)
+                : crcspeed64big_init(fn, table);
+}
+
+void crcspeed16native_init(crcfn16 fn, uint16_t table[8][256]) {
+    uint64_t n = 1;
+
+    *(char *)&n ? crcspeed16little_init(fn, table)
+                : crcspeed16big_init(fn, table);
+}
diff --git a/src/crcspeed.h b/src/crcspeed.h
new file mode 100644
index 000000000..d7ee95ebb
--- /dev/null
+++ b/src/crcspeed.h
@@ -0,0 +1,60 @@
+/* Copyright (c) 2014, Matt Stancliff <matt@genges.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE. */
+
+#ifndef CRCSPEED_H
+#define CRCSPEED_H
+
+#include <inttypes.h>
+#include <stdio.h>
+
+typedef uint64_t (*crcfn64)(uint64_t, const void *, const uint64_t);
+typedef uint16_t (*crcfn16)(uint16_t, const void *, const uint64_t);
+
+/* CRC-64 */
+void crcspeed64little_init(crcfn64 fn, uint64_t table[8][256]);
+void crcspeed64big_init(crcfn64 fn, uint64_t table[8][256]);
+void crcspeed64native_init(crcfn64 fn, uint64_t table[8][256]);
+
+uint64_t crcspeed64little(uint64_t table[8][256], uint64_t crc, void *buf,
+                          size_t len);
+uint64_t crcspeed64big(uint64_t table[8][256], uint64_t crc, void *buf,
+                       size_t len);
+uint64_t crcspeed64native(uint64_t table[8][256], uint64_t crc, void *buf,
+                          size_t len);
+
+/* CRC-16 */
+void crcspeed16little_init(crcfn16 fn, uint16_t table[8][256]);
+void crcspeed16big_init(crcfn16 fn, uint16_t table[8][256]);
+void crcspeed16native_init(crcfn16 fn, uint16_t table[8][256]);
+
+uint16_t crcspeed16little(uint16_t table[8][256], uint16_t crc, void *buf,
+                          size_t len);
+uint16_t crcspeed16big(uint16_t table[8][256], uint16_t crc, void *buf,
+                       size_t len);
+uint16_t crcspeed16native(uint16_t table[8][256], uint16_t crc, void *buf,
+                          size_t len);
+#endif

From 3497fd007f69f9d37e311e9b93dfccea6a826f95 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 27 Apr 2020 13:35:17 +0200
Subject: [PATCH 0312/1098] Fix STRALGO command flags.

---
 src/server.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index f2fe54e1c..db1d3780c 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1007,7 +1007,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"stralgo",stralgoCommand,-2,
-     "write use-memory @string",
+     "read-only @string",
      0,lcsGetKeys,0,0,0,0,0,0}
 };
 

From 4447ddc8bb36879db9fe49498165b360bf35ba1b Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 23 Apr 2020 15:04:42 +0300
Subject: [PATCH 0313/1098] Keep track of meaningful replication offset in
 replicas too

Now both master and replicas keep track of the last replication offset
that contains meaningful data (ignoring the tailing pings), and both
trim that tail from the replication backlog, and the offset with which
they try to use for psync.

the implication is that if someone missed some pings, or even have
excessive pings that the promoted replica has, it'll still be able to
psync (avoid full sync).

the downside (which was already committed) is that replicas running old
code may fail to psync, since the promoted replica trims pings form it's
backlog.

This commit adds a test that reproduces several cases of promotions and
demotions with stale and non-stale pings

Background:
The mearningful offset on the master was added recently to solve a problem were
the master is left all alone, injecting PINGs into it's backlog when no one is
listening and then gets demoted and tries to replicate from a replica that didn't
have any of the PINGs (or at least not the last ones).

however, consider this case:
master A has two replicas (B and C) replicating directly from it.
there's no traffic at all, and also no network issues, just many pings in the
tail of the backlog. now B gets promoted, A becomes a replica of B, and C
remains a replica of A. when A gets demoted, it trims the pings from its
backlog, and successfully replicate from B. however, C is still aware of
these PINGs, when it'll disconnect and re-connect to A, it'll ask for something
that's not in the backlog anymore (since A trimmed the tail of it's backlog),
and be forced to do a full sync (something it didn't have to do before the
meaningful offset fix).

Besides that, the psync2 test was always failing randomly here and there, it
turns out the reason were PINGs. Investigating it shows the following scenario:

cycle 1: redis #1 is master, and all the rest are direct replicas of #1
cycle 2: redis #2 is promoted to master, #1 is a replica of #2 and #3 is replica of #1
now we see that when #1 is demoted it prints:
17339:S 21 Apr 2020 11:16:38.523 * Using the meaningful offset 3929963 instead of 3929977 to exclude the final PINGs (14 bytes difference)
17339:S 21 Apr 2020 11:16:39.391 * Trying a partial resynchronization (request e2b3f8817735fdfe5fa4626766daa938b61419e5:3929964).
17339:S 21 Apr 2020 11:16:39.392 * Successful partial resynchronization with master.
and when #3 connects to the demoted #2, #2 says:
17339:S 21 Apr 2020 11:16:40.084 * Partial resynchronization not accepted: Requested offset for secondary ID was 3929978, but I can reply up to 3929964

so the issue here is that the meaningful offset feature saved the day for the
demoted master (since it needs to sync from a replica that didn't get the last
ping), but it didn't help one of the other replicas which did get the last ping.
---
 src/blocked.c                |   2 +-
 src/networking.c             |  93 +++++++++++-----------
 src/replication.c            |  66 +++++++++-------
 src/server.h                 |   1 -
 tests/integration/psync2.tcl | 144 ++++++++++++++++++++++++++++++-----
 5 files changed, 213 insertions(+), 93 deletions(-)

diff --git a/src/blocked.c b/src/blocked.c
index 00cc798d5..045369e93 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -110,7 +110,7 @@ void processUnblockedClients(void) {
          * the code is conceptually more correct this way. */
         if (!(c->flags & CLIENT_BLOCKED)) {
             if (c->querybuf && sdslen(c->querybuf) > 0) {
-                processInputBufferAndReplicate(c);
+                processInputBuffer(c);
             }
         }
     }
diff --git a/src/networking.c b/src/networking.c
index 744979d16..e4d40fdf0 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1671,12 +1671,55 @@ int processMultibulkBuffer(client *c) {
     return C_ERR;
 }
 
+/* Perform necessary tasks after a command was executed:
+ *
+ * 1. The client is reset unless there are reasons to avoid doing it.
+ * 2. In the case of master clients, the replication offset is updated.
+ * 3. Propagate commands we got from our master to replicas down the line. */
+void commandProcessed(client *c) {
+    int cmd_is_ping = c->cmd && c->cmd->proc == pingCommand;
+    long long prev_offset = c->reploff;
+    if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) {
+        /* Update the applied replication offset of our master. */
+        c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos;
+    }
+
+    /* Don't reset the client structure for clients blocked in a
+     * module blocking command, so that the reply callback will
+     * still be able to access the client argv and argc field.
+     * The client will be reset in unblockClientFromModule(). */
+    if (!(c->flags & CLIENT_BLOCKED) ||
+        c->btype != BLOCKED_MODULE)
+    {
+        resetClient(c);
+    }
+
+    /* If the client is a master we need to compute the difference
+     * between the applied offset before and after processing the buffer,
+     * to understand how much of the replication stream was actually
+     * applied to the master state: this quantity, and its corresponding
+     * part of the replication stream, will be propagated to the
+     * sub-replicas and to the replication backlog. */
+    if (c->flags & CLIENT_MASTER) {
+        long long applied = c->reploff - prev_offset;
+        long long prev_master_repl_meaningful_offset = server.master_repl_meaningful_offset;
+        if (applied) {
+            replicationFeedSlavesFromMasterStream(server.slaves,
+                    c->pending_querybuf, applied);
+            sdsrange(c->pending_querybuf,applied,-1);
+        }
+        /* The server.master_repl_meaningful_offset variable represents
+         * the offset of the replication stream without the pending PINGs. */
+        if (cmd_is_ping)
+            server.master_repl_meaningful_offset = prev_master_repl_meaningful_offset;
+    }
+}
+
 /* This function calls processCommand(), but also performs a few sub tasks
- * that are useful in that context:
+ * for the client that are useful in that context:
  *
  * 1. It sets the current client to the client 'c'.
- * 2. In the case of master clients, the replication offset is updated.
- * 3. The client is reset unless there are reasons to avoid doing it.
+ * 2. calls commandProcessed() if the command was handled.
  *
  * The function returns C_ERR in case the client was freed as a side effect
  * of processing the command, otherwise C_OK is returned. */
@@ -1684,20 +1727,7 @@ int processCommandAndResetClient(client *c) {
     int deadclient = 0;
     server.current_client = c;
     if (processCommand(c) == C_OK) {
-        if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) {
-            /* Update the applied replication offset of our master. */
-            c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos;
-        }
-
-        /* Don't reset the client structure for clients blocked in a
-         * module blocking command, so that the reply callback will
-         * still be able to access the client argv and argc field.
-         * The client will be reset in unblockClientFromModule(). */
-        if (!(c->flags & CLIENT_BLOCKED) ||
-            c->btype != BLOCKED_MODULE)
-        {
-            resetClient(c);
-        }
+        commandProcessed(c);
     }
     if (server.current_client == NULL) deadclient = 1;
     server.current_client = NULL;
@@ -1794,31 +1824,6 @@ void processInputBuffer(client *c) {
     }
 }
 
-/* This is a wrapper for processInputBuffer that also cares about handling
- * the replication forwarding to the sub-replicas, in case the client 'c'
- * is flagged as master. Usually you want to call this instead of the
- * raw processInputBuffer(). */
-void processInputBufferAndReplicate(client *c) {
-    if (!(c->flags & CLIENT_MASTER)) {
-        processInputBuffer(c);
-    } else {
-        /* If the client is a master we need to compute the difference
-         * between the applied offset before and after processing the buffer,
-         * to understand how much of the replication stream was actually
-         * applied to the master state: this quantity, and its corresponding
-         * part of the replication stream, will be propagated to the
-         * sub-replicas and to the replication backlog. */
-        size_t prev_offset = c->reploff;
-        processInputBuffer(c);
-        size_t applied = c->reploff - prev_offset;
-        if (applied) {
-            replicationFeedSlavesFromMasterStream(server.slaves,
-                    c->pending_querybuf, applied);
-            sdsrange(c->pending_querybuf,applied,-1);
-        }
-    }
-}
-
 void readQueryFromClient(connection *conn) {
     client *c = connGetPrivateData(conn);
     int nread, readlen;
@@ -1886,7 +1891,7 @@ void readQueryFromClient(connection *conn) {
 
     /* There is more data in the client input buffer, continue parsing it
      * in case to check if there is a full command to execute. */
-     processInputBufferAndReplicate(c);
+     processInputBuffer(c);
 }
 
 void getClientsMaxBuffers(unsigned long *longest_output_list,
@@ -3101,7 +3106,7 @@ int handleClientsWithPendingReadsUsingThreads(void) {
                 continue;
             }
         }
-        processInputBufferAndReplicate(c);
+        processInputBuffer(c);
     }
     return processed;
 }
diff --git a/src/replication.c b/src/replication.c
index 3e9910374..c59639cd1 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -39,6 +39,7 @@
 #include <sys/socket.h>
 #include <sys/stat.h>
 
+long long adjustMeaningfulReplOffset();
 void replicationDiscardCachedMaster(void);
 void replicationResurrectCachedMaster(connection *conn);
 void replicationSendAck(void);
@@ -2693,6 +2694,9 @@ void replicationCacheMaster(client *c) {
      * pending outputs to the master. */
     sdsclear(server.master->querybuf);
     sdsclear(server.master->pending_querybuf);
+    /* Adjust reploff and read_reploff to the last meaningful offset we executed.
+     * this is the offset the replica will use for future PSYNC. */
+    server.master->reploff = adjustMeaningfulReplOffset();
     server.master->read_reploff = server.master->reploff;
     if (c->flags & CLIENT_MULTI) discardTransaction(c);
     listEmpty(c->reply);
@@ -2717,6 +2721,38 @@ void replicationCacheMaster(client *c) {
     replicationHandleMasterDisconnection();
 }
 
+/* If the "meaningful" offset, that is the offset without the final PINGs
+ * in the stream, is different than the last offset, use it instead:
+ * often when the master is no longer reachable, replicas will never
+ * receive the PINGs, however the master will end with an incremented
+ * offset because of the PINGs and will not be able to incrementally
+ * PSYNC with the new master.
+ * This function trims the replication backlog when needed, and returns
+ * the offset to be used for future partial sync. */
+long long adjustMeaningfulReplOffset() {
+    if (server.master_repl_offset > server.master_repl_meaningful_offset) {
+        long long delta = server.master_repl_offset -
+                          server.master_repl_meaningful_offset;
+        serverLog(LL_NOTICE,
+            "Using the meaningful offset %lld instead of %lld to exclude "
+            "the final PINGs (%lld bytes difference)",
+                server.master_repl_meaningful_offset,
+                server.master_repl_offset,
+                delta);
+        server.master_repl_offset = server.master_repl_meaningful_offset;
+        if (server.repl_backlog_histlen <= delta) {
+            server.repl_backlog_histlen = 0;
+            server.repl_backlog_idx = 0;
+        } else {
+            server.repl_backlog_histlen -= delta;
+            server.repl_backlog_idx =
+                (server.repl_backlog_idx + (server.repl_backlog_size - delta)) %
+                server.repl_backlog_size;
+        }
+    }
+    return server.master_repl_offset;
+}
+
 /* This function is called when a master is turend into a slave, in order to
  * create from scratch a cached master for the new client, that will allow
  * to PSYNC with the slave that was promoted as the new master after a
@@ -2736,35 +2772,7 @@ void replicationCacheMasterUsingMyself(void) {
      * by replicationCreateMasterClient(). We'll later set the created
      * master as server.cached_master, so the replica will use such
      * offset for PSYNC. */
-    server.master_initial_offset = server.master_repl_offset;
-
-    /* However if the "meaningful" offset, that is the offset without
-     * the final PINGs in the stream, is different, use this instead:
-     * often when the master is no longer reachable, replicas will never
-     * receive the PINGs, however the master will end with an incremented
-     * offset because of the PINGs and will not be able to incrementally
-     * PSYNC with the new master. */
-    if (server.master_repl_offset > server.master_repl_meaningful_offset) {
-        long long delta = server.master_repl_offset -
-                          server.master_repl_meaningful_offset;
-        serverLog(LL_NOTICE,
-            "Using the meaningful offset %lld instead of %lld to exclude "
-            "the final PINGs (%lld bytes difference)",
-                server.master_repl_meaningful_offset,
-                server.master_repl_offset,
-                delta);
-        server.master_initial_offset = server.master_repl_meaningful_offset;
-        server.master_repl_offset = server.master_repl_meaningful_offset;
-        if (server.repl_backlog_histlen <= delta) {
-            server.repl_backlog_histlen = 0;
-            server.repl_backlog_idx = 0;
-        } else {
-            server.repl_backlog_histlen -= delta;
-            server.repl_backlog_idx =
-                (server.repl_backlog_idx + (server.repl_backlog_size - delta)) %
-                server.repl_backlog_size;
-        }
-    }
+    server.master_initial_offset = adjustMeaningfulReplOffset();
 
     /* The master client we create can be set to any DBID, because
      * the new master will start its replication stream with SELECT. */
diff --git a/src/server.h b/src/server.h
index 9e1e506af..41d767e13 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1600,7 +1600,6 @@ void setDeferredSetLen(client *c, void *node, long length);
 void setDeferredAttributeLen(client *c, void *node, long length);
 void setDeferredPushLen(client *c, void *node, long length);
 void processInputBuffer(client *c);
-void processInputBufferAndReplicate(client *c);
 void processGopherRequest(client *c);
 void acceptHandler(aeEventLoop *el, int fd, void *privdata, int mask);
 void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask);
diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl
index 333736ffa..4e1189e0b 100644
--- a/tests/integration/psync2.tcl
+++ b/tests/integration/psync2.tcl
@@ -44,6 +44,7 @@ start_server {} {
         set used [list $master_id]
         test "PSYNC2: \[NEW LAYOUT\] Set #$master_id as master" {
             $R($master_id) slaveof no one
+            $R($master_id) config set repl-ping-replica-period 1 ;# increse the chance that random ping will cause issues
             if {$counter_value == 0} {
                 $R($master_id) set x $counter_value
             }
@@ -114,23 +115,20 @@ start_server {} {
             }
         }
 
-        # wait for all the slaves to be in sync with the master
-        set master_ofs [status $R($master_id) master_repl_offset]
+        # wait for all the slaves to be in sync with the master, due to pings, we have to re-sample the master constantly too
         wait_for_condition 500 100 {
-            $master_ofs == [status $R(0) master_repl_offset] &&
-            $master_ofs == [status $R(1) master_repl_offset] &&
-            $master_ofs == [status $R(2) master_repl_offset] &&
-            $master_ofs == [status $R(3) master_repl_offset] &&
-            $master_ofs == [status $R(4) master_repl_offset]
+            [status $R($master_id) master_repl_offset] == [status $R(0) master_repl_offset] &&
+            [status $R($master_id) master_repl_offset] == [status $R(1) master_repl_offset] &&
+            [status $R($master_id) master_repl_offset] == [status $R(2) master_repl_offset] &&
+            [status $R($master_id) master_repl_offset] == [status $R(3) master_repl_offset] &&
+            [status $R($master_id) master_repl_offset] == [status $R(4) master_repl_offset]
         } else {
-            if {$debug_msg} {
-                for {set j 0} {$j < 5} {incr j} {
-                    puts "$j: sync_full: [status $R($j) sync_full]"
-                    puts "$j: id1      : [status $R($j) master_replid]:[status $R($j) master_repl_offset]"
-                    puts "$j: id2      : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]"
-                    puts "$j: backlog  : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]"
-                    puts "---"
-                }
+            for {set j 0} {$j < 5} {incr j} {
+                puts "$j: sync_full: [status $R($j) sync_full]"
+                puts "$j: id1      : [status $R($j) master_replid]:[status $R($j) master_repl_offset]"
+                puts "$j: id2      : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]"
+                puts "$j: backlog  : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]"
+                puts "---"
             }
             fail "Slaves are not in sync with the master after too long time."
         }
@@ -175,9 +173,14 @@ start_server {} {
             $R($j) slaveof $master_host $master_port
         }
 
-        # Wait for slaves to sync
+        # Wait for replicas to sync. it is not enough to just wait for connected_slaves==4
+        # since we might do the check before the master realized that they're disconnected
         wait_for_condition 50 1000 {
-            [status $R($master_id) connected_slaves] == 4
+            [status $R($master_id) connected_slaves] == 4 &&
+            [status $R([expr {($master_id+1)%5}]) master_link_status] == "up" &&
+            [status $R([expr {($master_id+2)%5}]) master_link_status] == "up" &&
+            [status $R([expr {($master_id+3)%5}]) master_link_status] == "up" &&
+            [status $R([expr {($master_id+4)%5}]) master_link_status] == "up"
         } else {
             fail "Replica not reconnecting"
         }
@@ -188,6 +191,7 @@ start_server {} {
         set slave_id [expr {($master_id+1)%5}]
         set sync_count [status $R($master_id) sync_full]
         set sync_partial [status $R($master_id) sync_partial_ok]
+        set sync_partial_err [status $R($master_id) sync_partial_err]
         catch {
             $R($slave_id) config rewrite
             $R($slave_id) debug restart
@@ -197,7 +201,11 @@ start_server {} {
         wait_for_condition 50 1000 {
             [status $R($master_id) sync_partial_ok] == $sync_partial + 1
         } else {
-            fail "Replica not reconnecting"
+            puts "prev sync_full: $sync_count"
+            puts "prev sync_partial_ok: $sync_partial"
+            puts "prev sync_partial_err: $sync_partial_err"
+            puts [$R($master_id) info stats]
+            fail "Replica didn't partial sync"
         }
         set new_sync_count [status $R($master_id) sync_full]
         assert {$sync_count == $new_sync_count}
@@ -271,3 +279,103 @@ start_server {} {
     }
 
 }}}}}
+
+start_server {tags {"psync2"}} {
+start_server {} {
+start_server {} {
+start_server {} {
+start_server {} {
+    test {pings at the end of replication stream are ignored for psync} {
+        set master [srv -4 client]
+        set master_host [srv -4 host]
+        set master_port [srv -4 port]
+        set replica1 [srv -3 client]
+        set replica2 [srv -2 client]
+        set replica3 [srv -1 client]
+        set replica4 [srv -0 client]
+
+        $replica1 replicaof $master_host $master_port
+        $replica2 replicaof $master_host $master_port
+        $replica3 replicaof $master_host $master_port
+        $replica4 replicaof $master_host $master_port
+        wait_for_condition 50 1000 {
+            [status $master connected_slaves] == 4
+        } else {
+            fail "replicas didn't connect"
+        }
+
+        $master incr x
+        wait_for_condition 50 1000 {
+            [$replica1 get x] == 1 && [$replica2 get x] == 1 &&
+            [$replica3 get x] == 1 && [$replica4 get x] == 1
+        } else {
+            fail "replicas didn't get incr"
+        }
+
+        # disconnect replica1 and replica2
+        # and wait for the master to send a ping to replica3 and replica4
+        $replica1 replicaof no one
+        $replica2 replicaof 127.0.0.1 1 ;# we can't promote it to master since that will cycle the replication id
+        $master config set repl-ping-replica-period 1
+        after 1500
+
+        # make everyone sync from the replica1 that didn't get the last ping from the old master
+        # replica4 will keep syncing from the old master which now syncs from replica1
+        # and replica2 will re-connect to the old master (which went back in time)
+        set new_master_host [srv -3 host]
+        set new_master_port [srv -3 port]
+        $replica3 replicaof $new_master_host $new_master_port
+        $master replicaof $new_master_host $new_master_port
+        $replica2 replicaof $master_host $master_port
+        wait_for_condition 50 1000 {
+            [status $replica2 master_link_status] == "up" &&
+            [status $replica3 master_link_status] == "up" &&
+            [status $replica4 master_link_status] == "up" &&
+            [status $master master_link_status] == "up"
+        } else {
+            fail "replicas didn't connect"
+        }
+
+        # make sure replication is still alive and kicking
+        $replica1 incr x
+        wait_for_condition 50 1000 {
+            [$replica2 get x] == 2 &&
+            [$replica3 get x] == 2 &&
+            [$replica4 get x] == 2 &&
+            [$master get x] == 2
+        } else {
+            fail "replicas didn't get incr"
+        }
+
+        # make sure there are full syncs other than the initial ones
+        assert_equal [status $master sync_full] 4
+        assert_equal [status $replica1 sync_full] 0
+        assert_equal [status $replica2 sync_full] 0
+        assert_equal [status $replica3 sync_full] 0
+        assert_equal [status $replica4 sync_full] 0
+
+        # force psync
+        $master client kill type master
+        $replica2 client kill type master
+        $replica3 client kill type master
+        $replica4 client kill type master
+
+        # make sure replication is still alive and kicking
+        $replica1 incr x
+        wait_for_condition 50 1000 {
+            [$replica2 get x] == 3 &&
+            [$replica3 get x] == 3 &&
+            [$replica4 get x] == 3 &&
+            [$master get x] == 3
+        } else {
+            fail "replicas didn't get incr"
+        }
+
+        # make sure there are full syncs other than the initial ones
+        assert_equal [status $master sync_full] 4
+        assert_equal [status $replica1 sync_full] 0
+        assert_equal [status $replica2 sync_full] 0
+        assert_equal [status $replica3 sync_full] 0
+        assert_equal [status $replica4 sync_full] 0
+}
+}}}}}

From 09a5c07886c8f4f4f2f44614610b0ff398109290 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 27 Apr 2020 23:17:19 +0300
Subject: [PATCH 0314/1098] allow dictFind using static robj

since the recent addition of OBJ_STATIC_REFCOUNT and the assertion in
incrRefCount it is now impossible to use dictFind using a static robj,
because dictEncObjKeyCompare will call getDecodedObject which tries to
increment the refcount just in order to decrement it later.
---
 src/server.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/server.c b/src/server.c
index e265fe9ab..1bfca1a0b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1219,11 +1219,16 @@ int dictEncObjKeyCompare(void *privdata, const void *key1,
         o2->encoding == OBJ_ENCODING_INT)
             return o1->ptr == o2->ptr;
 
-    o1 = getDecodedObject(o1);
-    o2 = getDecodedObject(o2);
+    /* due to OBJ_STATIC_REFCOUNT, we rather not call sdsEncodedObject unnecessarily */
+    if (!sdsEncodedObject(o1))
+        o1 = getDecodedObject(o1);
+    if (!sdsEncodedObject(o2))
+        o2 = getDecodedObject(o2);
     cmp = dictSdsKeyCompare(privdata,o1->ptr,o2->ptr);
-    decrRefCount(o1);
-    decrRefCount(o2);
+    if (o1!=key1)
+        decrRefCount(o1);
+    if (o2!=key2)
+        decrRefCount(o2);
     return cmp;
 }
 

From 31781e97b6ae052a12bbe7218ec2a6c030d708c8 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 27 Apr 2020 22:40:15 +0200
Subject: [PATCH 0315/1098] Rework comment in dictEncObjKeyCompare().

---
 src/server.c | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/server.c b/src/server.c
index 1bfca1a0b..1086dab48 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1219,16 +1219,15 @@ int dictEncObjKeyCompare(void *privdata, const void *key1,
         o2->encoding == OBJ_ENCODING_INT)
             return o1->ptr == o2->ptr;
 
-    /* due to OBJ_STATIC_REFCOUNT, we rather not call sdsEncodedObject unnecessarily */
-    if (!sdsEncodedObject(o1))
-        o1 = getDecodedObject(o1);
-    if (!sdsEncodedObject(o2))
-        o2 = getDecodedObject(o2);
+    /* Due to OBJ_STATIC_REFCOUNT, we avoid calling getDecodedObject() without
+     * good reasons, because it would incrRefCount() the object, which
+     * is invalid. So we check to make sure dictFind() works with static
+     * objects as well. */
+    if (!sdsEncodedObject(o1)) o1 = getDecodedObject(o1);
+    if (!sdsEncodedObject(o2)) o2 = getDecodedObject(o2);
     cmp = dictSdsKeyCompare(privdata,o1->ptr,o2->ptr);
-    if (o1!=key1)
-        decrRefCount(o1);
-    if (o2!=key2)
-        decrRefCount(o2);
+    if (o1!=key1) decrRefCount(o1);
+    if (o2!=key2) decrRefCount(o2);
     return cmp;
 }
 

From d31c0c52643d627a0dc06664eac0e890ea7aa24e Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 28 Apr 2020 09:18:01 +0300
Subject: [PATCH 0316/1098] fix loading race in psync2 tests

---
 tests/integration/psync2-pingoff.tcl |  1 +
 tests/integration/psync2-reg.tcl     |  5 ++++-
 tests/integration/psync2.tcl         | 10 ++++++++++
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/tests/integration/psync2-pingoff.tcl b/tests/integration/psync2-pingoff.tcl
index 1cea290e7..420747d21 100644
--- a/tests/integration/psync2-pingoff.tcl
+++ b/tests/integration/psync2-pingoff.tcl
@@ -20,6 +20,7 @@ start_server {} {
         $R(1) replicaof $R_host(0) $R_port(0)
         $R(0) set foo bar
         wait_for_condition 50 1000 {
+            [status $R(1) master_link_status] == "up" &&
             [$R(0) dbsize] == 1 && [$R(1) dbsize] == 1
         } else {
             fail "Replicas not replicating from master"
diff --git a/tests/integration/psync2-reg.tcl b/tests/integration/psync2-reg.tcl
index b5ad021e2..71a1c0eb2 100644
--- a/tests/integration/psync2-reg.tcl
+++ b/tests/integration/psync2-reg.tcl
@@ -28,7 +28,10 @@ start_server {} {
         $R(2) slaveof $R_host(0) $R_port(0)
         $R(0) set foo bar
         wait_for_condition 50 1000 {
-            [$R(1) dbsize] == 1 && [$R(2) dbsize] == 1
+            [status $R(1) master_link_status] == "up" &&
+            [status $R(2) master_link_status] == "up" &&
+            [$R(1) dbsize] == 1 &&
+            [$R(2) dbsize] == 1
         } else {
             fail "Replicas not replicating from master"
         }
diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl
index 4e1189e0b..5fe29caba 100644
--- a/tests/integration/psync2.tcl
+++ b/tests/integration/psync2.tcl
@@ -67,6 +67,16 @@ start_server {} {
             lappend used $slave_id
         }
 
+        # Wait for replicas to sync. so next loop won't get -LOADING error
+        wait_for_condition 50 1000 {
+            [status $R([expr {($master_id+1)%5}]) master_link_status] == "up" &&
+            [status $R([expr {($master_id+2)%5}]) master_link_status] == "up" &&
+            [status $R([expr {($master_id+3)%5}]) master_link_status] == "up" &&
+            [status $R([expr {($master_id+4)%5}]) master_link_status] == "up"
+        } else {
+            fail "Replica not reconnecting"
+        }
+
         # 3) Increment the counter and wait for all the instances
         # to converge.
         test "PSYNC2: cluster is consistent after failover" {

From 9a3dab0a2eeba881b56de62faacaf51875641035 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 28 Apr 2020 12:14:46 +0300
Subject: [PATCH 0317/1098] hickup, re-fix dictEncObjKeyCompare

come to think of it, in theory (not in practice), getDecodedObject can
return the same original object with refcount incremented, so the
pointer comparision in the previous commit was invalid.
so now instead of checking the encoding, we explicitly check the
refcount.
---
 src/server.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/server.c b/src/server.c
index 1086dab48..ab6280121 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1223,11 +1223,11 @@ int dictEncObjKeyCompare(void *privdata, const void *key1,
      * good reasons, because it would incrRefCount() the object, which
      * is invalid. So we check to make sure dictFind() works with static
      * objects as well. */
-    if (!sdsEncodedObject(o1)) o1 = getDecodedObject(o1);
-    if (!sdsEncodedObject(o2)) o2 = getDecodedObject(o2);
+    if (o1->refcount != OBJ_STATIC_REFCOUNT) o1 = getDecodedObject(o1);
+    if (o2->refcount != OBJ_STATIC_REFCOUNT) o2 = getDecodedObject(o2);
     cmp = dictSdsKeyCompare(privdata,o1->ptr,o2->ptr);
-    if (o1!=key1) decrRefCount(o1);
-    if (o2!=key2) decrRefCount(o2);
+    if (o1->refcount != OBJ_STATIC_REFCOUNT) decrRefCount(o1);
+    if (o2->refcount != OBJ_STATIC_REFCOUNT) decrRefCount(o2);
     return cmp;
 }
 

From 1e2aee39192c8839170743d324c799860dc38c8c Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Wed, 22 Apr 2020 16:05:57 +0300
Subject: [PATCH 0318/1098] Extend XINFO STREAM output

Introducing XINFO STREAM <key> FULL
---
 src/t_stream.c                     | 226 ++++++++++++++++++++++++-----
 tests/unit/type/stream-cgroups.tcl |  34 +++++
 2 files changed, 226 insertions(+), 34 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 155167af9..508c22225 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -2475,16 +2475,200 @@ void xtrimCommand(client *c) {
     addReplyLongLong(c,deleted);
 }
 
+/* Helper function for xinfoCommand.
+ * Handles the variants of XINFO STREAM */
+void xinfoReplyWithStreamInfo(client *c, stream *s) {
+    int full = 1;
+    long long count = 0;
+    robj **optv = c->argv + 3; /* Options start after XINFO STREAM <key> */
+    int optc = c->argc - 3;
+
+    /* Parse options. */
+    if (optc == 0) {
+        full = 0;
+    } else {
+        /* Valid options are [FULL] or [FULL COUNT <count>] */
+        if (optc != 1 && optc != 3) {
+            addReplySubcommandSyntaxError(c);
+            return;
+        }
+
+        /* First option must be "FULL" */
+        if (strcasecmp(optv[0]->ptr,"full")) {
+            addReplySubcommandSyntaxError(c);
+            return;
+        }
+
+        if (optc == 3) {
+            /* First option must be "FULL" */
+            if (strcasecmp(optv[1]->ptr,"count")) {
+                addReplySubcommandSyntaxError(c);
+                return;
+            }
+            if (getLongLongFromObjectOrReply(c,optv[2],&count,NULL) == C_ERR)
+                return;
+            if (count < 0) count = 0;
+        }
+    }
+
+    addReplyMapLen(c,full ? 6 : 7);
+    addReplyBulkCString(c,"length");
+    addReplyLongLong(c,s->length);
+    addReplyBulkCString(c,"radix-tree-keys");
+    addReplyLongLong(c,raxSize(s->rax));
+    addReplyBulkCString(c,"radix-tree-nodes");
+    addReplyLongLong(c,s->rax->numnodes);
+    addReplyBulkCString(c,"last-generated-id");
+    addReplyStreamID(c,&s->last_id);
+
+    if (!full) {
+        /* XINFO STREAM <key> */
+
+        addReplyBulkCString(c,"groups");
+        addReplyLongLong(c,s->cgroups ? raxSize(s->cgroups) : 0);
+
+        /* To emit the first/last entry we use streamReplyWithRange(). */
+        int emitted;
+        streamID start, end;
+        start.ms = start.seq = 0;
+        end.ms = end.seq = UINT64_MAX;
+        addReplyBulkCString(c,"first-entry");
+        emitted = streamReplyWithRange(c,s,&start,&end,1,0,NULL,NULL,
+                                     STREAM_RWR_RAWENTRIES,NULL);
+        if (!emitted) addReplyNull(c);
+        addReplyBulkCString(c,"last-entry");
+        emitted = streamReplyWithRange(c,s,&start,&end,1,1,NULL,NULL,
+                                     STREAM_RWR_RAWENTRIES,NULL);
+        if (!emitted) addReplyNull(c);
+    } else {
+        /* XINFO STREAM <key> FULL [COUNT <count>] */
+
+        /* Stream entries */
+        addReplyBulkCString(c,"entries");
+        streamReplyWithRange(c,s,NULL,NULL,count,0,NULL,NULL,0,NULL);
+
+        /* Consumer groups */
+        addReplyBulkCString(c,"groups");
+        if (s->cgroups == NULL) {
+            addReplyArrayLen(c,0);
+        } else {
+            addReplyArrayLen(c,raxSize(s->cgroups));
+            raxIterator ri_cgroups;
+            raxStart(&ri_cgroups,s->cgroups);
+            raxSeek(&ri_cgroups,"^",NULL,0);
+            while(raxNext(&ri_cgroups)) {
+                streamCG *cg = ri_cgroups.data;
+                addReplyMapLen(c,5);
+
+                /* Name */
+                addReplyBulkCString(c,"name");
+                addReplyBulkCBuffer(c,ri_cgroups.key,ri_cgroups.key_len);
+
+                /* Last delivered ID */
+                addReplyBulkCString(c,"last-delivered-id");
+                addReplyStreamID(c,&cg->last_id);
+
+                /* Group PEL count */
+                addReplyBulkCString(c,"pel-count");
+                addReplyLongLong(c,raxSize(cg->pel));
+
+                /* Group PEL */
+                addReplyBulkCString(c,"pending");
+                long long arraylen_cg_pel = 0;
+                void *arrayptr_cg_pel = addReplyDeferredLen(c);
+                raxIterator ri_cg_pel;
+                raxStart(&ri_cg_pel,cg->pel);
+                raxSeek(&ri_cg_pel,"^",NULL,0);
+                while(raxNext(&ri_cg_pel) && (!count || arraylen_cg_pel < count)) {
+                    streamNACK *nack = ri_cg_pel.data;
+                    addReplyArrayLen(c,4);
+
+                    /* Entry ID. */
+                    streamID id;
+                    streamDecodeID(ri_cg_pel.key,&id);
+                    addReplyStreamID(c,&id);
+
+                    /* Consumer name. */
+                    addReplyBulkCBuffer(c,nack->consumer->name,
+                                        sdslen(nack->consumer->name));
+
+                    /* Last delivery. */
+                    addReplyLongLong(c,nack->delivery_time);
+
+                    /* Number of deliveries. */
+                    addReplyLongLong(c,nack->delivery_count);
+
+                    arraylen_cg_pel++;
+                }
+                setDeferredArrayLen(c,arrayptr_cg_pel,arraylen_cg_pel);
+                raxStop(&ri_cg_pel);
+
+                /* Consumers */
+                addReplyBulkCString(c,"consumers");
+                addReplyArrayLen(c,raxSize(cg->consumers));
+                raxIterator ri_consumers;
+                raxStart(&ri_consumers,cg->consumers);
+                raxSeek(&ri_consumers,"^",NULL,0);
+                while(raxNext(&ri_consumers)) {
+                    streamConsumer *consumer = ri_consumers.data;
+                    addReplyMapLen(c,4);
+
+                    /* Consumer name */
+                    addReplyBulkCString(c,"name");
+                    addReplyBulkCBuffer(c,consumer->name,sdslen(consumer->name));
+
+                    /* Seen-time */
+                    addReplyBulkCString(c,"seen-time");
+                    addReplyLongLong(c,consumer->seen_time);
+
+                    /* Consumer PEL count */
+                    addReplyBulkCString(c,"pel-count");
+                    addReplyLongLong(c,raxSize(consumer->pel));
+
+                    /* Consumer PEL */
+                    addReplyBulkCString(c,"pending");
+                    long long arraylen_cpel = 0;
+                    void *arrayptr_cpel = addReplyDeferredLen(c);
+                    raxIterator ri_cpel;
+                    raxStart(&ri_cpel,consumer->pel);
+                    raxSeek(&ri_cpel,"^",NULL,0);
+                    while(raxNext(&ri_cpel) && (!count || arraylen_cpel < count)) {
+                        streamNACK *nack = ri_cpel.data;
+                        addReplyArrayLen(c,3);
+
+                        /* Entry ID. */
+                        streamID id;
+                        streamDecodeID(ri_cpel.key,&id);
+                        addReplyStreamID(c,&id);
+
+                        /* Last delivery. */
+                        addReplyLongLong(c,nack->delivery_time);
+
+                        /* Number of deliveries. */
+                        addReplyLongLong(c,nack->delivery_count);
+
+                        arraylen_cpel++;
+                    }
+                    setDeferredArrayLen(c,arrayptr_cpel,arraylen_cpel);
+                    raxStop(&ri_cpel);
+                }
+                raxStop(&ri_consumers);
+            }
+            raxStop(&ri_cgroups);
+        }
+    }
+}
+
 /* XINFO CONSUMERS <key> <group>
  * XINFO GROUPS <key>
- * XINFO STREAM <key>
+ * XINFO STREAM <key> [FULL [COUNT <count>]]
  * XINFO HELP. */
 void xinfoCommand(client *c) {
     const char *help[] = {
-"CONSUMERS <key> <groupname>  -- Show consumer groups of group <groupname>.",
-"GROUPS <key>                 -- Show the stream consumer groups.",
-"STREAM <key>                 -- Show information about the stream.",
-"HELP                         -- Print this help.",
+"CONSUMERS <key> <groupname>         -- Show consumer groups of group <groupname>.",
+"GROUPS <key>                        -- Show the stream consumer groups.",
+"STREAM <key> [FULL [COUNT <count>]] -- Show information about the stream.",
+"HELP                                -- Print this help.",
 NULL
     };
     stream *s = NULL;
@@ -2564,36 +2748,10 @@ NULL
             addReplyStreamID(c,&cg->last_id);
         }
         raxStop(&ri);
-    } else if (!strcasecmp(opt,"STREAM") && c->argc == 3) {
-        /* XINFO STREAM <key> (or the alias XINFO <key>). */
-        addReplyMapLen(c,7);
-        addReplyBulkCString(c,"length");
-        addReplyLongLong(c,s->length);
-        addReplyBulkCString(c,"radix-tree-keys");
-        addReplyLongLong(c,raxSize(s->rax));
-        addReplyBulkCString(c,"radix-tree-nodes");
-        addReplyLongLong(c,s->rax->numnodes);
-        addReplyBulkCString(c,"groups");
-        addReplyLongLong(c,s->cgroups ? raxSize(s->cgroups) : 0);
-        addReplyBulkCString(c,"last-generated-id");
-        addReplyStreamID(c,&s->last_id);
-
-        /* To emit the first/last entry we us the streamReplyWithRange()
-         * API. */
-        int count;
-        streamID start, end;
-        start.ms = start.seq = 0;
-        end.ms = end.seq = UINT64_MAX;
-        addReplyBulkCString(c,"first-entry");
-        count = streamReplyWithRange(c,s,&start,&end,1,0,NULL,NULL,
-                                     STREAM_RWR_RAWENTRIES,NULL);
-        if (!count) addReplyNull(c);
-        addReplyBulkCString(c,"last-entry");
-        count = streamReplyWithRange(c,s,&start,&end,1,1,NULL,NULL,
-                                     STREAM_RWR_RAWENTRIES,NULL);
-        if (!count) addReplyNull(c);
+    } else if (!strcasecmp(opt,"STREAM")) {
+        /* XINFO STREAM <key> [FULL [COUNT <count>]]. */
+        xinfoReplyWithStreamInfo(c,s);
     } else {
         addReplySubcommandSyntaxError(c);
     }
 }
-
diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl
index 04661707b..dfcd735f6 100644
--- a/tests/unit/type/stream-cgroups.tcl
+++ b/tests/unit/type/stream-cgroups.tcl
@@ -294,6 +294,40 @@ start_server {
         assert {[lindex $reply 0 3] == 2}
     }
 
+    test {XINFO FULL output} {
+        r del x
+        r XADD x 100 a 1
+        r XADD x 101 b 1
+        r XADD x 102 c 1
+        r XADD x 103 e 1
+        r XADD x 104 f 1
+        r XGROUP CREATE x g1 0
+        r XGROUP CREATE x g2 0
+        r XREADGROUP GROUP g1 Alice COUNT 1 STREAMS x >
+        r XREADGROUP GROUP g1 Bob COUNT 1 STREAMS x >
+        r XREADGROUP GROUP g1 Bob NOACK COUNT 1 STREAMS x >
+        r XREADGROUP GROUP g2 Charlie COUNT 4 STREAMS x >
+        r XDEL x 103
+
+        set reply [r XINFO STREAM x FULL]
+        assert_equal [llength $reply] 12
+        assert_equal [lindex $reply 1] 4 ;# stream length
+        assert_equal [lindex $reply 9] "{100-0 {a 1}} {101-0 {b 1}} {102-0 {c 1}} {104-0 {f 1}}" ;# entries
+        assert_equal [lindex $reply 11 0 1] "g1" ;# first group name
+        assert_equal [lindex $reply 11 0 7 0 0] "100-0" ;# first entry in group's PEL
+        assert_equal [lindex $reply 11 0 9 0 1] "Alice" ;# first consumer
+        assert_equal [lindex $reply 11 0 9 0 7 0 0] "100-0" ;# first entry in first consumer's PEL
+        assert_equal [lindex $reply 11 1 1] "g2" ;# second group name
+        assert_equal [lindex $reply 11 1 9 0 1] "Charlie" ;# first consumer
+        assert_equal [lindex $reply 11 1 9 0 7 0 0] "100-0" ;# first entry in first consumer's PEL
+        assert_equal [lindex $reply 11 1 9 0 7 1 0] "101-0" ;# second entry in first consumer's PEL
+
+        set reply [r XINFO STREAM x FULL COUNT 1]
+        assert_equal [llength $reply] 12
+        assert_equal [lindex $reply 1] 4
+        assert_equal [lindex $reply 9] "{100-0 {a 1}}"
+    }
+
     start_server {} {
         set master [srv -1 client]
         set master_host [srv -1 host]

From f95a88d988ffae6901fc186e780c64b747ab5a74 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 28 Apr 2020 16:40:15 +0200
Subject: [PATCH 0319/1098] Fix create-cluster BIN_PATH.

---
 utils/create-cluster/create-cluster | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster
index 4db0a6619..931f6f521 100755
--- a/utils/create-cluster/create-cluster
+++ b/utils/create-cluster/create-cluster
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Settings
-BIN_PATH="../../"
+BIN_PATH="../../src/"
 CLUSTER_HOST=127.0.0.1
 PORT=30000
 TIMEOUT=2000

From 057865a6a55b6cf41e16c9894033d9269bd95947 Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Tue, 28 Apr 2020 17:58:25 +0300
Subject: [PATCH 0320/1098] XINFO STREAM FULL should have a default COUNT of 10

---
 src/t_stream.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index d6a5e0009..5c1b9a523 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -2493,7 +2493,7 @@ void xtrimCommand(client *c) {
  * Handles the variants of XINFO STREAM */
 void xinfoReplyWithStreamInfo(client *c, stream *s) {
     int full = 1;
-    long long count = 0;
+    long long count = 10; /* Default COUNT is 10 so we don't block the server */
     robj **optv = c->argv + 3; /* Options start after XINFO STREAM <key> */
     int optc = c->argc - 3;
 
@@ -2521,7 +2521,7 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) {
             }
             if (getLongLongFromObjectOrReply(c,optv[2],&count,NULL) == C_ERR)
                 return;
-            if (count < 0) count = 0;
+            if (count < 0) count = 10;
         }
     }
 
@@ -2548,11 +2548,11 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) {
         end.ms = end.seq = UINT64_MAX;
         addReplyBulkCString(c,"first-entry");
         emitted = streamReplyWithRange(c,s,&start,&end,1,0,NULL,NULL,
-                                     STREAM_RWR_RAWENTRIES,NULL);
+                                       STREAM_RWR_RAWENTRIES,NULL);
         if (!emitted) addReplyNull(c);
         addReplyBulkCString(c,"last-entry");
         emitted = streamReplyWithRange(c,s,&start,&end,1,1,NULL,NULL,
-                                     STREAM_RWR_RAWENTRIES,NULL);
+                                       STREAM_RWR_RAWENTRIES,NULL);
         if (!emitted) addReplyNull(c);
     } else {
         /* XINFO STREAM <key> FULL [COUNT <count>] */
@@ -2682,6 +2682,10 @@ void xinfoCommand(client *c) {
 "CONSUMERS <key> <groupname>         -- Show consumer groups of group <groupname>.",
 "GROUPS <key>                        -- Show the stream consumer groups.",
 "STREAM <key> [FULL [COUNT <count>]] -- Show information about the stream.",
+"                                       FULL will return the full state of the stream,",
+"                                            including all entries, groups, consumers and PELs.",
+"                                            It's possible to show only the first stream/PEL entries",
+"                                            by using the COUNT modifier (Default is 10)",
 "HELP                                -- Print this help.",
 NULL
     };

From 4024bc7eee9b20b2f3bc85fb6f94e243f1a61b9d Mon Sep 17 00:00:00 2001
From: srzhao <srzhao@sysnew.com>
Date: Tue, 26 Nov 2019 10:43:57 +0800
Subject: [PATCH 0321/1098] fix pipelined WAIT performance issue.

If client gets blocked again in `processUnblockedClients`, redis will not send
`REPLCONF GETACK *` to slaves untill next eventloop, so the client will be
blocked for 100ms by default(10hz) if no other file event fired.

move server.get_ack_from_slaves sinppet after `processUnblockedClients`, so
that both the first WAIT command that puts client in blocked context and the
following WAIT command processed in processUnblockedClients would trigger
redis-sever to send `REPLCONF GETACK *`, so that the eventloop would get
`REPLCONG ACK <reploffset>` from slaves and unblocked ASAP.
---
 src/server.c | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/server.c b/src/server.c
index ab6280121..150476276 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2110,6 +2110,19 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     if (server.active_expire_enabled && server.masterhost == NULL)
         activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST);
 
+    /* Unblock all the clients blocked for synchronous replication
+     * in WAIT. */
+    if (listLength(server.clients_waiting_acks))
+        processClientsWaitingReplicas();
+
+    /* Check if there are clients unblocked by modules that implement
+     * blocking commands. */
+    if (moduleCount()) moduleHandleBlockedClients();
+
+    /* Try to process pending commands for clients that were just unblocked. */
+    if (listLength(server.unblocked_clients))
+        processUnblockedClients();
+
     /* Send all the slaves an ACK request if at least one client blocked
      * during the previous event loop iteration. */
     if (server.get_ack_from_slaves) {
@@ -2125,19 +2138,6 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
         server.get_ack_from_slaves = 0;
     }
 
-    /* Unblock all the clients blocked for synchronous replication
-     * in WAIT. */
-    if (listLength(server.clients_waiting_acks))
-        processClientsWaitingReplicas();
-
-    /* Check if there are clients unblocked by modules that implement
-     * blocking commands. */
-    if (moduleCount()) moduleHandleBlockedClients();
-
-    /* Try to process pending commands for clients that were just unblocked. */
-    if (listLength(server.unblocked_clients))
-        processUnblockedClients();
-
     /* Send the invalidation messages to clients participating to the
      * client side caching protocol in broadcasting (BCAST) mode. */
     trackingBroadcastInvalidationMessages();

From d2a6c5ffa297e849ef80a82e5c8fa40abca01207 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 29 Apr 2020 11:16:30 +0200
Subject: [PATCH 0322/1098] Comment clearly why we moved some code in #6623.

---
 src/server.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index 150476276..659604ef3 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2124,7 +2124,10 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
         processUnblockedClients();
 
     /* Send all the slaves an ACK request if at least one client blocked
-     * during the previous event loop iteration. */
+     * during the previous event loop iteration. Note that we do this after
+     * processUnblockedClients(), so if there are multiple pipelined WAITs
+     * and the just unblocked WAIT gets blocked again, we don't have to wait
+     * a server cron cycle in absence of other event loop events. See #6623. */
     if (server.get_ack_from_slaves) {
         robj *argv[3];
 

From 86a1386d6f5bb0655f710736aff4dc7ab7eedf38 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 29 Apr 2020 12:37:12 +0200
Subject: [PATCH 0323/1098] redis-cli: try to make clusterManagerFixOpenSlot()
 more readable.

Also improve the message to make clear that there is no *clear* owner,
not that there is no owner at all.
---
 src/redis-cli.c | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 72480d08c..469dbb0ff 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -4596,20 +4596,26 @@ static int clusterManagerFixOpenSlot(int slot) {
     /* Try to obtain the current slot owner, according to the current
      * nodes configuration. */
     int success = 1;
-    list *owners = listCreate();
+    list *owners = listCreate();    /* List of nodes claiming some ownership.
+                                       it could be stating in the configuration
+                                       to have the node ownership, or just
+                                       holding keys for such slot. */
     list *migrating = listCreate();
     list *importing = listCreate();
     sds migrating_str = sdsempty();
     sds importing_str = sdsempty();
-    clusterManagerNode *owner = NULL;
+    clusterManagerNode *owner = NULL; /* The obvious slot owner if any. */
+
+    /* Iterate all the nodes, looking for potential owners of this slot. */
     listIter li;
     listNode *ln;
     listRewind(cluster_manager.nodes, &li);
     while ((ln = listNext(&li)) != NULL) {
         clusterManagerNode *n = ln->value;
         if (n->flags & CLUSTER_MANAGER_FLAG_SLAVE) continue;
-        if (n->slots[slot]) listAddNodeTail(owners, n);
-        else {
+        if (n->slots[slot]) {
+            listAddNodeTail(owners, n);
+        } else {
             redisReply *r = CLUSTER_MANAGER_COMMAND(n,
                 "CLUSTER COUNTKEYSINSLOT %d", slot);
             success = clusterManagerCheckRedisReply(n, r, NULL);
@@ -4623,7 +4629,14 @@ static int clusterManagerFixOpenSlot(int slot) {
             if (!success) goto cleanup;
         }
     }
+
+    /* If we have only a single potential owner for this slot,
+     * set it as "owner". */
     if (listLength(owners) == 1) owner = listFirst(owners)->value;
+
+    /* Scan the list of nodes again, in order to populate the
+     * list of nodes in importing or migrating state for
+     * this slot. */
     listRewind(cluster_manager.nodes, &li);
     while ((ln = listNext(&li)) != NULL) {
         clusterManagerNode *n = ln->value;
@@ -4655,6 +4668,7 @@ static int clusterManagerFixOpenSlot(int slot) {
                 }
             }
         }
+
         /* If the node is neither migrating nor importing and it's not
          * the owner, then is added to the importing list in case
          * it has keys in the slot. */
@@ -4679,11 +4693,12 @@ static int clusterManagerFixOpenSlot(int slot) {
         printf("Set as migrating in: %s\n", migrating_str);
     if (sdslen(importing_str) > 0)
         printf("Set as importing in: %s\n", importing_str);
+
     /* If there is no slot owner, set as owner the node with the biggest
      * number of keys, among the set of migrating / importing nodes. */
     if (owner == NULL) {
-        clusterManagerLogInfo(">>> Nobody claims ownership, "
-                              "selecting an owner...\n");
+        clusterManagerLogInfo(">>> No single clear owner for the slot, "
+                              "selecting an owner by # of keys...\n");
         owner = clusterManagerGetNodeWithMostKeysInSlot(cluster_manager.nodes,
                                                         slot, NULL);
         // If we still don't have an owner, we can't fix it.
@@ -4714,6 +4729,7 @@ static int clusterManagerFixOpenSlot(int slot) {
         clusterManagerRemoveNodeFromList(migrating, owner);
         clusterManagerRemoveNodeFromList(importing, owner);
     }
+
     /* If there are multiple owners of the slot, we need to fix it
      * so that a single node is the owner and all the other nodes
      * are in importing state. Later the fix can be handled by one
@@ -4746,6 +4762,7 @@ static int clusterManagerFixOpenSlot(int slot) {
         }
     }
     int move_opts = CLUSTER_MANAGER_OPT_VERBOSE;
+
     /* Case 1: The slot is in migrating state in one node, and in
      *         importing state in 1 node. That's trivial to address. */
     if (listLength(migrating) == 1 && listLength(importing) == 1) {
@@ -4757,6 +4774,7 @@ static int clusterManagerFixOpenSlot(int slot) {
         move_opts |= CLUSTER_MANAGER_OPT_UPDATE;
         success = clusterManagerMoveSlot(src, dst, slot, move_opts, NULL);
     }
+
     /* Case 2: There are multiple nodes that claim the slot as importing,
      * they probably got keys about the slot after a restart so opened
      * the slot. In this case we just move all the keys to the owner
@@ -4787,6 +4805,7 @@ static int clusterManagerFixOpenSlot(int slot) {
             if (!success) goto cleanup;
         }
     }
+
     /* Case 3: The slot is in migrating state in one node but multiple
      * other nodes claim to be in importing state and don't have any key in
      * the slot. We search for the importing node having the same ID as

From 7c29c9eec18eb847aff6982c6a370b05c811dc2e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 29 Apr 2020 16:28:16 +0200
Subject: [PATCH 0324/1098] redis-cli: simplify cluster nodes coverage display.

---
 src/redis-cli.c | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 469dbb0ff..82a46216c 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -4295,17 +4295,18 @@ static int clusterManagerGetCoveredSlots(char *all_slots) {
 }
 
 static void clusterManagerPrintSlotsList(list *slots) {
+    clusterManagerNode n = {0};
     listIter li;
     listNode *ln;
     listRewind(slots, &li);
-    sds first = NULL;
     while ((ln = listNext(&li)) != NULL) {
-        sds slot = ln->value;
-        if (!first) first = slot;
-        else printf(", ");
-        printf("%s", slot);
+        int slot = atoi(ln->value);
+        if (slot >= 0 && slot < CLUSTER_MANAGER_SLOTS)
+            n.slots[slot] = 1;
     }
-    printf("\n");
+    sds nodeslist = clusterManagerNodeSlotsString(&n);
+    printf("%s\n", nodeslist);
+    sdsfree(nodeslist);
 }
 
 /* Return the node, among 'nodes' with the greatest number of keys
@@ -4398,15 +4399,10 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) {
     int i, fixed = 0;
     list *none = NULL, *single = NULL, *multi = NULL;
     clusterManagerLogInfo(">>> Fixing slots coverage...\n");
-    printf("List of not covered slots: \n");
-    int uncovered_count = 0;
-    sds log = sdsempty();
     for (i = 0; i < CLUSTER_MANAGER_SLOTS; i++) {
         int covered = all_slots[i];
         if (!covered) {
-            sds key = sdsfromlonglong((long long) i);
-            if (uncovered_count++ > 0) printf(",");
-            printf("%s", (char *) key);
+            sds slot = sdsfromlonglong((long long) i);
             list *slot_nodes = listCreate();
             sds slot_nodes_str = sdsempty();
             listIter li;
@@ -4433,13 +4429,11 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) {
                 }
                 freeReplyObject(reply);
             }
-            log = sdscatfmt(log, "\nSlot %S has keys in %u nodes: %S",
-                            key, listLength(slot_nodes), slot_nodes_str);
             sdsfree(slot_nodes_str);
-            dictAdd(clusterManagerUncoveredSlots, key, slot_nodes);
+            dictAdd(clusterManagerUncoveredSlots, slot, slot_nodes);
         }
     }
-    printf("\n%s\n", log);
+
     /* For every slot, take action depending on the actual condition:
      * 1) No node has keys for this slot.
      * 2) A single node has keys for this slot.
@@ -4581,7 +4575,6 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) {
         }
     }
 cleanup:
-    sdsfree(log);
     if (none) listRelease(none);
     if (single) listRelease(single);
     if (multi) listRelease(multi);

From 551fed316986af69121c0cdd0ce50efe91d010d6 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 29 Apr 2020 16:57:06 +0200
Subject: [PATCH 0325/1098] redis-cli: safer cluster fix with unreachalbe
 masters.

---
 src/redis-cli.c | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 82a46216c..ec0f58f5f 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -123,6 +123,7 @@
 #define CLUSTER_MANAGER_CMD_FLAG_COPY           1 << 7
 #define CLUSTER_MANAGER_CMD_FLAG_COLOR          1 << 8
 #define CLUSTER_MANAGER_CMD_FLAG_CHECK_OWNERS   1 << 9
+#define CLUSTER_MANAGER_CMD_FLAG_FIX_WITH_UNREACHABLE_MASTERS 1 << 10
 
 #define CLUSTER_MANAGER_OPT_GETFRIENDS  1 << 0
 #define CLUSTER_MANAGER_OPT_COLD        1 << 1
@@ -1599,6 +1600,9 @@ static int parseOptions(int argc, char **argv) {
         } else if (!strcmp(argv[i],"--cluster-search-multiple-owners")) {
             config.cluster_manager_command.flags |=
                 CLUSTER_MANAGER_CMD_FLAG_CHECK_OWNERS;
+        } else if (!strcmp(argv[i],"--cluster-fix-with-unreachable-masters")) {
+            config.cluster_manager_command.flags |=
+                CLUSTER_MANAGER_CMD_FLAG_FIX_WITH_UNREACHABLE_MASTERS;
 #ifdef USE_OPENSSL
         } else if (!strcmp(argv[i],"--tls")) {
             config.tls = 1;
@@ -2146,6 +2150,7 @@ static int evalMode(int argc, char **argv) {
 static struct clusterManager {
     list *nodes;    /* List of nodes in the configuration. */
     list *errors;
+    int unreachable_masters;    /* Masters we are not able to reach. */
 } cluster_manager;
 
 /* Used by clusterManagerFixSlotsCoverage */
@@ -2288,7 +2293,7 @@ clusterManagerCommandDef clusterManagerCommands[] = {
      "search-multiple-owners"},
     {"info", clusterManagerCommandInfo, -1, "host:port", NULL},
     {"fix", clusterManagerCommandFix, -1, "host:port",
-     "search-multiple-owners"},
+     "search-multiple-owners,fix-with-unreachable-masters"},
     {"reshard", clusterManagerCommandReshard, -1, "host:port",
      "from <arg>,to <arg>,slots <arg>,yes,timeout <arg>,pipeline <arg>,"
      "replace"},
@@ -4013,7 +4018,9 @@ static int clusterManagerLoadInfoFromNode(clusterManagerNode *node, int opts) {
                 if (friend->flags & (CLUSTER_MANAGER_FLAG_NOADDR |
                                      CLUSTER_MANAGER_FLAG_DISCONNECT |
                                      CLUSTER_MANAGER_FLAG_FAIL))
+                {
                     goto invalid_friend;
+                }
                 listAddNodeTail(cluster_manager.nodes, friend);
             } else {
                 clusterManagerLogErr("[ERR] Unable to load info for "
@@ -4023,6 +4030,8 @@ static int clusterManagerLoadInfoFromNode(clusterManagerNode *node, int opts) {
             }
             continue;
 invalid_friend:
+            if (!(friend->flags & CLUSTER_MANAGER_FLAG_SLAVE))
+                cluster_manager.unreachable_masters++;
             freeClusterManagerNode(friend);
         }
         listRelease(node->friends);
@@ -4396,6 +4405,14 @@ static clusterManagerNode *clusterManagerNodeMasterRandom() {
 }
 
 static int clusterManagerFixSlotsCoverage(char *all_slots) {
+    int force_fix = config.cluster_manager_command.flags &
+                    CLUSTER_MANAGER_CMD_FLAG_FIX_WITH_UNREACHABLE_MASTERS;
+
+    if (cluster_manager.unreachable_masters > 0 && !force_fix) {
+        clusterManagerLogWarn("*** Fixing slots coverage with %d unreachable masters is dangerous: redis-cli will assume that slots about masters that are not reachable are not covered, and will try to reassign them to the reachable nodes. This can cause data loss and is rarely what you want to do. If you really want to proceed use the --cluster-fix-with-unreachable-masters option.\n", cluster_manager.unreachable_masters);
+        exit(1);
+    }
+
     int i, fixed = 0;
     list *none = NULL, *single = NULL, *multi = NULL;
     clusterManagerLogInfo(">>> Fixing slots coverage...\n");
@@ -4585,6 +4602,14 @@ cleanup:
  * more nodes. This function fixes this condition by migrating keys where
  * it seems more sensible. */
 static int clusterManagerFixOpenSlot(int slot) {
+    int force_fix = config.cluster_manager_command.flags &
+                    CLUSTER_MANAGER_CMD_FLAG_FIX_WITH_UNREACHABLE_MASTERS;
+
+    if (cluster_manager.unreachable_masters > 0 && !force_fix) {
+        clusterManagerLogWarn("*** Fixing open slots with %d unreachable masters is dangerous: redis-cli will assume that slots about masters that are not reachable are not covered, and will try to reassign them to the reachable nodes. This can cause data loss and is rarely what you want to do. If you really want to proceed use the --cluster-fix-with-unreachable-masters option.\n", cluster_manager.unreachable_masters);
+        exit(1);
+    }
+
     clusterManagerLogInfo(">>> Fixing open slot %d\n", slot);
     /* Try to obtain the current slot owner, according to the current
      * nodes configuration. */

From fd8f39a28304179e1dc3b9499d1a0c4bde4191b8 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 29 Apr 2020 18:49:42 +0200
Subject: [PATCH 0326/1098] Fix tracking table max keys option in redis.conf.

---
 redis.conf | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/redis.conf b/redis.conf
index 01a2d801f..d96d26e1c 100644
--- a/redis.conf
+++ b/redis.conf
@@ -626,20 +626,23 @@ replica-priority 100
 # to track the keys fetched by many clients.
 #
 # For this reason it is possible to configure a maximum fill value for the
-# invalidation table. By default it is set to 10%, and once this limit is
-# reached, Redis will start to evict caching slots in the invalidation table
-# even if keys are not modified, just to reclaim memory: this will in turn
+# invalidation table. By default it is set to 1M of keys, and once this limit
+# is reached, Redis will start to evict keys in the invalidation table
+# even if they were not modified, just to reclaim memory: this will in turn
 # force the clients to invalidate the cached values. Basically the table
-# maximum fill rate is a trade off between the memory you want to spend server
+# maximum size is a trade off between the memory you want to spend server
 # side to track information about who cached what, and the ability of clients
 # to retain cached objects in memory.
 #
-# If you set the value to 0, it means there are no limits, and all the 16
-# millions of caching slots can be used at the same time. In the "stats"
-# INFO section, you can find information about the amount of caching slots
-# used at every given moment.
+# If you set the value to 0, it means there are no limits, and Redis will
+# retain as many keys as needed in the invalidation table.
+# In the "stats" INFO section, you can find information about the number of
+# keys in the invalidation table at every given moment.
 #
-# tracking-table-max-fill 10
+# Note: when key tracking is used in broadcasting mode, no memory is used
+# in the server side so this setting is useless.
+#
+# tracking-table-max-keys 1000000
 
 ################################## SECURITY ###################################
 

From 528ea98bd322a77ff4c07bfcb8a371db23ca1ad1 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Wed, 27 Feb 2019 19:38:03 +0800
Subject: [PATCH 0327/1098] lazyfree & eviction: record latency generated by
 lazyfree eviction

1. add eviction-lazyfree monitor
2. put eviction-del & eviction-lazyfree into eviction-cycle
   that means eviction-cycle contains all the latency in
   the eviction cycle including del and lazyfree
3. use getMaxmemoryState to check if we can break in lazyfree-evict
---
 src/evict.c | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/src/evict.c b/src/evict.c
index a829f1f99..0755acc0e 100644
--- a/src/evict.c
+++ b/src/evict.c
@@ -450,9 +450,10 @@ int freeMemoryIfNeeded(void) {
     if (server.masterhost && server.repl_slave_ignore_maxmemory) return C_OK;
 
     size_t mem_reported, mem_tofree, mem_freed;
-    mstime_t latency, eviction_latency;
+    mstime_t latency, eviction_latency, lazyfree_latency;
     long long delta;
     int slaves = listLength(server.slaves);
+    int result = C_ERR;
 
     /* When clients are paused the dataset should be static not just from the
      * POV of clients not being able to write, but also from the POV of
@@ -463,10 +464,10 @@ int freeMemoryIfNeeded(void) {
 
     mem_freed = 0;
 
+    latencyStartMonitor(latency);
     if (server.maxmemory_policy == MAXMEMORY_NO_EVICTION)
         goto cant_free; /* We need to free memory, but policy forbids. */
 
-    latencyStartMonitor(latency);
     while (mem_freed < mem_tofree) {
         int j, k, i;
         static unsigned int next_db = 0;
@@ -572,7 +573,6 @@ int freeMemoryIfNeeded(void) {
             signalModifiedKey(NULL,db,keyobj);
             latencyEndMonitor(eviction_latency);
             latencyAddSampleIfNeeded("eviction-del",eviction_latency);
-            latencyRemoveNestedEvent(latency,eviction_latency);
             delta -= (long long) zmalloc_used_memory();
             mem_freed += delta;
             server.stat_evictedkeys++;
@@ -601,25 +601,30 @@ int freeMemoryIfNeeded(void) {
                 }
             }
         } else {
-            latencyEndMonitor(latency);
-            latencyAddSampleIfNeeded("eviction-cycle",latency);
             goto cant_free; /* nothing to free... */
         }
     }
-    latencyEndMonitor(latency);
-    latencyAddSampleIfNeeded("eviction-cycle",latency);
-    return C_OK;
+    result = C_OK;
 
 cant_free:
     /* We are here if we are not able to reclaim memory. There is only one
      * last thing we can try: check if the lazyfree thread has jobs in queue
      * and wait... */
-    while(bioPendingJobsOfType(BIO_LAZY_FREE)) {
-        if (((mem_reported - zmalloc_used_memory()) + mem_freed) >= mem_tofree)
-            break;
-        usleep(1000);
+    if (result != C_OK) {
+        latencyStartMonitor(lazyfree_latency);
+        while(bioPendingJobsOfType(BIO_LAZY_FREE)) {
+            if (getMaxmemoryState(NULL,NULL,NULL,NULL) == C_OK) {
+                result = C_OK;
+                break;
+            }
+            usleep(1000);
+        }
+        latencyEndMonitor(lazyfree_latency);
+        latencyAddSampleIfNeeded("eviction-lazyfree",lazyfree_latency);
     }
-    return C_ERR;
+    latencyEndMonitor(latency);
+    latencyAddSampleIfNeeded("eviction-cycle",latency);
+    return result;
 }
 
 /* This is a wrapper for freeMemoryIfNeeded() that only really calls the

From cec388f208c44c2f636da42258d14c6ba5411faa Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 30 Apr 2020 09:58:06 +0200
Subject: [PATCH 0328/1098] CLIENT KILL USER <username>.

---
 src/networking.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index 7ffa99eb1..767206ab9 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2101,6 +2101,7 @@ void clientCommand(client *c) {
 "KILL <option> <value> [option value ...] -- Kill connections. Options are:",
 "     ADDR <ip:port>                      -- Kill connection made from <ip:port>",
 "     TYPE (normal|master|replica|pubsub) -- Kill connections by type.",
+"     USER <username>   -- Kill connections authenticated with such user.",
 "     SKIPME (yes|no)   -- Skip killing current connection (default: yes).",
 "LIST [options ...]     -- Return information about client connections. Options:",
 "     TYPE (normal|master|replica|pubsub) -- Return clients of specified type.",
@@ -2151,6 +2152,7 @@ NULL
         /* CLIENT KILL <ip:port>
          * CLIENT KILL <option> [value] ... <option> [value] */
         char *addr = NULL;
+        user *user = NULL;
         int type = -1;
         uint64_t id = 0;
         int skipme = 1;
@@ -2182,6 +2184,14 @@ NULL
                     }
                 } else if (!strcasecmp(c->argv[i]->ptr,"addr") && moreargs) {
                     addr = c->argv[i+1]->ptr;
+                } else if (!strcasecmp(c->argv[i]->ptr,"user") && moreargs) {
+                    user = ACLGetUserByName(c->argv[i+1]->ptr,
+                                            sdslen(c->argv[i+1]->ptr));
+                    if (user == NULL) {
+                        addReplyErrorFormat(c,"No such user '%s'",
+                            (char*) c->argv[i+1]->ptr);
+                        return;
+                    }
                 } else if (!strcasecmp(c->argv[i]->ptr,"skipme") && moreargs) {
                     if (!strcasecmp(c->argv[i+1]->ptr,"yes")) {
                         skipme = 1;
@@ -2209,6 +2219,7 @@ NULL
             if (addr && strcmp(getClientPeerId(client),addr) != 0) continue;
             if (type != -1 && getClientType(client) != type) continue;
             if (id != 0 && client->id != id) continue;
+            if (user && client->user != user) continue;
             if (c == client && skipme) continue;
 
             /* Kill it. */

From 058c727282312d641b935d0acdccc31a3591b245 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 30 Apr 2020 10:14:15 +0200
Subject: [PATCH 0329/1098] MIGRATE AUTH2 for ACL support.

---
 src/cluster.c | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 49a410d54..d8f83f08d 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -5101,15 +5101,17 @@ void migrateCloseTimedoutSockets(void) {
     dictReleaseIterator(di);
 }
 
-/* MIGRATE host port key dbid timeout [COPY | REPLACE | AUTH password]
+/* MIGRATE host port key dbid timeout [COPY | REPLACE | AUTH password |
+ *         AUTH2 username password]
  *
  * On in the multiple keys form:
  *
- * MIGRATE host port "" dbid timeout [COPY | REPLACE | AUTH password] KEYS key1
- * key2 ... keyN */
+ * MIGRATE host port "" dbid timeout [COPY | REPLACE | AUTH password |
+ *         AUTH2 username password] KEYS key1 key2 ... keyN */
 void migrateCommand(client *c) {
     migrateCachedSocket *cs;
     int copy = 0, replace = 0, j;
+    char *username = NULL;
     char *password = NULL;
     long timeout;
     long dbid;
@@ -5127,7 +5129,7 @@ void migrateCommand(client *c) {
 
     /* Parse additional options */
     for (j = 6; j < c->argc; j++) {
-        int moreargs = j < c->argc-1;
+        int moreargs = (c->argc-1) - j;
         if (!strcasecmp(c->argv[j]->ptr,"copy")) {
             copy = 1;
         } else if (!strcasecmp(c->argv[j]->ptr,"replace")) {
@@ -5139,6 +5141,13 @@ void migrateCommand(client *c) {
             }
             j++;
             password = c->argv[j]->ptr;
+        } else if (!strcasecmp(c->argv[j]->ptr,"auth2")) {
+            if (moreargs < 2) {
+                addReply(c,shared.syntaxerr);
+                return;
+            }
+            username = c->argv[++j]->ptr;
+            password = c->argv[++j]->ptr;
         } else if (!strcasecmp(c->argv[j]->ptr,"keys")) {
             if (sdslen(c->argv[3]->ptr) != 0) {
                 addReplyError(c,
@@ -5199,8 +5208,13 @@ try_again:
 
     /* Authentication */
     if (password) {
-        serverAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',2));
+        int arity = username ? 3 : 2;
+        serverAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',arity));
         serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"AUTH",4));
+        if (username) {
+            serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,username,
+                                 sdslen(username)));
+        }
         serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,password,
             sdslen(password)));
     }

From 6fcdf75325e4be85687571b50ae4017329b32bfd Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 30 Apr 2020 13:00:37 +0200
Subject: [PATCH 0330/1098] redis-cli command help updated.

---
 src/help.h | 190 ++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 165 insertions(+), 25 deletions(-)

diff --git a/src/help.h b/src/help.h
index 184d76724..6aa875541 100644
--- a/src/help.h
+++ b/src/help.h
@@ -28,6 +28,56 @@ struct commandHelp {
   int group;
   char *since;
 } commandHelp[] = {
+    { "ACL CAT",
+    "[categoryname]",
+    "List the ACL categories or the commands inside a category",
+    9,
+    "6.0.0" },
+    { "ACL DELUSER",
+    "username [username ...]",
+    "Remove the specified ACL users and the associated rules",
+    9,
+    "6.0.0" },
+    { "ACL GENPASS",
+    "[bits]",
+    "Generate a pseudorandom secure password to use for ACL users",
+    9,
+    "6.0.0" },
+    { "ACL LIST",
+    "-",
+    "List the current ACL rules in ACL config file format",
+    9,
+    "6.0.0" },
+    { "ACL LOAD",
+    "-",
+    "Reload the ACLs from the configured ACL file",
+    9,
+    "6.0.0" },
+    { "ACL LOG",
+    "[count or RESET]",
+    "List latest events denied because of ACLs in place",
+    9,
+    "6.0.0" },
+    { "ACL SAVE",
+    "-",
+    "Save the current ACL rules in the configured ACL file",
+    9,
+    "6.0.0" },
+    { "ACL SETUSER",
+    "rule [rule ...]",
+    "Modify or create the rules for a specific ACL user",
+    9,
+    "6.0.0" },
+    { "ACL USERS",
+    "-",
+    "List the username of all the configured ACL rules",
+    9,
+    "6.0.0" },
+    { "ACL WHOAMI",
+    "-",
+    "Return the name of the user associated to the current connection",
+    9,
+    "6.0.0" },
     { "APPEND",
     "key value",
     "Append a value to a key",
@@ -44,7 +94,7 @@ struct commandHelp {
     9,
     "1.0.0" },
     { "BGSAVE",
-    "-",
+    "[SCHEDULE]",
     "Asynchronously save the dataset to disk",
     9,
     "1.0.0" },
@@ -80,7 +130,7 @@ struct commandHelp {
     "2.0.0" },
     { "BRPOPLPUSH",
     "source destination timeout",
-    "Pop a value from a list, push it to another list and return it; or block until one is available",
+    "Pop an element from a list, push it to another list and return it; or block until one is available",
     2,
     "2.2.0" },
     { "BZPOPMAX",
@@ -93,11 +143,21 @@ struct commandHelp {
     "Remove and return the member with the lowest score from one or more sorted sets, or block until one is available",
     4,
     "5.0.0" },
+    { "CLIENT CACHING",
+    "YES|NO",
+    "Instruct the server about tracking or not keys in the next request",
+    9,
+    "6.0.0" },
     { "CLIENT GETNAME",
     "-",
     "Get the current connection name",
     9,
     "2.6.9" },
+    { "CLIENT GETREDIR",
+    "-",
+    "Get tracking notifications redirection client ID if any",
+    9,
+    "6.0.0" },
     { "CLIENT ID",
     "-",
     "Returns the client ID for the current connection",
@@ -109,7 +169,7 @@ struct commandHelp {
     9,
     "2.4.0" },
     { "CLIENT LIST",
-    "-",
+    "[TYPE normal|master|replica|pubsub]",
     "Get the list of client connections",
     9,
     "2.4.0" },
@@ -128,6 +188,11 @@ struct commandHelp {
     "Set the current connection name",
     9,
     "2.6.9" },
+    { "CLIENT TRACKING",
+    "ON|OFF [REDIRECT client-id] [PREFIX prefix] [BCAST] [OPTIN] [OPTOUT] [NOLOOP]",
+    "Enable or disable server assisted client side caching support",
+    9,
+    "6.0.0" },
     { "CLIENT UNBLOCK",
     "client-id [TIMEOUT|ERROR]",
     "Unblock a client blocked in a blocking command from a different connection",
@@ -138,6 +203,11 @@ struct commandHelp {
     "Assign new hash slots to receiving node",
     12,
     "3.0.0" },
+    { "CLUSTER BUMPEPOCH",
+    "-",
+    "Advance the cluster config epoch",
+    12,
+    "3.0.0" },
     { "CLUSTER COUNT-FAILURE-REPORTS",
     "node-id",
     "Return the number of failure reports active for a given node",
@@ -158,6 +228,11 @@ struct commandHelp {
     "Forces a replica to perform a manual failover of its master.",
     12,
     "3.0.0" },
+    { "CLUSTER FLUSHSLOTS",
+    "-",
+    "Delete a node's own slots information",
+    12,
+    "3.0.0" },
     { "CLUSTER FORGET",
     "node-id",
     "Remove a node from the nodes table",
@@ -183,6 +258,11 @@ struct commandHelp {
     "Force a node cluster to handshake with another node",
     12,
     "3.0.0" },
+    { "CLUSTER MYID",
+    "-",
+    "Return the node id",
+    12,
+    "3.0.0" },
     { "CLUSTER NODES",
     "-",
     "Get Cluster config for the node",
@@ -359,7 +439,7 @@ struct commandHelp {
     13,
     "3.2.0" },
     { "GEODIST",
-    "key member1 member2 [unit]",
+    "key member1 member2 [m|km|ft|mi]",
     "Returns the distance between two members of a geospatial index",
     13,
     "3.2.0" },
@@ -459,7 +539,7 @@ struct commandHelp {
     5,
     "2.8.0" },
     { "HSET",
-    "key field value",
+    "key field value [field value ...]",
     "Set the string value of a hash field",
     5,
     "2.0.0" },
@@ -508,13 +588,43 @@ struct commandHelp {
     "Get the UNIX time stamp of the last successful save to disk",
     9,
     "1.0.0" },
+    { "LATENCY DOCTOR",
+    "-",
+    "Return a human readable latency analysis report.",
+    9,
+    "2.8.13" },
+    { "LATENCY GRAPH",
+    "event",
+    "Return a latency graph for the event.",
+    9,
+    "2.8.13" },
+    { "LATENCY HELP",
+    "-",
+    "Show helpful text about the different subcommands.",
+    9,
+    "2.8.13" },
+    { "LATENCY HISTORY",
+    "event",
+    "Return timestamp-latency samples for the event.",
+    9,
+    "2.8.13" },
+    { "LATENCY LATEST",
+    "-",
+    "Return the latest latency samples for all events.",
+    9,
+    "2.8.13" },
+    { "LATENCY RESET",
+    "[event]",
+    "Reset latency data for one or more events.",
+    9,
+    "2.8.13" },
     { "LINDEX",
     "key index",
     "Get an element from a list by its index",
     2,
     "1.0.0" },
     { "LINSERT",
-    "key BEFORE|AFTER pivot value",
+    "key BEFORE|AFTER pivot element",
     "Insert an element before or after another element in a list",
     2,
     "2.2.0" },
@@ -523,19 +633,24 @@ struct commandHelp {
     "Get the length of a list",
     2,
     "1.0.0" },
+    { "LOLWUT",
+    "[VERSION version]",
+    "Display some computer art and the Redis version",
+    9,
+    "5.0.0" },
     { "LPOP",
     "key",
     "Remove and get the first element in a list",
     2,
     "1.0.0" },
     { "LPUSH",
-    "key value [value ...]",
-    "Prepend one or multiple values to a list",
+    "key element [element ...]",
+    "Prepend one or multiple elements to a list",
     2,
     "1.0.0" },
     { "LPUSHX",
-    "key value",
-    "Prepend a value to a list, only if the list exists",
+    "key element [element ...]",
+    "Prepend an element to a list, only if the list exists",
     2,
     "2.2.0" },
     { "LRANGE",
@@ -544,12 +659,12 @@ struct commandHelp {
     2,
     "1.0.0" },
     { "LREM",
-    "key count value",
+    "key count element",
     "Remove elements from a list",
     2,
     "1.0.0" },
     { "LSET",
-    "key index value",
+    "key index element",
     "Set the value of an element in a list by its index",
     2,
     "1.0.0" },
@@ -594,10 +709,25 @@ struct commandHelp {
     1,
     "1.0.0" },
     { "MIGRATE",
-    "host port key|"" destination-db timeout [COPY] [REPLACE] [KEYS key]",
+    "host port key|"" destination-db timeout [COPY] [REPLACE] [AUTH password] [KEYS key]",
     "Atomically transfer a key from a Redis instance to another one.",
     0,
     "2.6.0" },
+    { "MODULE LIST",
+    "-",
+    "List all modules loaded by the server",
+    9,
+    "4.0.0" },
+    { "MODULE LOAD",
+    "path [arg]",
+    "Load a module",
+    9,
+    "4.0.0" },
+    { "MODULE UNLOAD",
+    "name",
+    "Unload a module",
+    9,
+    "4.0.0" },
     { "MONITOR",
     "-",
     "Listen for all requests received by the server in real time",
@@ -673,6 +803,11 @@ struct commandHelp {
     "Listen for messages published to channels matching the given patterns",
     6,
     "2.0.0" },
+    { "PSYNC",
+    "replicationid offset",
+    "Internal command used for replication",
+    9,
+    "2.8.0" },
     { "PTTL",
     "key",
     "Get the time to live for a key in milliseconds",
@@ -729,7 +864,7 @@ struct commandHelp {
     9,
     "5.0.0" },
     { "RESTORE",
-    "key ttl serialized-value [REPLACE]",
+    "key ttl serialized-value [REPLACE] [ABSTTL] [IDLETIME seconds] [FREQ frequency]",
     "Create a key using the provided serialized value, previously obtained using DUMP.",
     0,
     "2.6.0" },
@@ -749,13 +884,13 @@ struct commandHelp {
     2,
     "1.2.0" },
     { "RPUSH",
-    "key value [value ...]",
-    "Append one or multiple values to a list",
+    "key element [element ...]",
+    "Append one or multiple elements to a list",
     2,
     "1.0.0" },
     { "RPUSHX",
-    "key value",
-    "Append a value to a list, only if the list exists",
+    "key element [element ...]",
+    "Append an element to a list, only if the list exists",
     2,
     "2.2.0" },
     { "SADD",
@@ -769,7 +904,7 @@ struct commandHelp {
     9,
     "1.0.0" },
     { "SCAN",
-    "cursor [MATCH pattern] [COUNT count]",
+    "cursor [MATCH pattern] [COUNT count] [TYPE type]",
     "Incrementally iterate the keys space",
     0,
     "2.8.0" },
@@ -819,7 +954,7 @@ struct commandHelp {
     8,
     "1.0.0" },
     { "SET",
-    "key value [expiration EX seconds|PX milliseconds] [NX|XX]",
+    "key value [EX seconds|PX milliseconds] [NX|XX] [KEEPTTL]",
     "Set the string value of a key",
     1,
     "1.0.0" },
@@ -908,6 +1043,11 @@ struct commandHelp {
     "Incrementally iterate Set elements",
     3,
     "2.8.0" },
+    { "STRALGO",
+    "LCS algo-specific-argument [algo-specific-argument ...]",
+    "Run algorithms (currently LCS) against strings",
+    1,
+    "6.0.0" },
     { "STRLEN",
     "key",
     "Get the length of the value stored in a key",
@@ -929,7 +1069,7 @@ struct commandHelp {
     3,
     "1.0.0" },
     { "SWAPDB",
-    "index index",
+    "index1 index2",
     "Swaps two Redis databases",
     8,
     "4.0.0" },
@@ -989,7 +1129,7 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XADD",
-    "key ID field string [field string ...]",
+    "key ID field value [field value ...]",
     "Appends a new entry to a stream",
     14,
     "5.0.0" },
@@ -1004,7 +1144,7 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XGROUP",
-    "[CREATE key groupname id-or-$] [SETID key id-or-$] [DESTROY key groupname] [DELCONSUMER key groupname consumername]",
+    "[CREATE key groupname id-or-$] [SETID key groupname id-or-$] [DESTROY key groupname] [DELCONSUMER key groupname consumername]",
     "Create, destroy, and manage consumer groups.",
     14,
     "5.0.0" },
@@ -1029,12 +1169,12 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XREAD",
-    "[COUNT count] [BLOCK milliseconds] STREAMS key [key ...] ID [ID ...]",
+    "[COUNT count] [BLOCK milliseconds] STREAMS key [key ...] id [id ...]",
     "Return never seen elements in multiple streams, with IDs greater than the ones reported by the caller for each stream. Can block.",
     14,
     "5.0.0" },
     { "XREADGROUP",
-    "GROUP group consumer [COUNT count] [BLOCK milliseconds] STREAMS key [key ...] ID [ID ...]",
+    "GROUP group consumer [COUNT count] [BLOCK milliseconds] [NOACK] STREAMS key [key ...] ID [ID ...]",
     "Return new entries from a stream using a consumer group, or access the history of the pending entries for a given consumer. Can block.",
     14,
     "5.0.0" },

From 684c5907dd8bc481470a45e5b1b1fc3beaa24cb7 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 30 Apr 2020 13:42:25 +0200
Subject: [PATCH 0331/1098] redis-cli: fix hints with subcommands.

---
 src/redis-cli.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index ec0f58f5f..880f3d70a 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -699,7 +699,8 @@ static char *hintsCallback(const char *buf, int *color, int *bold) {
     for (i = 0; i < helpEntriesLen; i++) {
         if (!(helpEntries[i].type & CLI_HELP_COMMAND)) continue;
 
-        if (strcasecmp(argv[0],helpEntries[i].full) == 0)
+        if (strcasecmp(argv[0],helpEntries[i].full) == 0 ||
+            strcasecmp(buf,helpEntries[i].full) == 0)
         {
             *color = 90;
             *bold = 0;

From 69ce235c7b870e43813da7e03160c320389ea6a9 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 30 Apr 2020 13:43:05 +0200
Subject: [PATCH 0332/1098] Update help.h again before Redis 6 GA.

---
 src/help.h | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/help.h b/src/help.h
index 6aa875541..6d3eb33ed 100644
--- a/src/help.h
+++ b/src/help.h
@@ -146,57 +146,57 @@ struct commandHelp {
     { "CLIENT CACHING",
     "YES|NO",
     "Instruct the server about tracking or not keys in the next request",
-    9,
+    8,
     "6.0.0" },
     { "CLIENT GETNAME",
     "-",
     "Get the current connection name",
-    9,
+    8,
     "2.6.9" },
     { "CLIENT GETREDIR",
     "-",
     "Get tracking notifications redirection client ID if any",
-    9,
+    8,
     "6.0.0" },
     { "CLIENT ID",
     "-",
     "Returns the client ID for the current connection",
-    9,
+    8,
     "5.0.0" },
     { "CLIENT KILL",
     "[ip:port] [ID client-id] [TYPE normal|master|slave|pubsub] [ADDR ip:port] [SKIPME yes/no]",
     "Kill the connection of a client",
-    9,
+    8,
     "2.4.0" },
     { "CLIENT LIST",
     "[TYPE normal|master|replica|pubsub]",
     "Get the list of client connections",
-    9,
+    8,
     "2.4.0" },
     { "CLIENT PAUSE",
     "timeout",
     "Stop processing commands from clients for some time",
-    9,
+    8,
     "2.9.50" },
     { "CLIENT REPLY",
     "ON|OFF|SKIP",
     "Instruct the server whether to reply to commands",
-    9,
+    8,
     "3.2" },
     { "CLIENT SETNAME",
     "connection-name",
     "Set the current connection name",
-    9,
+    8,
     "2.6.9" },
     { "CLIENT TRACKING",
     "ON|OFF [REDIRECT client-id] [PREFIX prefix] [BCAST] [OPTIN] [OPTOUT] [NOLOOP]",
     "Enable or disable server assisted client side caching support",
-    9,
+    8,
     "6.0.0" },
     { "CLIENT UNBLOCK",
     "client-id [TIMEOUT|ERROR]",
     "Unblock a client blocked in a blocking command from a different connection",
-    9,
+    8,
     "5.0.0" },
     { "CLUSTER ADDSLOTS",
     "slot [slot ...]",
@@ -488,6 +488,11 @@ struct commandHelp {
     "Delete one or more hash fields",
     5,
     "2.0.0" },
+    { "HELLO",
+    "protover [AUTH username password] [SETNAME clientname]",
+    "switch Redis protocol",
+    8,
+    "6.0.0" },
     { "HEXISTS",
     "key field",
     "Determine if a hash field exists",
@@ -1071,7 +1076,7 @@ struct commandHelp {
     { "SWAPDB",
     "index1 index2",
     "Swaps two Redis databases",
-    8,
+    9,
     "4.0.0" },
     { "SYNC",
     "-",

From fe980e2334ff3908c5d00543c8000b4be78421cf Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 1 May 2020 18:45:51 +0200
Subject: [PATCH 0333/1098] Save a call to stopThreadedIOIfNeeded() for the
 base case.

Probably no performance changes, but the code should be trivial to
read as in "No threading? Use the normal function and return".
---
 src/networking.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 767206ab9..d4c0306c3 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2997,9 +2997,9 @@ int handleClientsWithPendingWritesUsingThreads(void) {
     int processed = listLength(server.clients_pending_write);
     if (processed == 0) return 0; /* Return ASAP if there are no clients. */
 
-    /* If we have just a few clients to serve, don't use I/O threads, but the
-     * boring synchronous code. */
-    if (stopThreadedIOIfNeeded()) {
+    /* If I/O threads are disabled or we have few clients to serve, don't
+     * use I/O threads, but thejboring synchronous code. */
+    if (server.io_threads_num == 1 || stopThreadedIOIfNeeded()) {
         return handleClientsWithPendingWrites();
     }
 

From 365316aa59545d90de4e105cec57f22aa5b52ff9 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 1 May 2020 23:57:11 +0200
Subject: [PATCH 0334/1098] Revert "optimize memory usage of deferred replies"

This reverts commit fb732f7a944a4d4c90bb7375cb6030e88211f5aa.
---
 src/networking.c | 31 -------------------------------
 1 file changed, 31 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index d4c0306c3..c4a277e0a 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -436,36 +436,6 @@ void addReplyStatusFormat(client *c, const char *fmt, ...) {
     sdsfree(s);
 }
 
-/* Sometimes we are forced to create a new reply node, and we can't append to
- * the previous one, when that happens, we wanna try to trim the unused space
- * at the end of the last reply node which we won't use anymore. */
-void trimReplyUnusedTailSpace(client *c) {
-    listNode *ln = listLast(c->reply);
-    clientReplyBlock *tail = ln? listNodeValue(ln): NULL;
-
-    /* Note that 'tail' may be NULL even if we have a tail node, becuase when
-     * addDeferredMultiBulkLength() is used */
-    if (!tail) return;
-
-    /* We only try to trim the space is relatively high (more than a 1/4 of the
-     * allocation), otherwise there's a high chance realloc will NOP.
-     * Also, to avoid large memmove which happens as part of realloc, we only do
-     * that if the used part is small.  */
-    if (tail->size - tail->used > tail->size / 4 &&
-        tail->used < PROTO_REPLY_CHUNK_BYTES)
-    {
-        size_t old_size = tail->size;
-        tail = zrealloc(tail, tail->used + sizeof(clientReplyBlock));
-        /* If realloc was a NOP, we got the same value which has internal frag */
-        if (tail == listNodeValue(ln)) return;
-        /* take over the allocation's internal fragmentation (at least for
-         * memory usage tracking) */
-        tail->size = zmalloc_usable(tail) - sizeof(clientReplyBlock);
-        c->reply_bytes += tail->size - old_size;
-        listNodeValue(ln) = tail;
-    }
-}
-
 /* Adds an empty object to the reply list that will contain the multi bulk
  * length, which is not known when this function is called. */
 void *addReplyDeferredLen(client *c) {
@@ -473,7 +443,6 @@ void *addReplyDeferredLen(client *c) {
      * ready to be sent, since we are sure that before returning to the
      * event loop setDeferredAggregateLen() will be called. */
     if (prepareClientToWrite(c) != C_OK) return NULL;
-    trimReplyUnusedTailSpace(c);
     listAddNodeTail(c->reply,NULL); /* NULL is our placeholder. */
     return listLast(c->reply);
 }

From a07a4ada1919b920665df8f07dde177f6ff22eda Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Sat, 2 May 2020 00:02:18 +0200
Subject: [PATCH 0335/1098] Cast printf() argument to the format specifier.

We could use uint64_t specific macros, but after all it's simpler to
just use an obvious equivalent type plus casting: this will be a no op
and is simpler than fixed size types printf macros.
---
 src/rdb.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/rdb.c b/src/rdb.c
index 9f6bf13f1..4fc385848 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2292,7 +2292,9 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
                 serverLog(LL_WARNING,"RDB file was saved with checksum disabled: no check performed.");
             } else if (cksum != expected) {
                 serverLog(LL_WARNING,"Wrong RDB checksum expected: (%llx) but "
-                    "got (%llx). Aborting now.",expected,cksum);
+                    "got (%llx). Aborting now.",
+                        (unsigned long long)expected,
+                        (unsigned long long)cksum);
                 rdbExitReportCorruptRDB("RDB CRC error");
             }
         }

From 93021da221f71cf71fe874fd881ea59f325b82f2 Mon Sep 17 00:00:00 2001
From: Benjamin Sergeant <bsergean@gmail.com>
Date: Fri, 1 May 2020 20:57:51 -0700
Subject: [PATCH 0336/1098] Update redis-cli.c

---
 src/redis-cli.c | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 880f3d70a..208f7b35d 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -3416,6 +3416,7 @@ static redisReply *clusterManagerMigrateKeysInReply(clusterManagerNode *source,
     size_t *argv_len = NULL;
     int c = (replace ? 8 : 7);
     if (config.auth) c += 2;
+    if (config.user) c += 1;
     size_t argc = c + reply->elements;
     size_t i, offset = 6; // Keys Offset
     argv = zcalloc(argc * sizeof(char *));
@@ -3442,12 +3443,24 @@ static redisReply *clusterManagerMigrateKeysInReply(clusterManagerNode *source,
         offset++;
     }
     if (config.auth) {
-        argv[offset] = "AUTH";
-        argv_len[offset] = 4;
-        offset++;
-        argv[offset] = config.auth;
-        argv_len[offset] = strlen(config.auth);
-        offset++;
+        if (config.user) {
+            argv[offset] = "AUTH2";
+            argv_len[offset] = 5;
+            offset++;
+            argv[offset] = config.user;
+            argv_len[offset] = strlen(config.user);
+            offset++;
+            argv[offset] = config.auth;
+            argv_len[offset] = strlen(config.auth);
+            offset++;
+        } else {
+            argv[offset] = "AUTH";
+            argv_len[offset] = 4;
+            offset++;
+            argv[offset] = config.auth;
+            argv_len[offset] = strlen(config.auth);
+            offset++;
+        }
     }
     argv[offset] = "KEYS";
     argv_len[offset] = 4;

From 6726b3c2cb41e700c8cfd2f821df6c8c847a5ddc Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 24 Apr 2020 17:20:28 +0300
Subject: [PATCH 0337/1098] optimize memory usage of deferred replies - fixed

When deffered reply is added the previous reply node cannot be used so
all the extra space we allocated in it is wasted. in case someone uses
deffered replies in a loop, each time adding a small reply, each of
these reply nodes (the small string reply) would have consumed a 16k
block.
now when we add anther diferred reply node, we trim the unused portion
of the previous reply block.

see #7123

cherry picked from commit fb732f7a944a4d4c90bb7375cb6030e88211f5aa
with fix to handle a crash with LIBC allocator, which apparently can
return the same pointer despite changing it's size.
i.e. shrinking an allocation of 16k into 56 bytes without changing the
pointer.
---
 src/networking.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index c4a277e0a..d62533e3e 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -436,6 +436,34 @@ void addReplyStatusFormat(client *c, const char *fmt, ...) {
     sdsfree(s);
 }
 
+/* Sometimes we are forced to create a new reply node, and we can't append to
+ * the previous one, when that happens, we wanna try to trim the unused space
+ * at the end of the last reply node which we won't use anymore. */
+void trimReplyUnusedTailSpace(client *c) {
+    listNode *ln = listLast(c->reply);
+    clientReplyBlock *tail = ln? listNodeValue(ln): NULL;
+
+    /* Note that 'tail' may be NULL even if we have a tail node, becuase when
+     * addDeferredMultiBulkLength() is used */
+    if (!tail) return;
+
+    /* We only try to trim the space is relatively high (more than a 1/4 of the
+     * allocation), otherwise there's a high chance realloc will NOP.
+     * Also, to avoid large memmove which happens as part of realloc, we only do
+     * that if the used part is small.  */
+    if (tail->size - tail->used > tail->size / 4 &&
+        tail->used < PROTO_REPLY_CHUNK_BYTES)
+    {
+        size_t old_size = tail->size;
+        tail = zrealloc(tail, tail->used + sizeof(clientReplyBlock));
+        /* take over the allocation's internal fragmentation (at least for
+         * memory usage tracking) */
+        tail->size = zmalloc_usable(tail) - sizeof(clientReplyBlock);
+        c->reply_bytes += tail->size - old_size;
+        listNodeValue(ln) = tail;
+    }
+}
+
 /* Adds an empty object to the reply list that will contain the multi bulk
  * length, which is not known when this function is called. */
 void *addReplyDeferredLen(client *c) {
@@ -443,6 +471,7 @@ void *addReplyDeferredLen(client *c) {
      * ready to be sent, since we are sure that before returning to the
      * event loop setDeferredAggregateLen() will be called. */
     if (prepareClientToWrite(c) != C_OK) return NULL;
+    trimReplyUnusedTailSpace(c);
     listAddNodeTail(c->reply,NULL); /* NULL is our placeholder. */
     return listLast(c->reply);
 }

From 1a0deab2a548fa306171f03439e858c00836fe69 Mon Sep 17 00:00:00 2001
From: zhenwei pi <pizhenwei@bytedance.com>
Date: Sat, 2 May 2020 20:05:39 +0800
Subject: [PATCH 0338/1098] Support setcpuaffinity on linux/bsd

Currently, there are several types of threads/child processes of a
redis server. Sometimes we need deeply optimise the performance of
redis, so we would like to isolate threads/processes.

There were some discussion about cpu affinity cases in the issue:
https://github.com/antirez/redis/issues/2863

So implement cpu affinity setting by redis.conf in this patch, then
we can config server_cpulist/bio_cpulist/aof_rewrite_cpulist/
bgsave_cpulist by cpu list.

Examples of cpulist in redis.conf:
server_cpulist 0-7:2      means cpu affinity 0,2,4,6
bio_cpulist 1,3           means cpu affinity 1,3
aof_rewrite_cpulist 8-11  means cpu affinity 8,9,10,11
bgsave_cpulist 1,10-11    means cpu affinity 1,10,11

Test on linux/freebsd, both work fine.

Signed-off-by: zhenwei pi <pizhenwei@bytedance.com>
---
 redis.conf                   |  15 ++++
 src/Makefile                 |   2 +-
 src/aof.c                    |   1 +
 src/bio.c                    |   2 +
 src/config.c                 |   4 ++
 src/config.h                 |   6 ++
 src/networking.c             |   1 +
 src/rdb.c                    |   2 +
 src/server.c                 |   9 +++
 src/server.h                 |   6 ++
 src/setcpuaffinity.c         | 129 +++++++++++++++++++++++++++++++++++
 tests/unit/introspection.tcl |   4 ++
 12 files changed, 180 insertions(+), 1 deletion(-)
 create mode 100644 src/setcpuaffinity.c

diff --git a/redis.conf b/redis.conf
index d96d26e1c..756073a27 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1780,3 +1780,18 @@ rdb-save-incremental-fsync yes
 # Maximum number of set/hash/zset/list fields that will be processed from
 # the main dictionary scan
 # active-defrag-max-scan-fields 1000
+
+# Redis server/IO threads, bio threads, aof rewrite child process, and bgsave
+# child process cpu affinity list config. syntax of cpu list looks like taskset
+# command. serveral examples:
+# set redis server/io threads to cpu affinity 0,2,4,6
+# server_cpulist 0-7:2
+#
+# set bio threads to cpu affinity 1,3
+# bio_cpulist 1,3
+#
+# set aof rewrite child process to cpu affinity 8,9,10,11
+# aof_rewrite_cpulist 8-11
+#
+# set bgsave child process to cpu affinity 1,10,11
+# bgsave_cpulist 1,10-11
diff --git a/src/Makefile b/src/Makefile
index f9922afce..55f862cfc 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -206,7 +206,7 @@ endif
 
 REDIS_SERVER_NAME=redis-server
 REDIS_SENTINEL_NAME=redis-sentinel
-REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o
+REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o
 REDIS_CLI_NAME=redis-cli
 REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o
 REDIS_BENCHMARK_NAME=redis-benchmark
diff --git a/src/aof.c b/src/aof.c
index 301a40848..02409abe6 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1596,6 +1596,7 @@ int rewriteAppendOnlyFileBackground(void) {
 
         /* Child */
         redisSetProcTitle("redis-aof-rewrite");
+        redisSetCpuAffinity(server.aof_rewrite_cpulist);
         snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
         if (rewriteAppendOnlyFile(tmpfile) == C_OK) {
             sendChildCOWInfo(CHILD_INFO_TYPE_AOF, "AOF rewrite");
diff --git a/src/bio.c b/src/bio.c
index 85f681185..69c62fc6f 100644
--- a/src/bio.c
+++ b/src/bio.c
@@ -166,6 +166,8 @@ void *bioProcessBackgroundJobs(void *arg) {
         break;
     }
 
+    redisSetCpuAffinity(server.bio_cpulist);
+
     /* Make the thread killable at any time, so that bioKillThreads()
      * can work reliably. */
     pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
diff --git a/src/config.c b/src/config.c
index e0cbcc281..64854592c 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2133,6 +2133,10 @@ standardConfig configs[] = {
     createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.syslog_ident, "redis", NULL, NULL),
     createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.rdb_filename, "dump.rdb", isValidDBfilename, NULL),
     createStringConfig("appendfilename", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.aof_filename, "appendonly.aof", isValidAOFfilename, NULL),
+    createStringConfig("server_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.server_cpulist, NULL, NULL, NULL),
+    createStringConfig("bio_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bio_cpulist, NULL, NULL, NULL),
+    createStringConfig("aof_rewrite_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.aof_rewrite_cpulist, NULL, NULL, NULL),
+    createStringConfig("bgsave_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bgsave_cpulist, NULL, NULL, NULL),
 
     /* Enum Configs */
     createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE, NULL, NULL),
diff --git a/src/config.h b/src/config.h
index 40dc683ce..a322ce354 100644
--- a/src/config.h
+++ b/src/config.h
@@ -244,4 +244,10 @@ int pthread_setname_np(const char *name);
 #endif
 #endif
 
+/* Check if we can use setcpuaffinity(). */
+#if (defined __linux || defined __NetBSD__ || defined __FreeBSD__ || defined __OpenBSD__)
+#define USE_SETCPUAFFINITY
+void setcpuaffinity(const char *cpulist);
+#endif
+
 #endif
diff --git a/src/networking.c b/src/networking.c
index c4a277e0a..9d10c4bb4 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2843,6 +2843,7 @@ void *IOThreadMain(void *myid) {
 
     snprintf(thdname, sizeof(thdname), "io_thd_%ld", id);
     redis_set_thread_title(thdname);
+    redisSetCpuAffinity(server.server_cpulist);
 
     while(1) {
         /* Wait for start */
diff --git a/src/rdb.c b/src/rdb.c
index 9f6bf13f1..e2a2fb39f 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1351,6 +1351,7 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
 
         /* Child */
         redisSetProcTitle("redis-rdb-bgsave");
+        redisSetCpuAffinity(server.bgsave_cpulist);
         retval = rdbSave(filename,rsi);
         if (retval == C_OK) {
             sendChildCOWInfo(CHILD_INFO_TYPE_RDB, "RDB");
@@ -2458,6 +2459,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
         rioInitWithFd(&rdb,server.rdb_pipe_write);
 
         redisSetProcTitle("redis-rdb-to-slaves");
+        redisSetCpuAffinity(server.bgsave_cpulist);
 
         retval = rdbSaveRioWithEOFMark(&rdb,NULL,rsi);
         if (retval == C_OK && rioFlush(&rdb) == 0)
diff --git a/src/server.c b/src/server.c
index 659604ef3..1baa044be 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4850,6 +4850,14 @@ void redisSetProcTitle(char *title) {
 #endif
 }
 
+void redisSetCpuAffinity(const char *cpulist) {
+#ifdef USE_SETCPUAFFINITY
+    setcpuaffinity(cpulist);
+#else
+    UNUSED(cpulist);
+#endif
+}
+
 /*
  * Check whether systemd or upstart have been used to start redis.
  */
@@ -5118,6 +5126,7 @@ int main(int argc, char **argv) {
         serverLog(LL_WARNING,"WARNING: You specified a maxmemory value that is less than 1MB (current value is %llu bytes). Are you sure this is what you really want?", server.maxmemory);
     }
 
+    redisSetCpuAffinity(server.server_cpulist);
     aeSetBeforeSleepProc(server.el,beforeSleep);
     aeSetAfterSleepProc(server.el,afterSleep);
     aeMain(server.el);
diff --git a/src/server.h b/src/server.h
index 41d767e13..af435b148 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1433,6 +1433,11 @@ struct redisServer {
     int tls_replication;
     int tls_auth_clients;
     redisTLSContextConfig tls_ctx_config;
+    /* cpu affinity */
+    char *server_cpulist; /* cpu affinity list of redis server main/io thread. */
+    char *bio_cpulist; /* cpu affinity list of bio thread. */
+    char *aof_rewrite_cpulist; /* cpu affinity list of aof rewrite process. */
+    char *bgsave_cpulist; /* cpu affinity list of bgsave process. */
 };
 
 typedef struct pubsubPattern {
@@ -1585,6 +1590,7 @@ void exitFromChild(int retcode);
 size_t redisPopcount(void *s, long count);
 void redisSetProcTitle(char *title);
 int redisCommunicateSystemd(const char *sd_notify_msg);
+void redisSetCpuAffinity(const char *cpulist);
 
 /* networking.c -- Networking and Client related operations */
 client *createClient(connection *conn);
diff --git a/src/setcpuaffinity.c b/src/setcpuaffinity.c
new file mode 100644
index 000000000..1a12795cd
--- /dev/null
+++ b/src/setcpuaffinity.c
@@ -0,0 +1,129 @@
+/* ==========================================================================
+ * setproctitle.c - Linux/BSD setcpuaffinity.
+ * --------------------------------------------------------------------------
+ * Copyright (C) 2020  zhenwei pi
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to permit
+ * persons to whom the Software is furnished to do so, subject to the
+ * following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ * ==========================================================================
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#ifdef __linux__
+#include <sched.h>
+#endif
+#ifdef __FreeBSD__
+#include <sys/param.h>
+#include <sys/cpuset.h>
+#endif
+#include "config.h"
+
+#ifdef USE_SETCPUAFFINITY
+static const char *next_token(const char *q,  int sep) {
+    if (q)
+        q = strchr(q, sep);
+    if (q)
+        q++;
+
+    return q;
+}
+
+static int next_num(const char *str, char **end, int *result) {
+    if (!str || *str == '\0' || !isdigit(*str))
+        return -1;
+
+    *result = strtoul(str, end, 10);
+    if (str == *end)
+        return -1;
+
+    return 0;
+}
+
+/* set current thread cpu affinity to cpu list, this function works like
+ * taskset command (actually cpulist parsing logic reference to util-linux).
+ * example of this function: "0,2,3", "0,2-3", "0-20:2". */
+void setcpuaffinity(const char *cpulist) {
+    const char *p, *q;
+    char *end = NULL;
+#ifdef __linux__
+    cpu_set_t cpuset;
+#endif
+#ifdef __FreeBSD__
+    cpuset_t cpuset;
+#endif
+
+    if (!cpulist)
+        return;
+
+    CPU_ZERO(&cpuset);
+
+    q = cpulist;
+    while (p = q, q = next_token(q, ','), p) {
+        int a, b, s;
+        const char *c1, *c2;
+
+        if (next_num(p, &end, &a) != 0)
+            return;
+
+        b = a;
+        s = 1;
+        p = end;
+
+        c1 = next_token(p, '-');
+        c2 = next_token(p, ',');
+
+        if (c1 != NULL && (c2 == NULL || c1 < c2)) {
+            if (next_num(c1, &end, &b) != 0)
+                return;
+
+            c1 = end && *end ? next_token(end, ':') : NULL;
+            if (c1 != NULL && (c2 == NULL || c1 < c2)) {
+                if (next_num(c1, &end, &s) != 0)
+                    return;
+
+                if (s == 0)
+                    return;
+            }
+        }
+
+        if ((a > b))
+            return;
+
+        while (a <= b) {
+            CPU_SET(a, &cpuset);
+            a += s;
+        }
+    }
+
+    if (end && *end)
+        return;
+
+#ifdef __linux__
+    sched_setaffinity(0, sizeof(cpuset), &cpuset);
+#endif
+#ifdef __FreeBSD__
+    cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, sizeof(cpuset), &cpuset);
+#endif
+}
+
+#endif /* USE_SETCPUAFFINITY */
diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl
index cd905084a..b60ca0d48 100644
--- a/tests/unit/introspection.tcl
+++ b/tests/unit/introspection.tcl
@@ -94,6 +94,10 @@ start_server {tags {"introspection"}} {
             slaveof
             bind
             requirepass
+            server_cpulist
+            bio_cpulist
+            aof_rewrite_cpulist
+            bgsave_cpulist
         }
 
         set configs {}

From e697ef6dec901fb230ad81ee9c7f3599f4f20376 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Sat, 2 May 2020 19:20:44 -0400
Subject: [PATCH 0339/1098] Client Side Caching: Add Tracking Prefix Number
 Stats in Server Info

---
 src/server.c   | 2 ++
 src/server.h   | 1 +
 src/tracking.c | 5 +++++
 3 files changed, 8 insertions(+)

diff --git a/src/server.c b/src/server.c
index 659604ef3..1e2feeb58 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4281,6 +4281,7 @@ sds genRedisInfoString(const char *section) {
             "active_defrag_key_misses:%lld\r\n"
             "tracking_total_keys:%lld\r\n"
             "tracking_total_items:%lld\r\n"
+            "tracking_total_prefixes:%lld\r\n"
             "unexpected_error_replies:%lld\r\n",
             server.stat_numconnections,
             server.stat_numcommands,
@@ -4311,6 +4312,7 @@ sds genRedisInfoString(const char *section) {
             server.stat_active_defrag_key_misses,
             (unsigned long long) trackingGetTotalKeys(),
             (unsigned long long) trackingGetTotalItems(),
+            (unsigned long long) trackingGetTotalPrefixes(),
             server.stat_unexpected_error_replies);
     }
 
diff --git a/src/server.h b/src/server.h
index 41d767e13..847ccfb14 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1689,6 +1689,7 @@ void trackingInvalidateKeysOnFlush(int dbid);
 void trackingLimitUsedSlots(void);
 uint64_t trackingGetTotalItems(void);
 uint64_t trackingGetTotalKeys(void);
+uint64_t trackingGetTotalPrefixes(void);
 void trackingBroadcastInvalidationMessages(void);
 
 /* List data type */
diff --git a/src/tracking.c b/src/tracking.c
index 48d231627..a995817e2 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -518,3 +518,8 @@ uint64_t trackingGetTotalKeys(void) {
     if (TrackingTable == NULL) return 0;
     return raxSize(TrackingTable);
 }
+
+uint64_t trackingGetTotalPrefixes(void) {
+    if (PrefixTable == NULL) return 0;
+    return raxSize(PrefixTable);
+}

From e598d9b59ed5592eb7fda764733f9d88c1ade0ee Mon Sep 17 00:00:00 2001
From: Deliang Yang <yang623601391@gmail.com>
Date: Mon, 4 May 2020 01:57:38 +0800
Subject: [PATCH 0340/1098] reformat code

---
 src/ziplist.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ziplist.c b/src/ziplist.c
index ef40d6aa2..ddae0d96f 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -440,7 +440,7 @@ unsigned int zipStorePrevEntryLength(unsigned char *p, unsigned int len) {
     if ((prevlensize) == 1) {                                                  \
         (prevlen) = (ptr)[0];                                                  \
     } else if ((prevlensize) == 5) {                                           \
-        assert(sizeof((prevlen)) == 4);                                    \
+        assert(sizeof((prevlen)) == 4);                                        \
         memcpy(&(prevlen), ((char*)(ptr)) + 1, 4);                             \
         memrev32ifbe(&prevlen);                                                \
     }                                                                          \

From deee2c1ef2249120ae7db0e7523bfad4041b21a6 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 3 May 2020 09:31:50 +0300
Subject: [PATCH 0341/1098] add daily github actions with libc malloc and
 valgrind

* fix memlry leaks with diskless replica short read.
* fix a few timing issues with valgrind runs
* fix issue with valgrind and watchdog schedule signal

about the valgrind WD issue:
the stack trace test in logging.tcl, has issues with valgrind:
==28808== Can't extend stack to 0x1ffeffdb38 during signal delivery for thread 1:
==28808==   too small or bad protection modes

it seems to be some valgrind bug with SA_ONSTACK.
SA_ONSTACK seems unneeded since WD is not recursive (SA_NODEFER was removed),
also, not sure if it's even valid without a call to sigaltstack()
---
 .github/workflows/daily.yml         | 48 +++++++++++++++++++++++
 src/debug.c                         |  2 +-
 src/rdb.c                           | 60 +++++++++++++++++++++--------
 tests/integration/aof.tcl           | 12 ++++++
 tests/integration/replication-3.tcl |  2 +-
 5 files changed, 106 insertions(+), 18 deletions(-)
 create mode 100644 .github/workflows/daily.yml

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
new file mode 100644
index 000000000..b6a9abb68
--- /dev/null
+++ b/.github/workflows/daily.yml
@@ -0,0 +1,48 @@
+name: Daily
+
+on:
+  schedule:
+    - cron: '0 7 * * *'
+
+jobs:
+  test-jemalloc:
+    runs-on: ubuntu-latest
+    timeout-minutes: 1200
+    steps:
+    - uses: actions/checkout@v1
+    - name: make
+      run: make
+    - name: test
+      run: |
+        sudo apt-get install tcl8.5
+        ./runtest --accurate --verbose
+    - name: module api test
+      run: ./runtest-moduleapi --verbose
+
+  test-libc-malloc:
+    runs-on: ubuntu-latest
+    timeout-minutes: 1200
+    steps:
+    - uses: actions/checkout@v1
+    - name: make
+      run: make MALLOC=libc
+    - name: test
+      run: |
+        sudo apt-get install tcl8.5
+        ./runtest --accurate --verbose
+    - name: module api test
+      run: ./runtest-moduleapi --verbose
+
+  test-valgrind:
+    runs-on: ubuntu-latest
+    timeout-minutes: 14400
+    steps:
+    - uses: actions/checkout@v1
+    - name: make
+      run: make valgrind
+    - name: test
+      run: |
+        sudo apt-get install tcl8.5 valgrind -y
+        ./runtest --valgrind --verbose --clients 1
+    - name: module api test
+      run: ./runtest-moduleapi --valgrind --verbose --clients 1
diff --git a/src/debug.c b/src/debug.c
index cbb56cb71..587ff7c5d 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1636,7 +1636,7 @@ void enableWatchdog(int period) {
         /* Watchdog was actually disabled, so we have to setup the signal
          * handler. */
         sigemptyset(&act.sa_mask);
-        act.sa_flags = SA_ONSTACK | SA_SIGINFO;
+        act.sa_flags = SA_SIGINFO;
         act.sa_sigaction = watchdogSignalHandler;
         sigaction(SIGALRM, &act, NULL);
     }
diff --git a/src/rdb.c b/src/rdb.c
index 9f6bf13f1..14c5579e0 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1441,7 +1441,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
 
         /* Load every single element of the list */
         while(len--) {
-            if ((ele = rdbLoadEncodedStringObject(rdb)) == NULL) return NULL;
+            if ((ele = rdbLoadEncodedStringObject(rdb)) == NULL) {
+                decrRefCount(o);
+                return NULL;
+            }
             dec = getDecodedObject(ele);
             size_t len = sdslen(dec->ptr);
             quicklistPushTail(o->ptr, dec->ptr, len);
@@ -1468,8 +1471,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             long long llval;
             sds sdsele;
 
-            if ((sdsele = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL))
-                == NULL) return NULL;
+            if ((sdsele = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) {
+                decrRefCount(o);
+                return NULL;
+            }
 
             if (o->encoding == OBJ_ENCODING_INTSET) {
                 /* Fetch integer value from element. */
@@ -1508,13 +1513,23 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             double score;
             zskiplistNode *znode;
 
-            if ((sdsele = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL))
-                == NULL) return NULL;
+            if ((sdsele = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) {
+                decrRefCount(o);
+                return NULL;
+            }
 
             if (rdbtype == RDB_TYPE_ZSET_2) {
-                if (rdbLoadBinaryDoubleValue(rdb,&score) == -1) return NULL;
+                if (rdbLoadBinaryDoubleValue(rdb,&score) == -1) {
+                    decrRefCount(o);
+                    sdsfree(sdsele);
+                    return NULL;
+                }
             } else {
-                if (rdbLoadDoubleValue(rdb,&score) == -1) return NULL;
+                if (rdbLoadDoubleValue(rdb,&score) == -1) {
+                    decrRefCount(o);
+                    sdsfree(sdsele);
+                    return NULL;
+                }
             }
 
             /* Don't care about integer-encoded strings. */
@@ -1546,10 +1561,15 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
         while (o->encoding == OBJ_ENCODING_ZIPLIST && len > 0) {
             len--;
             /* Load raw strings */
-            if ((field = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL))
-                == NULL) return NULL;
-            if ((value = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL))
-                == NULL) return NULL;
+            if ((field = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) {
+                decrRefCount(o);
+                return NULL;
+            }
+            if ((value = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) {
+                sdsfree(field);
+                decrRefCount(o);
+                return NULL;
+            }
 
             /* Add pair to ziplist */
             o->ptr = ziplistPush(o->ptr, (unsigned char*)field,
@@ -1577,10 +1597,15 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
         while (o->encoding == OBJ_ENCODING_HT && len > 0) {
             len--;
             /* Load encoded strings */
-            if ((field = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL))
-                == NULL) return NULL;
-            if ((value = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL))
-                == NULL) return NULL;
+            if ((field = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) {
+                decrRefCount(o);
+                return NULL;
+            }
+            if ((value = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) {
+                sdsfree(field);
+                decrRefCount(o);
+                return NULL;
+            }
 
             /* Add pair to hash table */
             ret = dictAdd((dict*)o->ptr, field, value);
@@ -1600,7 +1625,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
         while (len--) {
             unsigned char *zl =
                 rdbGenericLoadStringObject(rdb,RDB_LOAD_PLAIN,NULL);
-            if (zl == NULL) return NULL;
+            if (zl == NULL) {
+                decrRefCount(o);
+                return NULL;
+            }
             quicklistAppendZiplist(o->ptr, zl);
         }
     } else if (rdbtype == RDB_TYPE_HASH_ZIPMAP  ||
diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl
index 2734de7f1..b82c87d71 100644
--- a/tests/integration/aof.tcl
+++ b/tests/integration/aof.tcl
@@ -54,6 +54,12 @@ tags {"aof"} {
 
         set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
 
+        wait_for_condition 50 100 {
+            [catch {$client ping} e] == 0
+        } else {
+            fail "Loading DB is taking too much time."
+        }
+
         test "Truncated AOF loaded: we expect foo to be equal to 5" {
             assert {[$client get foo] eq "5"}
         }
@@ -71,6 +77,12 @@ tags {"aof"} {
 
         set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
 
+        wait_for_condition 50 100 {
+            [catch {$client ping} e] == 0
+        } else {
+            fail "Loading DB is taking too much time."
+        }
+
         test "Truncated AOF loaded: we expect foo to be equal to 6 now" {
             assert {[$client get foo] eq "6"}
         }
diff --git a/tests/integration/replication-3.tcl b/tests/integration/replication-3.tcl
index 198e698f2..43eb53538 100644
--- a/tests/integration/replication-3.tcl
+++ b/tests/integration/replication-3.tcl
@@ -118,7 +118,7 @@ start_server {tags {"repl"}} {
             # correctly the RDB file: such file will contain "lua" AUX
             # sections with scripts already in the memory of the master.
 
-            wait_for_condition 50 100 {
+            wait_for_condition 500 100 {
                 [s -1 master_link_status] eq {up}
             } else {
                 fail "Replication not started."

From bce3d08c66a1bf22ea852295a57b92a8024d4a1b Mon Sep 17 00:00:00 2001
From: Guy Benoish <guy.benoish@redislabs.com>
Date: Sun, 3 May 2020 16:49:45 +0300
Subject: [PATCH 0342/1098] XPENDING should not update consumer's seen-time

Same goes for XGROUP DELCONSUMER (But in this case, it doesn't
have any visible effect)
---
 src/blocked.c  |  7 ++++---
 src/rdb.c      |  4 ++--
 src/stream.h   |  7 ++++++-
 src/t_stream.c | 33 ++++++++++++++++++++-------------
 4 files changed, 32 insertions(+), 19 deletions(-)

diff --git a/src/blocked.c b/src/blocked.c
index 045369e93..92f1cee65 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -371,9 +371,10 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) {
                 int noack = 0;
 
                 if (group) {
-                    consumer = streamLookupConsumer(group,
-                               receiver->bpop.xread_consumer->ptr,
-                               1);
+                    consumer =
+                        streamLookupConsumer(group,
+                                             receiver->bpop.xread_consumer->ptr,
+                                             SLC_NONE);
                     noack = receiver->bpop.xread_group_noack;
                 }
 
diff --git a/src/rdb.c b/src/rdb.c
index 9f6bf13f1..def8585ac 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1823,8 +1823,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                     decrRefCount(o);
                     return NULL;
                 }
-                streamConsumer *consumer = streamLookupConsumer(cgroup,cname,
-                                           1);
+                streamConsumer *consumer =
+                    streamLookupConsumer(cgroup,cname,SLC_NONE);
                 sdsfree(cname);
                 consumer->seen_time = rdbLoadMillisecondTime(rdb,RDB_VERSION);
                 if (rioGetReadError(rdb)) {
diff --git a/src/stream.h b/src/stream.h
index b69073994..0d3bf63fc 100644
--- a/src/stream.h
+++ b/src/stream.h
@@ -96,6 +96,11 @@ typedef struct streamPropInfo {
 /* Prototypes of exported APIs. */
 struct client;
 
+/* Flags for streamLookupConsumer */
+#define SLC_NONE      0
+#define SLC_NOCREAT   (1<<0) /* Do not create the consumer if it doesn't exist */
+#define SLC_NOREFRESH (1<<1) /* Do not update consumer's seen-time */
+
 stream *streamNew(void);
 void freeStream(stream *s);
 unsigned long streamLength(const robj *subject);
@@ -105,7 +110,7 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields);
 void streamIteratorGetField(streamIterator *si, unsigned char **fieldptr, unsigned char **valueptr, int64_t *fieldlen, int64_t *valuelen);
 void streamIteratorStop(streamIterator *si);
 streamCG *streamLookupCG(stream *s, sds groupname);
-streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int create);
+streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int flags);
 streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id);
 streamNACK *streamCreateNACK(streamConsumer *consumer);
 void streamDecodeID(void *buf, streamID *id);
diff --git a/src/t_stream.c b/src/t_stream.c
index 5c1b9a523..676ddd9bb 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1570,7 +1570,8 @@ void xreadCommand(client *c) {
             addReplyBulk(c,c->argv[streams_arg+i]);
             streamConsumer *consumer = NULL;
             if (groups) consumer = streamLookupConsumer(groups[i],
-                                                        consumername->ptr,1);
+                                                        consumername->ptr,
+                                                        SLC_NONE);
             streamPropInfo spi = {c->argv[i+streams_arg],groupname};
             int flags = 0;
             if (noack) flags |= STREAM_RWR_NOACK;
@@ -1706,7 +1707,9 @@ streamCG *streamLookupCG(stream *s, sds groupname) {
  * consumer does not exist it is automatically created as a side effect
  * of calling this function, otherwise its last seen time is updated and
  * the existing consumer reference returned. */
-streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int create) {
+streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int flags) {
+    int create = !(flags & SLC_NOCREAT);
+    int refresh = !(flags & SLC_NOREFRESH);
     streamConsumer *consumer = raxFind(cg->consumers,(unsigned char*)name,
                                sdslen(name));
     if (consumer == raxNotFound) {
@@ -1717,7 +1720,7 @@ streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int create) {
         raxInsert(cg->consumers,(unsigned char*)name,sdslen(name),
                   consumer,NULL);
     }
-    consumer->seen_time = mstime();
+    if (refresh) consumer->seen_time = mstime();
     return consumer;
 }
 
@@ -1725,7 +1728,8 @@ streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int create) {
  * may have pending messages: they are removed from the PEL, and the number
  * of pending messages "lost" is returned. */
 uint64_t streamDelConsumer(streamCG *cg, sds name) {
-    streamConsumer *consumer = streamLookupConsumer(cg,name,0);
+    streamConsumer *consumer =
+        streamLookupConsumer(cg,name,SLC_NOCREAT|SLC_NOREFRESH);
     if (consumer == NULL) return 0;
 
     uint64_t retval = raxSize(consumer->pel);
@@ -2068,15 +2072,18 @@ void xpendingCommand(client *c) {
     }
     /* XPENDING <key> <group> <start> <stop> <count> [<consumer>] variant. */
     else {
-        streamConsumer *consumer = consumername ?
-                                streamLookupConsumer(group,consumername->ptr,0):
-                                NULL;
+        streamConsumer *consumer = NULL;
+        if (consumername) {
+            consumer = streamLookupConsumer(group,
+                                            consumername->ptr,
+                                            SLC_NOCREAT|SLC_NOREFRESH);
 
-        /* If a consumer name was mentioned but it does not exist, we can
-         * just return an empty array. */
-        if (consumername && consumer == NULL) {
-            addReplyArrayLen(c,0);
-            return;
+            /* If a consumer name was mentioned but it does not exist, we can
+             * just return an empty array. */
+            if (consumer == NULL) {
+                addReplyArrayLen(c,0);
+                return;
+            }
         }
 
         rax *pel = consumer ? consumer->pel : group->pel;
@@ -2338,7 +2345,7 @@ void xclaimCommand(client *c) {
                 raxRemove(nack->consumer->pel,buf,sizeof(buf),NULL);
             /* Update the consumer and idle time. */
             if (consumer == NULL)
-                consumer = streamLookupConsumer(group,c->argv[3]->ptr,1);
+                consumer = streamLookupConsumer(group,c->argv[3]->ptr,SLC_NONE);
             nack->consumer = consumer;
             nack->delivery_time = deliverytime;
             /* Set the delivery attempts counter if given, otherwise

From 68b5eed41d9b2a58cd5e7c933d5692a3c5895e8c Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 4 May 2020 11:05:48 +0200
Subject: [PATCH 0343/1098] Rework a bit the documentation for CPU pinning.

---
 redis.conf           | 24 +++++++++++++++++-------
 src/setcpuaffinity.c |  2 +-
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/redis.conf b/redis.conf
index 756073a27..8841117fc 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1781,17 +1781,27 @@ rdb-save-incremental-fsync yes
 # the main dictionary scan
 # active-defrag-max-scan-fields 1000
 
-# Redis server/IO threads, bio threads, aof rewrite child process, and bgsave
-# child process cpu affinity list config. syntax of cpu list looks like taskset
-# command. serveral examples:
-# set redis server/io threads to cpu affinity 0,2,4,6
+# It is possible to pin different threads and processes of Redis to specific
+# CPUs in your system, in order to maximize the performances of the server.
+# This is useful both in order to pin different Redis threads in different
+# CPUs, but also in order to make sure that multiple Redis instances running
+# in the same host will be pinned to different CPUs.
+#
+# Normally you can do this using the "taskset" command, however it is also
+# possible to this via Redis configuration directly, both in Linux and FreeBSD.
+#
+# You can pin the server/IO threads, bio threads, aof rewrite child process, and
+# the bgsave child process. The syntax to specify the cpu list is the same as
+# the taskset command:
+#
+# Set redis server/io threads to cpu affinity 0,2,4,6:
 # server_cpulist 0-7:2
 #
-# set bio threads to cpu affinity 1,3
+# Set bio threads to cpu affinity 1,3:
 # bio_cpulist 1,3
 #
-# set aof rewrite child process to cpu affinity 8,9,10,11
+# Set aof rewrite child process to cpu affinity 8,9,10,11:
 # aof_rewrite_cpulist 8-11
 #
-# set bgsave child process to cpu affinity 1,10,11
+# Set bgsave child process to cpu affinity 1,10,11
 # bgsave_cpulist 1,10-11
diff --git a/src/setcpuaffinity.c b/src/setcpuaffinity.c
index 1a12795cd..dcae81c71 100644
--- a/src/setcpuaffinity.c
+++ b/src/setcpuaffinity.c
@@ -1,5 +1,5 @@
 /* ==========================================================================
- * setproctitle.c - Linux/BSD setcpuaffinity.
+ * setcpuaffinity.c - Linux/BSD setcpuaffinity.
  * --------------------------------------------------------------------------
  * Copyright (C) 2020  zhenwei pi
  *

From 6fe2b5236d2e7aeda2a99ae59ed3f1a7d8427a68 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 4 May 2020 11:08:57 +0200
Subject: [PATCH 0344/1098] Fix NetBSD build by fixing redis_set_thread_title()
 support.

See #7188.
---
 src/config.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/config.h b/src/config.h
index a322ce354..6025d4e96 100644
--- a/src/config.h
+++ b/src/config.h
@@ -230,9 +230,12 @@ void setproctitle(const char *fmt, ...);
 #ifdef __linux__
 #define redis_set_thread_title(name) pthread_setname_np(pthread_self(), name)
 #else
-#if (defined __NetBSD__ || defined __FreeBSD__ || defined __OpenBSD__)
+#if (defined __FreeBSD__ || defined __OpenBSD__)
 #include <pthread_np.h>
 #define redis_set_thread_title(name) pthread_set_name_np(pthread_self(), name)
+#elif defined __NetBSD__
+#include <pthread.h>
+#define redis_set_thread_title(name) pthread_setname_np(pthread_self(), name, NULL)
 #else
 #if (defined __APPLE__ && defined(MAC_OS_X_VERSION_10_7))
 int pthread_setname_np(const char *name);

From 4477ee33ac1354b3eecc581e48227d96d14f6c94 Mon Sep 17 00:00:00 2001
From: Muhammad Zahalqa <m@tryfinally.com>
Date: Mon, 4 May 2020 12:36:01 +0300
Subject: [PATCH 0345/1098] Fix compiler warnings on function rev(unsigned
 long)

---
 src/dict.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/dict.c b/src/dict.c
index ac6f8cfde..57619c280 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -766,9 +766,9 @@ dictEntry *dictGetFairRandomKey(dict *d) {
 /* Function to reverse bits. Algorithm from:
  * http://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel */
 static unsigned long rev(unsigned long v) {
-    unsigned long s = 8 * sizeof(v); // bit size; must be power of 2
-    unsigned long mask = ~0;
-    while ((s >>= 1) > 0) {
+    unsigned long s = CHAR_BIT * sizeof(v); // bit size; must be power of 2
+    unsigned long mask = ~0UL;
+    while ((s >>= 1) > 0UL) {
         mask ^= (mask << s);
         v = ((v >> s) & mask) | ((v << s) & ~mask);
     }

From 538c2b2d8dad6a70faf99ba2494b5d561abd7ee0 Mon Sep 17 00:00:00 2001
From: Benjamin Sergeant <bsergean@gmail.com>
Date: Mon, 4 May 2020 08:09:21 -0700
Subject: [PATCH 0346/1098] Add --user argument to redis-benchmark.c (ACL)

---
 src/redis-benchmark.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 2df41580b..77daf981c 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -94,6 +94,7 @@ static struct config {
     sds dbnumstr;
     char *tests;
     char *auth;
+    const char *user;
     int precision;
     int num_threads;
     struct benchmarkThread **threads;
@@ -258,7 +259,10 @@ static redisConfig *getRedisConfig(const char *ip, int port,
 
     if(config.auth) {
         void *authReply = NULL;
-        redisAppendCommand(c, "AUTH %s", config.auth);
+        if (config.user == NULL)
+            redisAppendCommand(c, "AUTH %s", config.auth);
+        else
+            redisAppendCommand(c, "AUTH %s %s", config.user, config.auth);
         if (REDIS_OK != redisGetReply(c, &authReply)) goto fail;
         if (reply) freeReplyObject(reply);
         reply = ((redisReply *) authReply);
@@ -628,7 +632,12 @@ static client createClient(char *cmd, size_t len, client from, int thread_id) {
     c->prefix_pending = 0;
     if (config.auth) {
         char *buf = NULL;
-        int len = redisFormatCommand(&buf, "AUTH %s", config.auth);
+        int len;
+        if (config.user == NULL)
+            len = redisFormatCommand(&buf, "AUTH %s", config.auth);
+        else
+            len = redisFormatCommand(&buf, "AUTH %s %s",
+                                     config.user, config.auth);
         c->obuf = sdscatlen(c->obuf, buf, len);
         free(buf);
         c->prefix_pending++;
@@ -1299,6 +1308,9 @@ int parseOptions(int argc, const char **argv) {
         } else if (!strcmp(argv[i],"-a") ) {
             if (lastarg) goto invalid;
             config.auth = strdup(argv[++i]);
+        } else if (!strcmp(argv[i],"--user")) {
+            if (lastarg) goto invalid;
+            config.user = argv[++i];
         } else if (!strcmp(argv[i],"-d")) {
             if (lastarg) goto invalid;
             config.datasize = atoi(argv[++i]);
@@ -1385,6 +1397,7 @@ usage:
 " -p <port>          Server port (default 6379)\n"
 " -s <socket>        Server socket (overrides host and port)\n"
 " -a <password>      Password for Redis Auth\n"
+" --user <username>  Used to send ACL style 'AUTH username pass'. Needs -a.\n"
 " -c <clients>       Number of parallel connections (default 50)\n"
 " -n <requests>      Total number of requests (default 100000)\n"
 " -d <size>          Data size of SET/GET value in bytes (default 3)\n"

From 1063f1d833754cd35e0180e129a0ae61028677d8 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 5 May 2020 10:09:28 +0200
Subject: [PATCH 0347/1098] Fix CRC64 initialization outside the Redis server
 itself.

---
 src/crc64.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/crc64.c b/src/crc64.c
index 4cbc019f6..9c555e98b 100644
--- a/src/crc64.c
+++ b/src/crc64.c
@@ -29,6 +29,7 @@
 #include "crc64.h"
 #include "crcspeed.h"
 static uint64_t crc64_table[8][256] = {{0}};
+static int crc64_table_initialized = 0;
 
 #define POLY UINT64_C(0xad93d23594c935a9)
 /******************** BEGIN GENERATED PYCRC FUNCTIONS ********************/
@@ -115,10 +116,12 @@ uint64_t _crc64(uint_fast64_t crc, const void *in_data, const uint64_t len) {
 /* Initializes the 16KB lookup tables. */
 void crc64_init(void) {
     crcspeed64native_init(_crc64, crc64_table);
+    crc64_table_initialized = 1;
 }
 
 /* Compute crc64 */
 uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) {
+    if (!crc64_table_initialized) crc64_init();
     return crcspeed64native(crc64_table, crc, (void *) s, l);
 }
 

From 8257a3c1f61ae05b9545446c0ee0a29cdc080e1a Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 5 May 2020 10:20:48 +0200
Subject: [PATCH 0348/1098] Move CRC64 initialization in main().

---
 src/crc64.c  | 3 ---
 src/server.c | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/crc64.c b/src/crc64.c
index 9c555e98b..4cbc019f6 100644
--- a/src/crc64.c
+++ b/src/crc64.c
@@ -29,7 +29,6 @@
 #include "crc64.h"
 #include "crcspeed.h"
 static uint64_t crc64_table[8][256] = {{0}};
-static int crc64_table_initialized = 0;
 
 #define POLY UINT64_C(0xad93d23594c935a9)
 /******************** BEGIN GENERATED PYCRC FUNCTIONS ********************/
@@ -116,12 +115,10 @@ uint64_t _crc64(uint_fast64_t crc, const void *in_data, const uint64_t len) {
 /* Initializes the 16KB lookup tables. */
 void crc64_init(void) {
     crcspeed64native_init(_crc64, crc64_table);
-    crc64_table_initialized = 1;
 }
 
 /* Compute crc64 */
 uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) {
-    if (!crc64_table_initialized) crc64_init();
     return crcspeed64native(crc64_table, crc, (void *) s, l);
 }
 
diff --git a/src/server.c b/src/server.c
index f2f5c8b3f..416dcd747 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2899,7 +2899,6 @@ void initServer(void) {
     scriptingInit(1);
     slowlogInit();
     latencyMonitorInit();
-    crc64_init();
 }
 
 /* Some steps in server initialization need to be done last (after modules
@@ -4963,6 +4962,7 @@ int main(int argc, char **argv) {
     zmalloc_set_oom_handler(redisOutOfMemoryHandler);
     srand(time(NULL)^getpid());
     gettimeofday(&tv,NULL);
+    crc64_init();
 
     uint8_t hashseed[16];
     getRandomBytes(hashseed,sizeof(hashseed));

From 5fdc5e5c25451c2afc30c2a8aa16a1491c75015e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 5 May 2020 10:27:27 +0200
Subject: [PATCH 0349/1098] Drop not needed part from #7194.

---
 src/dict.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/dict.c b/src/dict.c
index 57619c280..13c185253 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -768,7 +768,7 @@ dictEntry *dictGetFairRandomKey(dict *d) {
 static unsigned long rev(unsigned long v) {
     unsigned long s = CHAR_BIT * sizeof(v); // bit size; must be power of 2
     unsigned long mask = ~0UL;
-    while ((s >>= 1) > 0UL) {
+    while ((s >>= 1) > 0) {
         mask ^= (mask << s);
         v = ((v >> s) & mask) | ((v << s) & ~mask);
     }

From 1f246bfcc8fc2e55f05dd863f286a2057ad29ef7 Mon Sep 17 00:00:00 2001
From: Titouan Christophe <titouan.christophe@railnova.eu>
Date: Mon, 4 May 2020 14:11:43 +0200
Subject: [PATCH 0350/1098] make struct user anonymous (only typedefed)

This works because this struct is never referenced by its name,
but always by its type.

This prevents a conflict with struct user from <sys/user.h>
when compiling against uclibc.

Signed-off-by: Titouan Christophe <titouan.christophe@railnova.eu>
---
 src/server.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.h b/src/server.h
index 9c2b761c4..59cf1370e 100644
--- a/src/server.h
+++ b/src/server.h
@@ -732,7 +732,7 @@ typedef struct readyList {
                                            no AUTH is needed, and every
                                            connection is immediately
                                            authenticated. */
-typedef struct user {
+typedef struct {
     sds name;       /* The username as an SDS string. */
     uint64_t flags; /* See USER_FLAG_* */
 

From e49d97298a91827434312b41693a83c1a2969fa0 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 5 May 2020 13:18:53 +0200
Subject: [PATCH 0351/1098] Test: --dont-clean should do first cleanup.

---
 tests/test_helper.tcl | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 3eee1aeb7..de0a64728 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -216,9 +216,6 @@ proc run_solo {name code} {
 }
 
 proc cleanup {} {
-    if {$::dont_clean} {
-        return
-    }
     if {!$::quiet} {puts -nonewline "Cleanup: may take some time... "}
     flush stdout
     catch {exec rm -rf {*}[glob tests/tmp/redis.conf.*]}
@@ -456,11 +453,11 @@ proc the_end {} {
         foreach failed $::failed_tests {
             puts "*** $failed"
         }
-        cleanup
+        if {!$::dont_clean} cleanup
         exit 1
     } else {
         puts "\n[colorstr bold-white {\o/}] [colorstr bold-green {All tests passed without errors!}]\n"
-        cleanup
+        if {!$::dont_clean} cleanup
         exit 0
     }
 }

From 51ab31dbabd925550d2c41f15c53832b4ad8982b Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 5 May 2020 13:40:33 +0200
Subject: [PATCH 0352/1098] Dump recent backlog on master query generating
 errors.

---
 src/networking.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index fd1159e06..75c0c16b1 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -393,6 +393,35 @@ void addReplyErrorLength(client *c, const char *s, size_t len) {
         serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error "
                              "to its %s: '%s' after processing the command "
                              "'%s'", from, to, s, cmdname);
+        if (ctype == CLIENT_TYPE_MASTER && server.repl_backlog &&
+            server.repl_backlog_histlen > 0)
+        {
+            long long dumplen = 256;
+            if (server.repl_backlog_histlen < dumplen)
+                dumplen = server.repl_backlog_histlen;
+
+            /* Identify the first byte to dump. */
+            long long idx =
+              (server.repl_backlog_idx + (server.repl_backlog_size - dumplen)) %
+               server.repl_backlog_size;
+
+            /* Scan the circular buffer to collect 'dumplen' bytes. */
+            sds dump = sdsempty();
+            while(dumplen) {
+                long long thislen =
+                    ((server.repl_backlog_size - idx) < dumplen) ?
+                    (server.repl_backlog_size - idx) : dumplen;
+
+                dump = sdscatrepr(dump,server.repl_backlog+idx,thislen);
+                dumplen -= thislen;
+                idx = 0;
+            }
+
+            /* Finally log such bytes: this is vital debugging info to
+             * understand what happened. */
+            serverLog(LL_WARNING,"Latest backlog is: '%s'", dump);
+            sdsfree(dump);
+        }
         server.stat_unexpected_error_replies++;
     }
 }

From 373a3384acb83e98f9a806b46e1dbb3a43d5f3a5 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 5 May 2020 15:50:00 +0200
Subject: [PATCH 0353/1098] Don't propagate spurious MULTI on DEBUG LOADAOF.

---
 src/multi.c  | 5 ++++-
 src/server.c | 4 ++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/multi.c b/src/multi.c
index cbbd2c513..a331a6240 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -172,7 +172,10 @@ void execCommand(client *c) {
          * This way we'll deliver the MULTI/..../EXEC block as a whole and
          * both the AOF and the replication link will have the same consistency
          * and atomicity guarantees. */
-        if (!must_propagate && !(c->cmd->flags & (CMD_READONLY|CMD_ADMIN))) {
+        if (!must_propagate &&
+            !server.loading &&
+            !(c->cmd->flags & (CMD_READONLY|CMD_ADMIN)))
+        {
             execCommandPropagateMulti(c);
             must_propagate = 1;
         }
diff --git a/src/server.c b/src/server.c
index 416dcd747..e2b4b6f3d 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3204,8 +3204,8 @@ void call(client *c, int flags) {
 
     server.fixed_time_expire++;
 
-    /* Sent the command to clients in MONITOR mode, only if the commands are
-     * not generated from reading an AOF. */
+    /* Send the command to clients in MONITOR mode if applicable.
+     * Administrative commands are considered too dangerous to be shown. */
     if (listLength(server.monitors) &&
         !server.loading &&
         !(c->cmd->flags & (CMD_SKIP_MONITOR|CMD_ADMIN)))

From 5508c228e76d0e832321148f0251478cb6d5308e Mon Sep 17 00:00:00 2001
From: Brad Dunbar <dunbarb2@gmail.com>
Date: Tue, 5 May 2020 09:57:01 -0400
Subject: [PATCH 0354/1098] Remove unreachable branch.

---
 src/util.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/util.c b/src/util.c
index d173f776f..537c9313c 100644
--- a/src/util.c
+++ b/src/util.c
@@ -67,8 +67,6 @@ int stringmatchlen(const char *pattern, int patternLen,
             return 0; /* no match */
             break;
         case '?':
-            if (stringLen == 0)
-                return 0; /* no match */
             string++;
             stringLen--;
             break;

From 150ba0e1ff0a24d2136f893ede550d19e78a96a0 Mon Sep 17 00:00:00 2001
From: ShooterIT <wangyuancode@163.com>
Date: Tue, 5 May 2020 23:09:45 +0800
Subject: [PATCH 0355/1098] Redis Benchmark: Fix coredump because of double
 free

---
 src/redis-benchmark.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 2df41580b..8e32d4f3c 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -275,7 +275,7 @@ static redisConfig *getRedisConfig(const char *ip, int port,
     for (; i < 2; i++) {
         int res = redisGetReply(c, &r);
         if (reply) freeReplyObject(reply);
-        reply = ((redisReply *) r);
+        reply = res == REDIS_OK ? ((redisReply *) r) : NULL;
         if (res != REDIS_OK || !r) goto fail;
         if (reply->type == REDIS_REPLY_ERROR) {
             fprintf(stderr, "ERROR: %s\n", reply->str);

From 24173d11bf5962b00747333136d4b4324b8cd81a Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Tue, 5 May 2020 23:35:08 -0400
Subject: [PATCH 0356/1098] add include guard for lolwut.h

---
 src/lolwut.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/lolwut.h b/src/lolwut.h
index 38c0de423..682d00531 100644
--- a/src/lolwut.h
+++ b/src/lolwut.h
@@ -34,6 +34,10 @@
 /* This represents a very simple generic canvas in order to draw stuff.
  * It's up to each LOLWUT versions to translate what they draw to the
  * screen, depending on the result to accomplish. */
+
+#ifndef __LOLWUT_H
+#define __LOLWUT_H
+
 typedef struct lwCanvas {
     int width;
     int height;
@@ -47,3 +51,5 @@ void lwDrawPixel(lwCanvas *canvas, int x, int y, int color);
 int lwGetPixel(lwCanvas *canvas, int x, int y);
 void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color);
 void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle, int color);
+
+#endif

From 9cfb1c88d65c4358e9b0cf84e255d24f11cc6cf7 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Wed, 6 May 2020 01:07:17 -0400
Subject: [PATCH 0357/1098] add jemalloc-bg-thread config in redis conf

---
 redis.conf | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/redis.conf b/redis.conf
index 8841117fc..a469d8b61 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1781,6 +1781,9 @@ rdb-save-incremental-fsync yes
 # the main dictionary scan
 # active-defrag-max-scan-fields 1000
 
+# Jemalloc background thread for purging will be enabled by default
+jemalloc-bg-thread yes
+
 # It is possible to pin different threads and processes of Redis to specific
 # CPUs in your system, in order to maximize the performances of the server.
 # This is useful both in order to pin different Redis threads in different

From e17f9311c8ee18996e4b932a1284e4cfe29c05b4 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 6 May 2020 16:18:21 +0200
Subject: [PATCH 0358/1098] stringmatchlen() should not expect null terminated
 strings.

---
 src/util.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/util.c b/src/util.c
index 537c9313c..0d48f5701 100644
--- a/src/util.c
+++ b/src/util.c
@@ -51,7 +51,7 @@ int stringmatchlen(const char *pattern, int patternLen,
     while(patternLen && stringLen) {
         switch(pattern[0]) {
         case '*':
-            while (pattern[1] == '*') {
+            while (patternLen && pattern[1] == '*') {
                 pattern++;
                 patternLen--;
             }
@@ -94,7 +94,7 @@ int stringmatchlen(const char *pattern, int patternLen,
                     pattern--;
                     patternLen++;
                     break;
-                } else if (pattern[1] == '-' && patternLen >= 3) {
+                } else if (patternLen >= 3 && pattern[1] == '-') {
                     int start = pattern[0];
                     int end = pattern[2];
                     int c = string[0];

From 4afa243ff18a8863650906fd4b5f9dce0d5ba106 Mon Sep 17 00:00:00 2001
From: ShooterIT <wangyuancode@163.com>
Date: Thu, 7 May 2020 11:04:08 +0800
Subject: [PATCH 0359/1098] Use dictSize to get the size of dict in dict.c

---
 src/dict.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/dict.c b/src/dict.c
index 13c185253..45aab66f9 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -478,7 +478,7 @@ dictEntry *dictFind(dict *d, const void *key)
     dictEntry *he;
     uint64_t h, idx, table;
 
-    if (d->ht[0].used + d->ht[1].used == 0) return NULL; /* dict is empty */
+    if (dictSize(d) == 0) return NULL; /* dict is empty */
     if (dictIsRehashing(d)) _dictRehashStep(d);
     h = dictHashKey(d, key);
     for (table = 0; table <= 1; table++) {
@@ -1044,7 +1044,7 @@ dictEntry **dictFindEntryRefByPtrAndHash(dict *d, const void *oldptr, uint64_t h
     dictEntry *he, **heref;
     unsigned long idx, table;
 
-    if (d->ht[0].used + d->ht[1].used == 0) return NULL; /* dict is empty */
+    if (dictSize(d) == 0) return NULL; /* dict is empty */
     for (table = 0; table <= 1; table++) {
         idx = hash & d->ht[table].sizemask;
         heref = &d->ht[table].table[idx];

From 00a3bc435915c6f3526a135af40ebbb46a942005 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 8 May 2020 11:38:07 +0200
Subject: [PATCH 0360/1098] Cluster: introduce data_received field.

We want to send pings and pongs at specific intervals, since our packets
also contain information about the configuration of the cluster and are
used for gossip. However since our cluster bus is used in a mixed way
for data (such as Pub/Sub or modules cluster messages) and metadata,
sometimes a very busy channel may delay the reception of pong packets.
So after discussing it in #7216, this commit introduces a new field that
is not exposed in the cluster, is only an internal information about
the last time we received any data from a given node: we use this field
in order to avoid detecting failures, claiming data reception of new
data from the node is a proof of liveness.
---
 src/cluster.c | 36 ++++++++++++++++++++++++++----------
 src/cluster.h |  1 +
 2 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index d8f83f08d..78524bd34 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -749,6 +749,7 @@ clusterNode *createClusterNode(char *nodename, int flags) {
     node->slaves = NULL;
     node->slaveof = NULL;
     node->ping_sent = node->pong_received = 0;
+    node->data_received = 0;
     node->fail_time = 0;
     node->link = NULL;
     memset(node->ip,0,sizeof(node->ip));
@@ -1678,6 +1679,7 @@ int clusterProcessPacket(clusterLink *link) {
     clusterMsg *hdr = (clusterMsg*) link->rcvbuf;
     uint32_t totlen = ntohl(hdr->totlen);
     uint16_t type = ntohs(hdr->type);
+    mstime_t now = mstime();
 
     if (type < CLUSTERMSG_TYPE_COUNT)
         server.cluster->stats_bus_messages_received[type]++;
@@ -1741,6 +1743,13 @@ int clusterProcessPacket(clusterLink *link) {
 
     /* Check if the sender is a known node. */
     sender = clusterLookupNode(hdr->sender);
+
+    /* Update the last time we saw any data from this node. We
+     * use this in order to avoid detecting a timeout from a node that
+     * is just sending a lot of data in the cluster bus, for instance
+     * because of Pub/Sub. */
+    if (sender) sender->data_received = now;
+
     if (sender && !nodeInHandshake(sender)) {
         /* Update our curretEpoch if we see a newer epoch in the cluster. */
         senderCurrentEpoch = ntohu64(hdr->currentEpoch);
@@ -1755,7 +1764,7 @@ int clusterProcessPacket(clusterLink *link) {
         }
         /* Update the replication offset info for this node. */
         sender->repl_offset = ntohu64(hdr->offset);
-        sender->repl_offset_time = mstime();
+        sender->repl_offset_time = now;
         /* If we are a slave performing a manual failover and our master
          * sent its offset while already paused, populate the MF state. */
         if (server.cluster->mf_end &&
@@ -1869,7 +1878,7 @@ int clusterProcessPacket(clusterLink *link) {
                  * address. */
                 serverLog(LL_DEBUG,"PONG contains mismatching sender ID. About node %.40s added %d ms ago, having flags %d",
                     link->node->name,
-                    (int)(mstime()-(link->node->ctime)),
+                    (int)(now-(link->node->ctime)),
                     link->node->flags);
                 link->node->flags |= CLUSTER_NODE_NOADDR;
                 link->node->ip[0] = '\0';
@@ -1904,7 +1913,7 @@ int clusterProcessPacket(clusterLink *link) {
 
         /* Update our info about the node */
         if (link->node && type == CLUSTERMSG_TYPE_PONG) {
-            link->node->pong_received = mstime();
+            link->node->pong_received = now;
             link->node->ping_sent = 0;
 
             /* The PFAIL condition can be reversed without external
@@ -2051,7 +2060,7 @@ int clusterProcessPacket(clusterLink *link) {
                     "FAIL message received from %.40s about %.40s",
                     hdr->sender, hdr->data.fail.about.nodename);
                 failing->flags |= CLUSTER_NODE_FAIL;
-                failing->fail_time = mstime();
+                failing->fail_time = now;
                 failing->flags &= ~CLUSTER_NODE_PFAIL;
                 clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|
                                      CLUSTER_TODO_UPDATE_STATE);
@@ -2104,9 +2113,9 @@ int clusterProcessPacket(clusterLink *link) {
         /* Manual failover requested from slaves. Initialize the state
          * accordingly. */
         resetManualFailover();
-        server.cluster->mf_end = mstime() + CLUSTER_MF_TIMEOUT;
+        server.cluster->mf_end = now + CLUSTER_MF_TIMEOUT;
         server.cluster->mf_slave = sender;
-        pauseClients(mstime()+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT));
+        pauseClients(now+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT));
         serverLog(LL_WARNING,"Manual failover requested by replica %.40s.",
             sender->name);
     } else if (type == CLUSTERMSG_TYPE_UPDATE) {
@@ -3529,7 +3538,6 @@ void clusterCron(void) {
     while((de = dictNext(di)) != NULL) {
         clusterNode *node = dictGetVal(de);
         now = mstime(); /* Use an updated time at every iteration. */
-        mstime_t delay;
 
         if (node->flags &
             (CLUSTER_NODE_MYSELF|CLUSTER_NODE_NOADDR|CLUSTER_NODE_HANDSHAKE))
@@ -3553,7 +3561,7 @@ void clusterCron(void) {
                 this_slaves = okslaves;
         }
 
-        /* If we are waiting for the PONG more than half the cluster
+        /* If we are not receiving any data for more than half the cluster
          * timeout, reconnect the link: maybe there is a connection
          * issue even if the node is alive. */
         if (node->link && /* is connected */
@@ -3562,7 +3570,9 @@ void clusterCron(void) {
             node->ping_sent && /* we already sent a ping */
             node->pong_received < node->ping_sent && /* still waiting pong */
             /* and we are waiting for the pong more than timeout/2 */
-            now - node->ping_sent > server.cluster_node_timeout/2)
+            now - node->ping_sent > server.cluster_node_timeout/2 &&
+            /* and in such interval we are not seeing any traffic at all. */
+            now - node->data_received > server.cluster_node_timeout/2)
         {
             /* Disconnect the link, it will be reconnected automatically. */
             freeClusterLink(node->link);
@@ -3597,7 +3607,13 @@ void clusterCron(void) {
         /* Compute the delay of the PONG. Note that if we already received
          * the PONG, then node->ping_sent is zero, so can't reach this
          * code at all. */
-        delay = now - node->ping_sent;
+        mstime_t delay = now - node->ping_sent;
+
+        /* We consider every incoming data as proof of liveness, since
+         * our cluster bus link is also used for data: under heavy data
+         * load pong delays are possible. */
+        mstime_t data_delay = now - node->data_received;
+        if (data_delay < delay) delay = data_delay;
 
         if (delay > server.cluster_node_timeout) {
             /* Timeout reached. Set the node as possibly failing if it is
diff --git a/src/cluster.h b/src/cluster.h
index 35fc0cbfa..d3af4a355 100644
--- a/src/cluster.h
+++ b/src/cluster.h
@@ -124,6 +124,7 @@ typedef struct clusterNode {
                                     tables. */
     mstime_t ping_sent;      /* Unix time we sent latest ping */
     mstime_t pong_received;  /* Unix time we received the pong */
+    mstime_t data_received;  /* Unix time we received any data */
     mstime_t fail_time;      /* Unix time when FAIL flag was set */
     mstime_t voted_time;     /* Last time we voted for a slave of this master */
     mstime_t repl_offset_time;  /* Unix time we received offset for this node */

From 79de9d6c787aa28c4fb2bc49b34bb81526ae7f91 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 8 May 2020 18:10:16 +0200
Subject: [PATCH 0361/1098] Cluster: refactor ping/data delay handling.

---
 src/cluster.c | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 78524bd34..50c79f4f6 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -3564,15 +3564,17 @@ void clusterCron(void) {
         /* If we are not receiving any data for more than half the cluster
          * timeout, reconnect the link: maybe there is a connection
          * issue even if the node is alive. */
+        mstime_t ping_delay = now - node->ping_sent;
+        mstime_t data_delay = now - node->data_received;
         if (node->link && /* is connected */
             now - node->link->ctime >
             server.cluster_node_timeout && /* was not already reconnected */
             node->ping_sent && /* we already sent a ping */
             node->pong_received < node->ping_sent && /* still waiting pong */
             /* and we are waiting for the pong more than timeout/2 */
-            now - node->ping_sent > server.cluster_node_timeout/2 &&
+            ping_delay > server.cluster_node_timeout/2 &&
             /* and in such interval we are not seeing any traffic at all. */
-            now - node->data_received > server.cluster_node_timeout/2)
+            data_delay > server.cluster_node_timeout/2)
         {
             /* Disconnect the link, it will be reconnected automatically. */
             freeClusterLink(node->link);
@@ -3604,18 +3606,18 @@ void clusterCron(void) {
         /* Check only if we have an active ping for this instance. */
         if (node->ping_sent == 0) continue;
 
-        /* Compute the delay of the PONG. Note that if we already received
-         * the PONG, then node->ping_sent is zero, so can't reach this
-         * code at all. */
-        mstime_t delay = now - node->ping_sent;
-
-        /* We consider every incoming data as proof of liveness, since
+        /* Check if this node looks unreachable.
+         * Note that if we already received the PONG, then node->ping_sent
+         * is zero, so can't reach this code at all, so we don't risk of
+         * checking for a PONG delay if we didn't sent the PING.
+         *
+         * We also consider every incoming data as proof of liveness, since
          * our cluster bus link is also used for data: under heavy data
          * load pong delays are possible. */
-        mstime_t data_delay = now - node->data_received;
-        if (data_delay < delay) delay = data_delay;
+        mstime_t node_delay = (ping_delay < data_delay) ? ping_delay :
+                                                          data_delay;
 
-        if (delay > server.cluster_node_timeout) {
+        if (node_delay > server.cluster_node_timeout) {
             /* Timeout reached. Set the node as possibly failing if it is
              * not already in this state. */
             if (!(node->flags & (CLUSTER_NODE_PFAIL|CLUSTER_NODE_FAIL))) {

From 4387ba6a17887d23c1919619f517b521eacbb665 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Sat, 9 May 2020 11:12:51 +0200
Subject: [PATCH 0362/1098] Cluster: clarify we always resolve the sender.

---
 src/cluster.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/cluster.c b/src/cluster.c
index 50c79f4f6..a2fab323a 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -1741,7 +1741,9 @@ int clusterProcessPacket(clusterLink *link) {
         if (totlen != explen) return 1;
     }
 
-    /* Check if the sender is a known node. */
+    /* Check if the sender is a known node. Note that for incoming connections
+     * we don't store link->node information, but resolve the node by the
+     * ID in the header each time in the current implementation. */
     sender = clusterLookupNode(hdr->sender);
 
     /* Update the last time we saw any data from this node. We

From 4d1178cc2447c2b993ed565f62afdf7eb8fe367b Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 10 May 2020 17:38:04 +0300
Subject: [PATCH 0363/1098] TLS: Fix test failures on recent Debian/Ubuntu.

Seems like on some systems choosing specific TLS v1/v1.1 versions no
longer works as expected. Test is reduced for v1.2 now which is still
good enough to test the mechansim, and matters most anyway.
---
 tests/unit/tls.tcl | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/tests/unit/tls.tcl b/tests/unit/tls.tcl
index 950f65557..2b04590cd 100644
--- a/tests/unit/tls.tcl
+++ b/tests/unit/tls.tcl
@@ -25,26 +25,6 @@ start_server {tags {"tls"}} {
         }
 
         test {TLS: Verify tls-protocols behaves as expected} {
-            r CONFIG SET tls-protocols TLSv1
-
-            set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1 0}]
-            catch {$s PING} e
-            assert_match {*I/O error*} $e
-
-            set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1 1}]
-            catch {$s PING} e
-            assert_match {PONG} $e
-
-            r CONFIG SET tls-protocols TLSv1.1
-
-            set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1.1 0}]
-            catch {$s PING} e
-            assert_match {*I/O error*} $e
-
-            set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1.1 1}]
-            catch {$s PING} e
-            assert_match {PONG} $e
-
             r CONFIG SET tls-protocols TLSv1.2
 
             set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1.2 0}]

From 450f0d724834cf5d4d32d6d8caae7e4167c184b8 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 10 May 2020 17:35:27 +0300
Subject: [PATCH 0364/1098] TLS: Add crypto locks for older OpenSSL support.

This is really required only for older OpenSSL versions.

Also, at the moment Redis does not use OpenSSL from multiple threads so
this will only be useful if modules end up doing that.
---
 src/tls.c | 45 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/src/tls.c b/src/tls.c
index c18aafebe..cfd6ea234 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -93,11 +93,56 @@ static int parseProtocolsConfig(const char *str) {
  * served to the reader yet. */
 static list *pending_list = NULL;
 
+/**
+ * OpenSSL global initialization and locking handling callbacks.
+ * Note that this is only required for OpenSSL < 1.1.0.
+ */
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+#define USE_CRYPTO_LOCKS
+#endif
+
+#ifdef USE_CRYPTO_LOCKS
+
+static pthread_mutex_t *openssl_locks;
+
+static void sslLockingCallback(int mode, int lock_id, const char *f, int line) {
+    pthread_mutex_t *mt = openssl_locks + lock_id;
+
+    if (mode & CRYPTO_LOCK) {
+        pthread_mutex_lock(mt);
+    } else {
+        pthread_mutex_unlock(mt);
+    }
+
+    (void)f;
+    (void)line;
+}
+
+static void initCryptoLocks(void) {
+    unsigned i, nlocks;
+    if (CRYPTO_get_locking_callback() != NULL) {
+        /* Someone already set the callback before us. Don't destroy it! */
+        return;
+    }
+    nlocks = CRYPTO_num_locks();
+    openssl_locks = zmalloc(sizeof(*openssl_locks) * nlocks);
+    for (i = 0; i < nlocks; i++) {
+        pthread_mutex_init(openssl_locks + i, NULL);
+    }
+    CRYPTO_set_locking_callback(sslLockingCallback);
+}
+#endif /* USE_CRYPTO_LOCKS */
+
 void tlsInit(void) {
     ERR_load_crypto_strings();
     SSL_load_error_strings();
     SSL_library_init();
 
+#ifdef USE_CRYPTO_LOCKS
+    initCryptoLocks();
+#endif
+
     if (!RAND_poll()) {
         serverLog(LL_WARNING, "OpenSSL: Failed to seed random number generator.");
     }

From bc9efb577bca53b6339a406d2a957312371e5e24 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Mon, 11 May 2020 13:13:51 +0800
Subject: [PATCH 0365/1098] Add a test to prove current tcl cluster client can
 not handle keys with hash tag.

---
 tests/cluster/tests/15-cluster-slots.tcl | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/cluster/tests/15-cluster-slots.tcl b/tests/cluster/tests/15-cluster-slots.tcl
index dc9938ef6..1b33c57bd 100644
--- a/tests/cluster/tests/15-cluster-slots.tcl
+++ b/tests/cluster/tests/15-cluster-slots.tcl
@@ -41,4 +41,10 @@ test "client do not break when cluster slot" {
     if { [catch {R 0 cluster slots}] } {
         fail "output overflow when cluster slots"
     }
-}
\ No newline at end of file
+}
+
+test "client can handle keys with hash tag" {
+    set cluster [redis_cluster 127.0.0.1:[get_instance_attrib redis 0 port]]
+    $cluster set foo{tag} bar
+    $cluster close
+}

From 70c4851f2cb096566536d3cba9f1ff253ca09a67 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Mon, 11 May 2020 13:07:05 +0800
Subject: [PATCH 0366/1098] Handle keys with hash tag when computing hash slot
 using tcl cluster client.

---
 tests/support/cluster.tcl | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/tests/support/cluster.tcl b/tests/support/cluster.tcl
index 74587e1f7..64b079ff8 100644
--- a/tests/support/cluster.tcl
+++ b/tests/support/cluster.tcl
@@ -286,8 +286,29 @@ proc ::redis_cluster::crc16 {s} {
 # Hash a single key returning the slot it belongs to, Implemented hash
 # tags as described in the Redis Cluster specification.
 proc ::redis_cluster::hash {key} {
-    # TODO: Handle hash slots.
-    expr {[::redis_cluster::crc16 $key] & 16383}
+    set keylen [string length $key]
+    set s {}
+    set e {}
+    for {set s 0} {$s < $keylen} {incr s} {
+        if {[string index $key $s] eq "\{"} break
+    }
+
+    if {[expr {$s == $keylen}]} {
+        set res [expr {[crc16 $key] & 16383}]
+        return $res
+    }
+
+    for {set e [expr {$s+1}]} {$e < $keylen} {incr e} {
+        if {[string index $key $e] == "\}"} break
+    }
+
+    if {$e == $keylen || $e == [expr {$s+1}]} {
+        set res [expr {[crc16 $key] & 16383}]
+        return $res
+    }
+
+    set key_sub [string range $key [expr {$s+1}] [expr {$e-1}]]
+    return [expr {[crc16 $key_sub] & 16383}]
 }
 
 # Return the slot the specified keys hash to.

From 905e28ee87eb0bbea448ec28c11dc74991359bf5 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 10 May 2020 19:13:47 +0300
Subject: [PATCH 0367/1098] fix redis 6.0 not freeing closed connections during
 loading.

This bug was introduced by a recent change in which readQueryFromClient
is using freeClientAsync, and despite the fact that now
freeClientsInAsyncFreeQueue is in beforeSleep, that's not enough since
it's not called during loading in processEventsWhileBlocked.
furthermore, afterSleep was called in that case but beforeSleep wasn't.

This bug also caused slowness sine the level-triggered mode of epoll
kept signaling these connections as readable causing us to keep doing
connRead again and again for ll of these, which keep accumulating.

now both before and after sleep are called, but not all of their actions
are performed during loading, some are only reserved for the main loop.

fixes issue #7215
---
 src/ae.c                  |   5 +-
 src/server.c              | 124 ++++++++++++++++++++++----------------
 tests/integration/rdb.tcl |  52 ++++++++++++++++
 3 files changed, 128 insertions(+), 53 deletions(-)

diff --git a/src/ae.c b/src/ae.c
index 1bf6cbfbf..7aa204250 100644
--- a/src/ae.c
+++ b/src/ae.c
@@ -428,6 +428,9 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
             tvp = &tv;
         }
 
+        if (eventLoop->beforesleep != NULL)
+            eventLoop->beforesleep(eventLoop);
+
         /* Call the multiplexing API, will return only on timeout or when
          * some event fires. */
         numevents = aeApiPoll(eventLoop, tvp);
@@ -522,8 +525,6 @@ int aeWait(int fd, int mask, long long milliseconds) {
 void aeMain(aeEventLoop *eventLoop) {
     eventLoop->stop = 0;
     while (!eventLoop->stop) {
-        if (eventLoop->beforesleep != NULL)
-            eventLoop->beforesleep(eventLoop);
         aeProcessEvents(eventLoop, AE_ALL_EVENTS|AE_CALL_AFTER_SLEEP);
     }
 }
diff --git a/src/server.c b/src/server.c
index e2b4b6f3d..fd75f25d6 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2081,14 +2081,26 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
     return 1000/server.hz;
 }
 
+extern int ProcessingEventsWhileBlocked;
+
 /* This function gets called every time Redis is entering the
  * main loop of the event driven library, that is, before to sleep
- * for ready file descriptors. */
+ * for ready file descriptors.
+ * Note: This function is (currently) called from two functions:
+ * 1. aeMain - The main server loop
+ * 2. processEventsWhileBlocked - Process clients during RDB/AOF load
+ * If it was called from processEventsWhileBlocked we don't want
+ * to perform all actions (For example, we don't want to expire
+ * keys), but we do need to perform some actions.
+ * The most important is freeClientsInAsyncFreeQueue but we also
+ * call some other low-risk functions. */
 void beforeSleep(struct aeEventLoop *eventLoop) {
     UNUSED(eventLoop);
 
-    /* Handle precise timeouts of blocked clients. */
-    handleBlockedClientsTimeout();
+    if (!ProcessingEventsWhileBlocked) {
+        /* Handle precise timeouts of blocked clients. */
+        handleBlockedClientsTimeout();
+    }
 
     /* We should handle pending reads clients ASAP after event loop. */
     handleClientsWithPendingReadsUsingThreads();
@@ -2099,65 +2111,69 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     /* If tls still has pending unread data don't sleep at all. */
     aeSetDontWait(server.el, tlsHasPendingData());
 
-    /* Call the Redis Cluster before sleep function. Note that this function
-     * may change the state of Redis Cluster (from ok to fail or vice versa),
-     * so it's a good idea to call it before serving the unblocked clients
-     * later in this function. */
-    if (server.cluster_enabled) clusterBeforeSleep();
+    if (!ProcessingEventsWhileBlocked) {
+        /* Call the Redis Cluster before sleep function. Note that this function
+         * may change the state of Redis Cluster (from ok to fail or vice versa),
+         * so it's a good idea to call it before serving the unblocked clients
+         * later in this function. */
+        if (server.cluster_enabled) clusterBeforeSleep();
 
-    /* Run a fast expire cycle (the called function will return
-     * ASAP if a fast cycle is not needed). */
-    if (server.active_expire_enabled && server.masterhost == NULL)
-        activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST);
+        /* Run a fast expire cycle (the called function will return
+         * ASAP if a fast cycle is not needed). */
+        if (server.active_expire_enabled && server.masterhost == NULL)
+            activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST);
 
-    /* Unblock all the clients blocked for synchronous replication
-     * in WAIT. */
-    if (listLength(server.clients_waiting_acks))
-        processClientsWaitingReplicas();
+        /* Unblock all the clients blocked for synchronous replication
+         * in WAIT. */
+        if (listLength(server.clients_waiting_acks))
+            processClientsWaitingReplicas();
 
-    /* Check if there are clients unblocked by modules that implement
-     * blocking commands. */
-    if (moduleCount()) moduleHandleBlockedClients();
+        /* Check if there are clients unblocked by modules that implement
+         * blocking commands. */
+        if (moduleCount()) moduleHandleBlockedClients();
 
-    /* Try to process pending commands for clients that were just unblocked. */
-    if (listLength(server.unblocked_clients))
-        processUnblockedClients();
+        /* Try to process pending commands for clients that were just unblocked. */
+        if (listLength(server.unblocked_clients))
+            processUnblockedClients();
 
-    /* Send all the slaves an ACK request if at least one client blocked
-     * during the previous event loop iteration. Note that we do this after
-     * processUnblockedClients(), so if there are multiple pipelined WAITs
-     * and the just unblocked WAIT gets blocked again, we don't have to wait
-     * a server cron cycle in absence of other event loop events. See #6623. */
-    if (server.get_ack_from_slaves) {
-        robj *argv[3];
+        /* Send all the slaves an ACK request if at least one client blocked
+         * during the previous event loop iteration. Note that we do this after
+         * processUnblockedClients(), so if there are multiple pipelined WAITs
+         * and the just unblocked WAIT gets blocked again, we don't have to wait
+         * a server cron cycle in absence of other event loop events. See #6623. */
+        if (server.get_ack_from_slaves) {
+            robj *argv[3];
 
-        argv[0] = createStringObject("REPLCONF",8);
-        argv[1] = createStringObject("GETACK",6);
-        argv[2] = createStringObject("*",1); /* Not used argument. */
-        replicationFeedSlaves(server.slaves, server.slaveseldb, argv, 3);
-        decrRefCount(argv[0]);
-        decrRefCount(argv[1]);
-        decrRefCount(argv[2]);
-        server.get_ack_from_slaves = 0;
+            argv[0] = createStringObject("REPLCONF",8);
+            argv[1] = createStringObject("GETACK",6);
+            argv[2] = createStringObject("*",1); /* Not used argument. */
+            replicationFeedSlaves(server.slaves, server.slaveseldb, argv, 3);
+            decrRefCount(argv[0]);
+            decrRefCount(argv[1]);
+            decrRefCount(argv[2]);
+            server.get_ack_from_slaves = 0;
+        }
+
+        /* Send the invalidation messages to clients participating to the
+         * client side caching protocol in broadcasting (BCAST) mode. */
+        trackingBroadcastInvalidationMessages();
+
+        /* Write the AOF buffer on disk */
+        flushAppendOnlyFile(0);
     }
 
-    /* Send the invalidation messages to clients participating to the
-     * client side caching protocol in broadcasting (BCAST) mode. */
-    trackingBroadcastInvalidationMessages();
-
-    /* Write the AOF buffer on disk */
-    flushAppendOnlyFile(0);
-
     /* Handle writes with pending output buffers. */
     handleClientsWithPendingWritesUsingThreads();
 
     /* Close clients that need to be closed asynchronous */
     freeClientsInAsyncFreeQueue();
 
-    /* Before we are going to sleep, let the threads access the dataset by
-     * releasing the GIL. Redis main thread will not touch anything at this
-     * time. */
-    if (moduleCount()) moduleReleaseGIL();
+    if (!ProcessingEventsWhileBlocked) {
+        /* Before we are going to sleep, let the threads access the dataset by
+         * releasing the GIL. Redis main thread will not touch anything at this
+         * time. */
+        if (moduleCount()) moduleReleaseGIL();
+    }
 }
 
 /* This function is called immadiately after the event loop multiplexing
@@ -2165,7 +2181,10 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
  * the different events callbacks. */
 void afterSleep(struct aeEventLoop *eventLoop) {
     UNUSED(eventLoop);
-    if (moduleCount()) moduleAcquireGIL();
+
+    if (!ProcessingEventsWhileBlocked) {
+        if (moduleCount()) moduleAcquireGIL();
+    }
 }
 
 /* =========================== Server initialization ======================== */
@@ -2873,6 +2892,11 @@ void initServer(void) {
                 "blocked clients subsystem.");
     }
 
+    /* Register before and after sleep handlers (note this needs to be done
+     * before loading persistence since it is used by processEventsWhileBlocked. */
+    aeSetBeforeSleepProc(server.el,beforeSleep);
+    aeSetAfterSleepProc(server.el,afterSleep);
+
     /* Open the AOF file if needed. */
     if (server.aof_state == AOF_ON) {
         server.aof_fd = open(server.aof_filename,
@@ -5129,8 +5153,6 @@ int main(int argc, char **argv) {
     }
 
     redisSetCpuAffinity(server.server_cpulist);
-    aeSetBeforeSleepProc(server.el,beforeSleep);
-    aeSetAfterSleepProc(server.el,afterSleep);
     aeMain(server.el);
     aeDeleteEventLoop(server.el);
     return 0;
diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl
index b364291ee..123e9c8b6 100644
--- a/tests/integration/rdb.tcl
+++ b/tests/integration/rdb.tcl
@@ -128,4 +128,56 @@ start_server {} {
         # make sure the server is still writable
         r set x xx
     }
+}
+
+test {client freed during loading} {
+    start_server [list overrides [list key-load-delay 10 rdbcompression no]] {
+        # create a big rdb that will take long to load. it is important
+        # for keys to be big since the server processes events only once in 2mb.
+        # 100mb of rdb, 100k keys will load in more than 1 second
+        r debug populate 100000 key 1000
+
+        catch {
+            r debug restart
+        }
+
+        set stdout [srv 0 stdout]
+        while 1 {
+            # check that the new server actually started and is ready for connections
+            if {[exec grep -i "Server initialized" | wc -l < $stdout] > 1} {
+                break
+            }
+            after 10
+        }
+        # make sure it's still loading
+        assert_equal [s loading] 1
+
+        # connect and disconnect 10 clients
+        set clients {}
+        for {set j 0} {$j < 10} {incr j} {
+            lappend clients [redis_deferring_client]
+        }
+        foreach rd $clients {
+            $rd debug log bla
+        }
+        foreach rd $clients {
+            $rd read
+        }
+        foreach rd $clients {
+            $rd close
+        }
+
+        # make sure the server freed the clients
+        wait_for_condition 100 100 {
+            [s connected_clients] < 3
+        } else {
+            fail "clients didn't disconnect"
+        }
+
+        # make sure it's still loading
+        assert_equal [s loading] 1
+
+        # no need to keep waiting for loading to complete
+        exec kill [srv 0 pid]
+    }
 }
\ No newline at end of file

From b4416280cf9488f9e836d5fca1708849757c5f5c Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 12 May 2020 08:59:09 +0300
Subject: [PATCH 0368/1098] fix unstable replication test

this test which has coverage for varoius flows of diskless master was
failing randomly from time to time.

the failure was:
[err]: diskless all replicas drop during rdb pipe in tests/integration/replication.tcl
log message of '*Diskless rdb transfer, last replica dropped, killing fork child*' not found

what seemed to have happened is that the master didn't detect that all
replicas dropped by the time the replication ended, it thought that one
replica is still connected.

now the test takes a few seconds longer but it seems stable.
---
 tests/integration/replication.tcl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl
index f69002b36..22b0bfeaf 100644
--- a/tests/integration/replication.tcl
+++ b/tests/integration/replication.tcl
@@ -513,9 +513,9 @@ start_server {tags {"repl"}} {
     set master_port [srv 0 port]
     set master_pid [srv 0 pid]
     # put enough data in the db that the rdb file will be bigger than the socket buffers
-    # and since we'll have key-load-delay of 100, 10000 keys will take at least 1 second
+    # and since we'll have key-load-delay of 100, 20000 keys will take at least 2 seconds
     # we also need the replica to process requests during transfer (which it does only once in 2mb)
-    $master debug populate 10000 test 10000
+    $master debug populate 20000 test 10000
     $master config set rdbcompression no
     # If running on Linux, we also measure utime/stime to detect possible I/O handling issues
     set os [catch {exec unamee}]

From 27e25e9d1ed1b0ba3f639eb64c8f27b852753045 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 12 May 2020 13:07:44 +0200
Subject: [PATCH 0369/1098] Some rework of #7234.

---
 src/ae.c         |   7 ++-
 src/ae.h         |   9 ++--
 src/networking.c |   5 ++-
 src/server.c     | 111 +++++++++++++++++++++++++----------------------
 4 files changed, 72 insertions(+), 60 deletions(-)

diff --git a/src/ae.c b/src/ae.c
index 7aa204250..379cfd1e6 100644
--- a/src/ae.c
+++ b/src/ae.c
@@ -370,6 +370,7 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
  * if flags has AE_DONT_WAIT set the function returns ASAP until all
  * the events that's possible to process without to wait are processed.
  * if flags has AE_CALL_AFTER_SLEEP set, the aftersleep callback is called.
+ * if flags has AE_CALL_BEFORE_SLEEP set, the beforesleep callback is called.
  *
  * The function returns the number of events processed. */
 int aeProcessEvents(aeEventLoop *eventLoop, int flags)
@@ -428,7 +429,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
             tvp = &tv;
         }
 
-        if (eventLoop->beforesleep != NULL)
+        if (eventLoop->beforesleep != NULL && flags & AE_CALL_BEFORE_SLEEP)
             eventLoop->beforesleep(eventLoop);
 
         /* Call the multiplexing API, will return only on timeout or when
@@ -525,7 +526,9 @@ int aeWait(int fd, int mask, long long milliseconds) {
 void aeMain(aeEventLoop *eventLoop) {
     eventLoop->stop = 0;
     while (!eventLoop->stop) {
-        aeProcessEvents(eventLoop, AE_ALL_EVENTS|AE_CALL_AFTER_SLEEP);
+        aeProcessEvents(eventLoop, AE_ALL_EVENTS|
+                                   AE_CALL_BEFORE_SLEEP|
+                                   AE_CALL_AFTER_SLEEP);
     }
 }
 
diff --git a/src/ae.h b/src/ae.h
index 9acd72434..63b306615 100644
--- a/src/ae.h
+++ b/src/ae.h
@@ -47,11 +47,12 @@
                            things to disk before sending replies, and want
                            to do that in a group fashion. */
 
-#define AE_FILE_EVENTS 1
-#define AE_TIME_EVENTS 2
+#define AE_FILE_EVENTS (1<<0)
+#define AE_TIME_EVENTS (1<<1)
 #define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS)
-#define AE_DONT_WAIT 4
-#define AE_CALL_AFTER_SLEEP 8
+#define AE_DONT_WAIT (1<<2)
+#define AE_CALL_BEFORE_SLEEP (1<<3)
+#define AE_CALL_AFTER_SLEEP (1<<4)
 
 #define AE_NOMORE -1
 #define AE_DELETED_EVENT_ID -1
diff --git a/src/networking.c b/src/networking.c
index 75c0c16b1..ce5b0ae38 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2863,8 +2863,9 @@ int processEventsWhileBlocked(void) {
     ProcessingEventsWhileBlocked = 1;
     while (iterations--) {
         int events = 0;
-        events += aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT);
-        events += handleClientsWithPendingWrites();
+        events += aeProcessEvents(server.el,
+            AE_FILE_EVENTS|AE_DONT_WAIT|
+            AE_CALL_BEFORE_SLEEP|AE_CALL_AFTER_SLEEP);
         if (!events) break;
         count += events;
     }
diff --git a/src/server.c b/src/server.c
index fd75f25d6..380a6d5b6 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2086,22 +2086,33 @@ extern int ProcessingEventsWhileBlocked;
 /* This function gets called every time Redis is entering the
  * main loop of the event driven library, that is, before to sleep
  * for ready file descriptors.
+ *
  * Note: This function is (currently) called from two functions:
  * 1. aeMain - The main server loop
  * 2. processEventsWhileBlocked - Process clients during RDB/AOF load
+ *
  * If it was called from processEventsWhileBlocked we don't want
  * to perform all actions (For example, we don't want to expire
  * keys), but we do need to perform some actions.
+ *
  * The most important is freeClientsInAsyncFreeQueue but we also
  * call some other low-risk functions. */
 void beforeSleep(struct aeEventLoop *eventLoop) {
     UNUSED(eventLoop);
 
-    if (!ProcessingEventsWhileBlocked) {
-        /* Handle precise timeouts of blocked clients. */
-        handleBlockedClientsTimeout();
+    /* Just call a subset of vital functions in case we are re-entering
+     * the event loop from processEventsWhileBlocked(). */
+    if (ProcessingEventsWhileBlocked) {
+        handleClientsWithPendingReadsUsingThreads();
+        tlsProcessPendingData();
+        handleClientsWithPendingWrites();
+        freeClientsInAsyncFreeQueue();
+        return;
     }
 
+    /* Handle precise timeouts of blocked clients. */
+    handleBlockedClientsTimeout();
+
     /* We should handle pending reads clients ASAP after event loop. */
     handleClientsWithPendingReadsUsingThreads();
 
@@ -2111,69 +2122,65 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     /* If tls still has pending unread data don't sleep at all. */
     aeSetDontWait(server.el, tlsHasPendingData());
 
-    if (!ProcessingEventsWhileBlocked) {
-        /* Call the Redis Cluster before sleep function. Note that this function
-         * may change the state of Redis Cluster (from ok to fail or vice versa),
-         * so it's a good idea to call it before serving the unblocked clients
-         * later in this function. */
-        if (server.cluster_enabled) clusterBeforeSleep();
+    /* Call the Redis Cluster before sleep function. Note that this function
+     * may change the state of Redis Cluster (from ok to fail or vice versa),
+     * so it's a good idea to call it before serving the unblocked clients
+     * later in this function. */
+    if (server.cluster_enabled) clusterBeforeSleep();
 
-        /* Run a fast expire cycle (the called function will return
-         * ASAP if a fast cycle is not needed). */
-        if (server.active_expire_enabled && server.masterhost == NULL)
-            activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST);
+    /* Run a fast expire cycle (the called function will return
+     * ASAP if a fast cycle is not needed). */
+    if (server.active_expire_enabled && server.masterhost == NULL)
+        activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST);
 
-        /* Unblock all the clients blocked for synchronous replication
-         * in WAIT. */
-        if (listLength(server.clients_waiting_acks))
-            processClientsWaitingReplicas();
+    /* Unblock all the clients blocked for synchronous replication
+     * in WAIT. */
+    if (listLength(server.clients_waiting_acks))
+        processClientsWaitingReplicas();
 
-        /* Check if there are clients unblocked by modules that implement
-         * blocking commands. */
-        if (moduleCount()) moduleHandleBlockedClients();
+    /* Check if there are clients unblocked by modules that implement
+     * blocking commands. */
+    if (moduleCount()) moduleHandleBlockedClients();
 
-        /* Try to process pending commands for clients that were just unblocked. */
-        if (listLength(server.unblocked_clients))
-            processUnblockedClients();
+    /* Try to process pending commands for clients that were just unblocked. */
+    if (listLength(server.unblocked_clients))
+        processUnblockedClients();
 
-        /* Send all the slaves an ACK request if at least one client blocked
-         * during the previous event loop iteration. Note that we do this after
-         * processUnblockedClients(), so if there are multiple pipelined WAITs
-         * and the just unblocked WAIT gets blocked again, we don't have to wait
-         * a server cron cycle in absence of other event loop events. See #6623. */
-        if (server.get_ack_from_slaves) {
-            robj *argv[3];
+    /* Send all the slaves an ACK request if at least one client blocked
+     * during the previous event loop iteration. Note that we do this after
+     * processUnblockedClients(), so if there are multiple pipelined WAITs
+     * and the just unblocked WAIT gets blocked again, we don't have to wait
+     * a server cron cycle in absence of other event loop events. See #6623. */
+    if (server.get_ack_from_slaves) {
+        robj *argv[3];
 
-            argv[0] = createStringObject("REPLCONF",8);
-            argv[1] = createStringObject("GETACK",6);
-            argv[2] = createStringObject("*",1); /* Not used argument. */
-            replicationFeedSlaves(server.slaves, server.slaveseldb, argv, 3);
-            decrRefCount(argv[0]);
-            decrRefCount(argv[1]);
-            decrRefCount(argv[2]);
-            server.get_ack_from_slaves = 0;
-        }
-
-        /* Send the invalidation messages to clients participating to the
-         * client side caching protocol in broadcasting (BCAST) mode. */
-        trackingBroadcastInvalidationMessages();
-
-        /* Write the AOF buffer on disk */
-        flushAppendOnlyFile(0);
+        argv[0] = createStringObject("REPLCONF",8);
+        argv[1] = createStringObject("GETACK",6);
+        argv[2] = createStringObject("*",1); /* Not used argument. */
+        replicationFeedSlaves(server.slaves, server.slaveseldb, argv, 3);
+        decrRefCount(argv[0]);
+        decrRefCount(argv[1]);
+        decrRefCount(argv[2]);
+        server.get_ack_from_slaves = 0;
     }
 
+    /* Send the invalidation messages to clients participating to the
+     * client side caching protocol in broadcasting (BCAST) mode. */
+    trackingBroadcastInvalidationMessages();
+
+    /* Write the AOF buffer on disk */
+    flushAppendOnlyFile(0);
+
     /* Handle writes with pending output buffers. */
     handleClientsWithPendingWritesUsingThreads();
 
     /* Close clients that need to be closed asynchronous */
     freeClientsInAsyncFreeQueue();
 
-    if (!ProcessingEventsWhileBlocked) {
-        /* Before we are going to sleep, let the threads access the dataset by
-         * releasing the GIL. Redis main thread will not touch anything at this
-         * time. */
-        if (moduleCount()) moduleReleaseGIL();
-    }
+    /* Before we are going to sleep, let the threads access the dataset by
+     * releasing the GIL. Redis main thread will not touch anything at this
+     * time. */
+    if (moduleCount()) moduleReleaseGIL();
 }
 
 /* This function is called immadiately after the event loop multiplexing

From 4715ce59037d5806ede71e605f53b1addb7014ed Mon Sep 17 00:00:00 2001
From: David Carlier <devnexen@gmail.com>
Date: Tue, 12 May 2020 21:19:12 +0100
Subject: [PATCH 0370/1098] NetBSD build update.

This platform supports CPU affinity (but not OpenBSD).
---
 src/Makefile         | 10 ++++++++++
 src/config.h         |  2 +-
 src/setcpuaffinity.c | 19 +++++++++++++++++++
 3 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/src/Makefile b/src/Makefile
index 55f862cfc..b8c05c32b 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -128,6 +128,14 @@ else
 ifeq ($(uname_S),DragonFly)
 	# FreeBSD
 	FINAL_LIBS+= -lpthread -lexecinfo
+else
+ifeq ($(uname_S),OpenBSD)
+	# OpenBSD
+	FINAL_LIBS+= -lpthread -lexecinfo
+else
+ifeq ($(uname_S),NetBSD)
+	# NetBSD
+	FINAL_LIBS+= -lpthread -lexecinfo
 else
 	# All the other OSes (notably Linux)
 	FINAL_LDFLAGS+= -rdynamic
@@ -138,6 +146,8 @@ endif
 endif
 endif
 endif
+endif
+endif
 # Include paths to dependencies
 FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src
 
diff --git a/src/config.h b/src/config.h
index 6025d4e96..0fcc42972 100644
--- a/src/config.h
+++ b/src/config.h
@@ -248,7 +248,7 @@ int pthread_setname_np(const char *name);
 #endif
 
 /* Check if we can use setcpuaffinity(). */
-#if (defined __linux || defined __NetBSD__ || defined __FreeBSD__ || defined __OpenBSD__)
+#if (defined __linux || defined __NetBSD__ || defined __FreeBSD__)
 #define USE_SETCPUAFFINITY
 void setcpuaffinity(const char *cpulist);
 #endif
diff --git a/src/setcpuaffinity.c b/src/setcpuaffinity.c
index dcae81c71..13594113c 100644
--- a/src/setcpuaffinity.c
+++ b/src/setcpuaffinity.c
@@ -36,6 +36,10 @@
 #include <sys/param.h>
 #include <sys/cpuset.h>
 #endif
+#ifdef __NetBSD__
+#include <pthread.h>
+#include <sched.h>
+#endif
 #include "config.h"
 
 #ifdef USE_SETCPUAFFINITY
@@ -71,11 +75,18 @@ void setcpuaffinity(const char *cpulist) {
 #ifdef __FreeBSD__
     cpuset_t cpuset;
 #endif
+#ifdef __NetBSD__
+    cpuset_t *cpuset;
+#endif
 
     if (!cpulist)
         return;
 
+#ifndef __NetBSD__
     CPU_ZERO(&cpuset);
+#else
+    cpuset = cpuset_create();
+#endif
 
     q = cpulist;
     while (p = q, q = next_token(q, ','), p) {
@@ -110,7 +121,11 @@ void setcpuaffinity(const char *cpulist) {
             return;
 
         while (a <= b) {
+#ifndef __NetBSD__
             CPU_SET(a, &cpuset);
+#else
+            cpuset_set(a, cpuset);
+#endif
             a += s;
         }
     }
@@ -124,6 +139,10 @@ void setcpuaffinity(const char *cpulist) {
 #ifdef __FreeBSD__
     cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, sizeof(cpuset), &cpuset);
 #endif
+#ifdef __NetBSD__
+    pthread_setaffinity_np(pthread_self(), cpuset_size(cpuset), cpuset);
+    cpuset_destroy(cpuset);
+#endif
 }
 
 #endif /* USE_SETCPUAFFINITY */

From 827e6cc6443e0727a0cfbf3fcf87303c9783dc78 Mon Sep 17 00:00:00 2001
From: Benjamin Sergeant <bsergean@gmail.com>
Date: Wed, 13 May 2020 09:24:51 -0700
Subject: [PATCH 0371/1098] Redis-cli 6.0.1 `--cluster-yes` doesn't work (fix
 #7246)

This make it so that all prompts for all redis-cli --cluster commands are automatically answered with a yes.
---
 src/redis-cli.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 208f7b35d..2624a528a 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -1798,6 +1798,10 @@ static void usage(void) {
 }
 
 static int confirmWithYes(char *msg) {
+    if (config.cluster_manager_command.flags & CLUSTER_MANAGER_CMD_FLAG_YES) {
+        return 1;
+    }
+    
     printf("%s (type 'yes' to accept): ", msg);
     fflush(stdout);
     char buf[4];
@@ -8084,7 +8088,7 @@ int main(int argc, char **argv) {
         if (cliConnect(0) == REDIS_ERR) exit(1);
         if (config.interval == 0) config.interval = 1000000;
         statMode();
-    }
+(    }
 
     /* Scan mode */
     if (config.scan_mode) {

From db39b04d760eb3f1450184d0e4759db81de9500a Mon Sep 17 00:00:00 2001
From: Benjamin Sergeant <bsergeant@mz.com>
Date: Wed, 13 May 2020 09:32:27 -0700
Subject: [PATCH 0372/1098] fix typo ...

---
 src/redis-cli.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 2624a528a..7f952a4c6 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -8088,7 +8088,7 @@ int main(int argc, char **argv) {
         if (cliConnect(0) == REDIS_ERR) exit(1);
         if (config.interval == 0) config.interval = 1000000;
         statMode();
-(    }
+    }
 
     /* Scan mode */
     if (config.scan_mode) {

From bc4667acbcbeced700d0f64735e8ba8c7ffb8357 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 14 May 2020 10:02:57 +0200
Subject: [PATCH 0373/1098] Track events processed while blocked globally.

Related to #7234.
---
 src/connection.h |  2 +-
 src/networking.c | 19 ++++++++++++-------
 src/server.c     | 16 +++++++++++-----
 src/server.h     |  5 +++--
 src/tls.c        |  7 +++++--
 5 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/src/connection.h b/src/connection.h
index db09dfd83..0fd6c5f24 100644
--- a/src/connection.h
+++ b/src/connection.h
@@ -222,6 +222,6 @@ const char *connGetInfo(connection *conn, char *buf, size_t buf_len);
 
 /* Helpers for tls special considerations */
 int tlsHasPendingData();
-void tlsProcessPendingData();
+int tlsProcessPendingData();
 
 #endif  /* __REDIS_CONNECTION_H */
diff --git a/src/networking.c b/src/networking.c
index ce5b0ae38..671e374f4 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1257,7 +1257,10 @@ void freeClientAsync(client *c) {
     pthread_mutex_unlock(&async_free_queue_mutex);
 }
 
-void freeClientsInAsyncFreeQueue(void) {
+/* Free the clietns marked as CLOSE_ASAP, return the number of clients
+ * freed. */
+int freeClientsInAsyncFreeQueue(void) {
+    int freed = listLength(server.clients_to_close);
     while (listLength(server.clients_to_close)) {
         listNode *ln = listFirst(server.clients_to_close);
         client *c = listNodeValue(ln);
@@ -1266,6 +1269,7 @@ void freeClientsInAsyncFreeQueue(void) {
         freeClient(c);
         listDelNode(server.clients_to_close,ln);
     }
+    return freed;
 }
 
 /* Return a client by ID, or NULL if the client ID is not in the set
@@ -2852,9 +2856,8 @@ int clientsArePaused(void) {
  * write, close sequence needed to serve a client.
  *
  * The function returns the total number of events processed. */
-int processEventsWhileBlocked(void) {
+void processEventsWhileBlocked(void) {
     int iterations = 4; /* See the function top-comment. */
-    int count = 0;
 
     /* Note: when we are processing events while blocked (for instance during
      * busy Lua scripts), we set a global flag. When such flag is set, we
@@ -2862,15 +2865,17 @@ int processEventsWhileBlocked(void) {
      * See https://github.com/antirez/redis/issues/6988 for more info. */
     ProcessingEventsWhileBlocked = 1;
     while (iterations--) {
-        int events = 0;
-        events += aeProcessEvents(server.el,
+        long long startval = server.events_processed_while_blocked;
+        long long ae_events = aeProcessEvents(server.el,
             AE_FILE_EVENTS|AE_DONT_WAIT|
             AE_CALL_BEFORE_SLEEP|AE_CALL_AFTER_SLEEP);
+        /* Note that server.events_processed_while_blocked will also get
+         * incremeted by callbacks called by the event loop handlers. */
+        server.events_processed_while_blocked += ae_events;
+        long long events = server.events_processed_while_blocked - startval;
         if (!events) break;
-        count += events;
     }
     ProcessingEventsWhileBlocked = 0;
-    return count;
 }
 
 /* ==========================================================================
diff --git a/src/server.c b/src/server.c
index 380a6d5b6..1bed3959d 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2101,12 +2101,17 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     UNUSED(eventLoop);
 
     /* Just call a subset of vital functions in case we are re-entering
-     * the event loop from processEventsWhileBlocked(). */
+     * the event loop from processEventsWhileBlocked(). Note that in this
+     * case we keep track of the number of events we are processing, since
+     * processEventsWhileBlocked() wants to stop ASAP if there are no longer
+     * events to handle. */
     if (ProcessingEventsWhileBlocked) {
-        handleClientsWithPendingReadsUsingThreads();
-        tlsProcessPendingData();
-        handleClientsWithPendingWrites();
-        freeClientsInAsyncFreeQueue();
+        uint64_t processed = 0;
+        processed += handleClientsWithPendingReadsUsingThreads();
+        processed += tlsProcessPendingData();
+        processed += handleClientsWithPendingWrites();
+        processed += freeClientsInAsyncFreeQueue();
+        server.events_processed_while_blocked += processed;
         return;
     }
 
@@ -2757,6 +2762,7 @@ void initServer(void) {
     server.clients_waiting_acks = listCreate();
     server.get_ack_from_slaves = 0;
     server.clients_paused = 0;
+    server.events_processed_while_blocked = 0;
     server.system_memory_size = zmalloc_get_memory_size();
 
     if (server.tls_port && tlsConfigure(&server.tls_ctx_config) == C_ERR) {
diff --git a/src/server.h b/src/server.h
index 59cf1370e..55ee2d300 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1095,6 +1095,7 @@ struct redisServer {
                                    queries. Will still serve RESP2 queries. */
     int io_threads_num;         /* Number of IO threads to use. */
     int io_threads_do_reads;    /* Read and parse from IO threads? */
+    long long events_processed_while_blocked; /* processEventsWhileBlocked() */
 
     /* RDB / AOF loading information */
     int loading;                /* We are loading data from disk if true */
@@ -1652,7 +1653,7 @@ void rewriteClientCommandVector(client *c, int argc, ...);
 void rewriteClientCommandArgument(client *c, int i, robj *newval);
 void replaceClientCommandVector(client *c, int argc, robj **argv);
 unsigned long getClientOutputBufferMemoryUsage(client *c);
-void freeClientsInAsyncFreeQueue(void);
+int freeClientsInAsyncFreeQueue(void);
 void asyncCloseClientOnOutputBufferLimitReached(client *c);
 int getClientType(client *c);
 int getClientTypeByName(char *name);
@@ -1662,7 +1663,7 @@ void disconnectSlaves(void);
 int listenToPort(int port, int *fds, int *count);
 void pauseClients(mstime_t duration);
 int clientsArePaused(void);
-int processEventsWhileBlocked(void);
+void processEventsWhileBlocked(void);
 int handleClientsWithPendingWrites(void);
 int handleClientsWithPendingWritesUsingThreads(void);
 int handleClientsWithPendingReadsUsingThreads(void);
diff --git a/src/tls.c b/src/tls.c
index c18aafebe..ee85bd302 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -768,15 +768,17 @@ int tlsHasPendingData() {
     return listLength(pending_list) > 0;
 }
 
-void tlsProcessPendingData() {
+int tlsProcessPendingData() {
     listIter li;
     listNode *ln;
 
+    int processed = listLength(pending_list);
     listRewind(pending_list,&li);
     while((ln = listNext(&li))) {
         tls_connection *conn = listNodeValue(ln);
         tlsHandleEvent(conn, AE_READABLE);
     }
+    return processed;
 }
 
 #else   /* USE_OPENSSL */
@@ -804,7 +806,8 @@ int tlsHasPendingData() {
     return 0;
 }
 
-void tlsProcessPendingData() {
+int tlsProcessPendingData() {
+    return 0;
 }
 
 #endif

From a4d0720fa105b5bc8bd4a5607864862185b78105 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 14 May 2020 10:41:28 +0200
Subject: [PATCH 0374/1098] Tracking: send eviction messages when evicting
 entries.

A fix for #7249.
---
 src/server.c   |  6 ++++++
 src/tracking.c | 35 +++++++++++++++++++++++------------
 2 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/src/server.c b/src/server.c
index e2b4b6f3d..847a6a95a 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2053,6 +2053,12 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
     /* Stop the I/O threads if we don't have enough pending work. */
     stopThreadedIOIfNeeded();
 
+    /* Resize tracking keys table if needed. This is also done at every
+     * command execution, but we want to be sure that if the last command
+     * executed changes the value via CONFIG SET, the server will perform
+     * the operation even if completely idle. */
+    if (server.tracking_clients) trackingLimitUsedSlots();
+
     /* Start a scheduled BGSAVE if the corresponding flag is set. This is
      * useful when we are forced to postpone a BGSAVE because an AOF
      * rewrite is in progress.
diff --git a/src/tracking.c b/src/tracking.c
index a995817e2..cfde26fc9 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -279,15 +279,22 @@ void trackingRememberKeyToBroadcast(client *c, char *keyname, size_t keylen) {
  *
  * Note that 'c' may be NULL in case the operation was performed outside the
  * context of a client modifying the database (for instance when we delete a
- * key because of expire). */
-void trackingInvalidateKey(client *c, robj *keyobj) {
+ * key because of expire).
+ *
+ * The last argument 'bcast' tells the function if it should also schedule
+ * the key for broadcasting to clients in BCAST mode. This is the case when
+ * the function is called from the Redis core once a key is modified, however
+ * we also call the function in order to evict keys in the key table in case
+ * of memory pressure: in that case the key didn't really change, so we want
+ * just to notify the clients that are in the table for this key, that would
+ * otherwise miss the fact we are no longer tracking the key for them. */
+void trackingInvalidateKeyRaw(client *c, char *key, size_t keylen, int bcast) {
     if (TrackingTable == NULL) return;
-    sds sdskey = keyobj->ptr;
 
-    if (raxSize(PrefixTable) > 0)
-        trackingRememberKeyToBroadcast(c,sdskey,sdslen(sdskey));
+    if (bcast && raxSize(PrefixTable) > 0)
+        trackingRememberKeyToBroadcast(c,key,keylen);
 
-    rax *ids = raxFind(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey));
+    rax *ids = raxFind(TrackingTable,(unsigned char*)key,keylen);
     if (ids == raxNotFound) return;
 
     raxIterator ri;
@@ -317,7 +324,7 @@ void trackingInvalidateKey(client *c, robj *keyobj) {
             continue;
         }
 
-        sendTrackingMessage(target,sdskey,sdslen(sdskey),0);
+        sendTrackingMessage(target,key,keylen,0);
     }
     raxStop(&ri);
 
@@ -325,7 +332,13 @@ void trackingInvalidateKey(client *c, robj *keyobj) {
      * again if more keys will be modified in this caching slot. */
     TrackingTableTotalItems -= raxSize(ids);
     raxFree(ids);
-    raxRemove(TrackingTable,(unsigned char*)sdskey,sdslen(sdskey),NULL);
+    raxRemove(TrackingTable,(unsigned char*)key,keylen,NULL);
+}
+
+/* Wrapper (the one actually called across the core) to pass the key
+ * as object. */
+void trackingInvalidateKey(client *c, robj *keyobj) {
+    trackingInvalidateKeyRaw(c,keyobj->ptr,sdslen(keyobj->ptr),1);
 }
 
 /* This function is called when one or all the Redis databases are flushed
@@ -392,10 +405,8 @@ void trackingLimitUsedSlots(void) {
         effort--;
         raxSeek(&ri,"^",NULL,0);
         raxRandomWalk(&ri,0);
-        rax *ids = ri.data;
-        TrackingTableTotalItems -= raxSize(ids);
-        raxFree(ids);
-        raxRemove(TrackingTable,ri.key,ri.key_len,NULL);
+        if (raxEOF(&ri)) break;
+        trackingInvalidateKeyRaw(NULL,(char*)ri.key,ri.key_len,0);
         if (raxSize(TrackingTable) <= max_keys) {
             timeout_counter = 0;
             raxStop(&ri);

From 3e6acc24d6e00c7ca14e02bf7e81a8650f48f63a Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 14 May 2020 11:17:47 +0200
Subject: [PATCH 0375/1098] rax.c updated from upstream antirez/rax.

---
 src/rax.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/rax.c b/src/rax.c
index a560dde02..7dcf04582 100644
--- a/src/rax.c
+++ b/src/rax.c
@@ -1,6 +1,8 @@
 /* Rax -- A radix tree implementation.
  *
- * Copyright (c) 2017-2018, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Version 1.2 -- 7 February 2019
+ *
+ * Copyright (c) 2017-2019, Salvatore Sanfilippo <antirez at gmail dot com>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -1737,7 +1739,7 @@ int raxRandomWalk(raxIterator *it, size_t steps) {
     }
 
     if (steps == 0) {
-        size_t fle = floor(log(it->rt->numele));
+        size_t fle = 1+floor(log(it->rt->numele));
         fle *= 2;
         steps = 1 + rand() % fle;
     }

From dec6fd3adc59ecb6e3dc8320c448e9f20ea00b9d Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 14 May 2020 11:27:31 +0200
Subject: [PATCH 0376/1098] Regression test for #7249.

---
 tests/unit/tracking.tcl | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index 43bb5f864..0332fa726 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -107,5 +107,27 @@ start_server {tags {"tracking"}} {
         assert {$keys eq {mykey}}
     }
 
+    test {Tracking gets notification on tracking table key eviction} {
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on REDIRECT $redir NOLOOP
+        r MSET key1 1 key2 2
+        # Let the server track the two keys for us
+        r MGET key1 key2
+        # Force the eviction of all the keys but one:
+        r config set tracking-table-max-keys 1
+        # Note that we may have other keys in the table for this client,
+        # since we disabled/enabled tracking multiple time with the same
+        # ID, and tracking does not do ID cleanups for performance reasons.
+        # So we check that eventually we'll receive one or the other key,
+        # otherwise the test will die for timeout.
+        while 1 {
+            set keys [lindex [$rd1 read] 2]
+            if {$keys eq {key1} || $keys eq {key2}} break
+        }
+        # We should receive an expire notification for one of
+        # the two keys (only one must remain)
+        assert {$keys eq {key1} || $keys eq {key2}}
+    }
+
     $rd1 close
 }

From 41cabca2ab4478c3288d0d97419f4412785652bc Mon Sep 17 00:00:00 2001
From: Madelyn Olson <matolson@amazon.com>
Date: Thu, 14 May 2020 08:37:24 -0700
Subject: [PATCH 0377/1098] Added a refcount on timer events to prevent
 deletion of recursive timer calls

---
 src/ae.c | 10 ++++++++++
 src/ae.h |  2 ++
 2 files changed, 12 insertions(+)

diff --git a/src/ae.c b/src/ae.c
index 379cfd1e6..689a27d16 100644
--- a/src/ae.c
+++ b/src/ae.c
@@ -238,6 +238,7 @@ long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
     te->clientData = clientData;
     te->prev = NULL;
     te->next = eventLoop->timeEventHead;
+    te->refcount = 0;
     if (te->next)
         te->next->prev = te;
     eventLoop->timeEventHead = te;
@@ -316,6 +317,13 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
         /* Remove events scheduled for deletion. */
         if (te->id == AE_DELETED_EVENT_ID) {
             aeTimeEvent *next = te->next;
+            /* If a reference exists for this timer event,
+             * don't free it. This is currently incremented
+             * for recursive timerProc calls */
+            if (te->refcount) {
+                te = next;
+                continue;
+            }
             if (te->prev)
                 te->prev->next = te->next;
             else
@@ -345,7 +353,9 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
             int retval;
 
             id = te->id;
+            te->refcount++;
             retval = te->timeProc(eventLoop, id, te->clientData);
+            te->refcount--;
             processed++;
             if (retval != AE_NOMORE) {
                 aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms);
diff --git a/src/ae.h b/src/ae.h
index 63b306615..d1b7f34bf 100644
--- a/src/ae.h
+++ b/src/ae.h
@@ -86,6 +86,8 @@ typedef struct aeTimeEvent {
     void *clientData;
     struct aeTimeEvent *prev;
     struct aeTimeEvent *next;
+    int refcount; /* refcount to prevent timer events from being
+  		   * freed in recursive time event calls. */
 } aeTimeEvent;
 
 /* A fired event */

From f0f30fc43f027ad981c46a2a0b8dd12b9e9c53b5 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <matolson@amazon.com>
Date: Thu, 14 May 2020 11:07:51 -0700
Subject: [PATCH 0378/1098] Converge hash validation for adding and removing

---
 src/acl.c | 35 +++++++++++++++++++++--------------
 1 file changed, 21 insertions(+), 14 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 3194feb5b..bcca116bb 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -166,6 +166,25 @@ sds ACLHashPassword(unsigned char *cleartext, size_t len) {
     return sdsnewlen(hex,HASH_PASSWORD_LEN);
 }
 
+/* Given a hash and the hash length, returns C_OK if it is a valid password 
+ * hash, or C_ERR otherwise. */
+int ACLCheckPasswordHash(unsigned char *hash, int hashlen) {
+    if (hashlen != HASH_PASSWORD_LEN) {
+        return C_ERR;      
+    }
+ 
+    /* Password hashes can only be characters that represent
+     * hexadecimal values, which are numbers and lowercase 
+     * characters 'a' through 'f'. */
+    for(int i = 0; i < HASH_PASSWORD_LEN; i++) {
+        char c = hash[i];
+        if ((c < 'a' || c > 'f') && (c < '0' || c > '9')) {
+            return C_ERR;
+        }
+    }
+    return C_OK;
+}
+
 /* =============================================================================
  * Low level ACL API
  * ==========================================================================*/
@@ -753,22 +772,10 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
         if (op[0] == '>') {
             newpass = ACLHashPassword((unsigned char*)op+1,oplen-1);
         } else {
-            if (oplen != HASH_PASSWORD_LEN + 1) {
+            if (ACLCheckPasswordHash((unsigned char*)op+1,oplen-1) == C_ERR) {
                 errno = EBADMSG;
                 return C_ERR;
             }
-
-            /* Password hashes can only be characters that represent
-             * hexadecimal values, which are numbers and lowercase
-             * characters 'a' through 'f'.
-             */
-            for(int i = 1; i < HASH_PASSWORD_LEN + 1; i++) {
-                char c = op[i];
-                if ((c < 'a' || c > 'f') && (c < '0' || c > '9')) {
-                    errno = EBADMSG;
-                    return C_ERR;
-                }
-            }
             newpass = sdsnewlen(op+1,oplen-1);
         }
 
@@ -784,7 +791,7 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
         if (op[0] == '<') {
             delpass = ACLHashPassword((unsigned char*)op+1,oplen-1);
         } else {
-            if (oplen != HASH_PASSWORD_LEN + 1) {
+            if (ACLCheckPasswordHash((unsigned char*)op+1,oplen-1) == C_ERR) {
                 errno = EBADMSG;
                 return C_ERR;
             }

From f4d2abd3df760e370510c79d2d5e27c22aa1d1ce Mon Sep 17 00:00:00 2001
From: Benjamin Sergeant <bsergeant@mz.com>
Date: Thu, 14 May 2020 15:29:06 -0700
Subject: [PATCH 0379/1098] do not handle --cluster-yes for cluster fix mode

---
 src/redis-cli.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 7f952a4c6..75845f346 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -1797,11 +1797,14 @@ static void usage(void) {
     exit(1);
 }
 
-static int confirmWithYes(char *msg) {
-    if (config.cluster_manager_command.flags & CLUSTER_MANAGER_CMD_FLAG_YES) {
+static int confirmWithYes(char *msg, int force) {
+    /* if force is true and --cluster-yes option is on,
+     * do not prompt for an answer */
+    if (force &&
+        (config.cluster_manager_command.flags & CLUSTER_MANAGER_CMD_FLAG_YES)) {
         return 1;
     }
-    
+
     printf("%s (type 'yes' to accept): ", msg);
     fflush(stdout);
     char buf[4];
@@ -4489,12 +4492,16 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) {
     }
     dictReleaseIterator(iter);
 
+    /* we want explicit manual confirmation from users for all the fix cases */
+    int force = 0;
+
     /*  Handle case "1": keys in no node. */
     if (listLength(none) > 0) {
         printf("The following uncovered slots have no keys "
                "across the cluster:\n");
         clusterManagerPrintSlotsList(none);
-        if (confirmWithYes("Fix these slots by covering with a random node?")){
+        if (confirmWithYes("Fix these slots by covering with a random node?",
+                           force)) {
             listIter li;
             listNode *ln;
             listRewind(none, &li);
@@ -4520,7 +4527,8 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) {
     if (listLength(single) > 0) {
         printf("The following uncovered slots have keys in just one node:\n");
         clusterManagerPrintSlotsList(single);
-        if (confirmWithYes("Fix these slots by covering with those nodes?")){
+        if (confirmWithYes("Fix these slots by covering with those nodes?",
+                           force)) {
             listIter li;
             listNode *ln;
             listRewind(single, &li);
@@ -4552,7 +4560,7 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) {
         printf("The following uncovered slots have keys in multiple nodes:\n");
         clusterManagerPrintSlotsList(multi);
         if (confirmWithYes("Fix these slots by moving keys "
-                           "into a single node?")) {
+                           "into a single node?", force)) {
             listIter li;
             listNode *ln;
             listRewind(multi, &li);
@@ -5515,7 +5523,8 @@ assign_replicas:
     }
     clusterManagerOptimizeAntiAffinity(ip_nodes, ip_count);
     clusterManagerShowNodes();
-    if (confirmWithYes("Can I set the above configuration?")) {
+    int force = 1;
+    if (confirmWithYes("Can I set the above configuration?", force)) {
         listRewind(cluster_manager.nodes, &li);
         while ((ln = listNext(&li)) != NULL) {
             clusterManagerNode *node = ln->value;

From 1a7cd2c0e25bd4488dd2c28f0ffea1b62dc0b3ad Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 15 May 2020 10:19:13 +0200
Subject: [PATCH 0380/1098] Cache master without checking of deferred close
 flags.

The context is issue #7205: since the introduction of threaded I/O we close
clients asynchronously by default from readQueryFromClient(). So we
should no longer prevent the caching of the master client, to later
PSYNC incrementally, if such flags are set. However we also don't want
the master client to be cached with such flags (would be closed
immediately after being restored). And yet we want a way to understand
if a master was closed because of a protocol error, and in that case
prevent the caching.
---
 src/networking.c  | 10 ++++------
 src/replication.c |  8 ++++++--
 src/server.h      |  1 +
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 671e374f4..6de00be09 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1141,10 +1141,7 @@ void freeClient(client *c) {
      * some unexpected state, by checking its flags. */
     if (server.master && c->flags & CLIENT_MASTER) {
         serverLog(LL_WARNING,"Connection with master lost.");
-        if (!(c->flags & (CLIENT_CLOSE_AFTER_REPLY|
-                          CLIENT_CLOSE_ASAP|
-                          CLIENT_BLOCKED)))
-        {
+        if (!(c->flags & (CLIENT_PROTOCOL_ERROR|CLIENT_BLOCKED))) {
             replicationCacheMaster(c);
             return;
         }
@@ -1558,7 +1555,8 @@ int processInlineBuffer(client *c) {
 }
 
 /* Helper function. Record protocol erro details in server log,
- * and set the client as CLIENT_CLOSE_AFTER_REPLY. */
+ * and set the client as CLIENT_CLOSE_AFTER_REPLY and
+ * CLIENT_PROTOCOL_ERROR. */
 #define PROTO_DUMP_LEN 128
 static void setProtocolError(const char *errstr, client *c) {
     if (server.verbosity <= LL_VERBOSE) {
@@ -1584,7 +1582,7 @@ static void setProtocolError(const char *errstr, client *c) {
             "Protocol error (%s) from client: %s. %s", errstr, client, buf);
         sdsfree(client);
     }
-    c->flags |= CLIENT_CLOSE_AFTER_REPLY;
+    c->flags |= (CLIENT_CLOSE_AFTER_REPLY|CLIENT_PROTOCOL_ERROR);
 }
 
 /* Process the query buffer for client 'c', setting up the client argument
diff --git a/src/replication.c b/src/replication.c
index c59639cd1..f433d6413 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2688,14 +2688,18 @@ void replicationCacheMaster(client *c) {
     /* Unlink the client from the server structures. */
     unlinkClient(c);
 
+    /* Clear flags that can create issues once we reconnect the client. */
+    c->flags &= ~(CLIENT_CLOSE_ASAP|CLIENT_CLOSE_AFTER_REPLY);
+
     /* Reset the master client so that's ready to accept new commands:
      * we want to discard te non processed query buffers and non processed
      * offsets, including pending transactions, already populated arguments,
      * pending outputs to the master. */
     sdsclear(server.master->querybuf);
     sdsclear(server.master->pending_querybuf);
-    /* Adjust reploff and read_reploff to the last meaningful offset we executed.
-     * this is the offset the replica will use for future PSYNC. */
+
+    /* Adjust reploff and read_reploff to the last meaningful offset we
+     * executed. This is the offset the replica will use for future PSYNC. */
     server.master->reploff = adjustMeaningfulReplOffset();
     server.master->read_reploff = server.master->reploff;
     if (c->flags & CLIENT_MULTI) discardTransaction(c);
diff --git a/src/server.h b/src/server.h
index 55ee2d300..f835bf5e9 100644
--- a/src/server.h
+++ b/src/server.h
@@ -255,6 +255,7 @@ typedef long long ustime_t; /* microsecond time type. */
 #define CLIENT_TRACKING_NOLOOP (1ULL<<37) /* Don't send invalidation messages
                                              about writes performed by myself.*/
 #define CLIENT_IN_TO_TABLE (1ULL<<38) /* This client is in the timeout table. */
+#define CLIENT_PROTOCOL_ERROR (1ULL<<39) /* Protocol error chatting with it. */
 
 /* Client block type (btype field in client structure)
  * if CLIENT_BLOCKED flag is set. */

From 624742d9b40d20afe8b694c2dc0b0349f8b7aaad Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Sat, 16 May 2020 17:15:35 +0200
Subject: [PATCH 0381/1098] Remove the client from CLOSE_ASAP list before
 caching the master.

This was broken in 1a7cd2c: we identified a crash in the CI, what
was happening before the fix should be like that:

1. The client gets in the async free list.
2. However freeClient() gets called again against the same client
   which is a master.
3. The client arrived in freeClient() with the CLOSE_ASAP flag set.
4. The master gets cached, but NOT removed from the CLOSE_ASAP linked
   list.
5. The master client that was cached was immediately removed since it
   was still in the list.
6. Redis accessed a freed cached master.

This is how the crash looked like:

=== REDIS BUG REPORT START: Cut & paste starting from here ===
1092:S 16 May 2020 11:44:09.731 # Redis 999.999.999 crashed by signal: 11
1092:S 16 May 2020 11:44:09.731 # Crashed running the instruction at: 0x447e18
1092:S 16 May 2020 11:44:09.731 # Accessing address: 0xffffffffffffffff
1092:S 16 May 2020 11:44:09.731 # Failed assertion:  (:0)

------ STACK TRACE ------
EIP:
src/redis-server 127.0.0.1:21300(readQueryFromClient+0x48)[0x447e18]

And the 0xffff address access likely comes from accessing an SDS that is
set to NULL (we go -1 offset to read the header).
---
 src/networking.c  | 21 ++++++++++++---------
 src/replication.c |  3 ---
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 6de00be09..0a04c0489 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1134,6 +1134,16 @@ void freeClient(client *c) {
     /* Notify module system that this client auth status changed. */
     moduleNotifyUserChanged(c);
 
+    /* If this client was scheduled for async freeing we need to remove it
+     * from the queue. Note that we need to do this here, because later
+     * we may call replicationCacheMaster() and the client should already
+     * be removed from the list of clients to free. */
+    if (c->flags & CLIENT_CLOSE_ASAP) {
+        ln = listSearchKey(server.clients_to_close,c);
+        serverAssert(ln != NULL);
+        listDelNode(server.clients_to_close,ln);
+    }
+
     /* If it is our master that's beging disconnected we should make sure
      * to cache the state to try a partial resynchronization later.
      *
@@ -1142,6 +1152,7 @@ void freeClient(client *c) {
     if (server.master && c->flags & CLIENT_MASTER) {
         serverLog(LL_WARNING,"Connection with master lost.");
         if (!(c->flags & (CLIENT_PROTOCOL_ERROR|CLIENT_BLOCKED))) {
+            c->flags &= ~(CLIENT_CLOSE_ASAP|CLIENT_CLOSE_AFTER_REPLY);
             replicationCacheMaster(c);
             return;
         }
@@ -1209,15 +1220,7 @@ void freeClient(client *c) {
      * we lost the connection with the master. */
     if (c->flags & CLIENT_MASTER) replicationHandleMasterDisconnection();
 
-    /* If this client was scheduled for async freeing we need to remove it
-     * from the queue. */
-    if (c->flags & CLIENT_CLOSE_ASAP) {
-        ln = listSearchKey(server.clients_to_close,c);
-        serverAssert(ln != NULL);
-        listDelNode(server.clients_to_close,ln);
-    }
-
-    /* Remove the contribution that this client gave to our
+   /* Remove the contribution that this client gave to our
      * incrementally computed memory usage. */
     server.stat_clients_type_memory[c->client_cron_last_memory_type] -=
         c->client_cron_last_memory_usage;
diff --git a/src/replication.c b/src/replication.c
index f433d6413..b10635f25 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2688,9 +2688,6 @@ void replicationCacheMaster(client *c) {
     /* Unlink the client from the server structures. */
     unlinkClient(c);
 
-    /* Clear flags that can create issues once we reconnect the client. */
-    c->flags &= ~(CLIENT_CLOSE_ASAP|CLIENT_CLOSE_AFTER_REPLY);
-
     /* Reset the master client so that's ready to accept new commands:
      * we want to discard te non processed query buffers and non processed
      * offsets, including pending transactions, already populated arguments,

From 357aace8953e5c904d9d9d408258cd824df1717c Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 17 May 2020 18:26:02 +0300
Subject: [PATCH 0382/1098] add regression test for the race in #7205

with the original version of 6.0.0, this test detects an excessive full
sync.
with the fix in 1a7cd2c0e, this test detects memory corruption,
especially when using libc allocator with or without valgrind.
---
 tests/integration/replication.tcl | 52 +++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl
index 22b0bfeaf..e5079d9dd 100644
--- a/tests/integration/replication.tcl
+++ b/tests/integration/replication.tcl
@@ -637,3 +637,55 @@ start_server {tags {"repl"}} {
         }
     }
 }
+
+test {replicaof right after disconnection} {
+    # this is a rare race condition that was reproduced sporadically by the psync2 unit.
+    # see details in #7205
+    start_server {tags {"repl"}} {
+        set replica1 [srv 0 client]
+        set replica1_host [srv 0 host]
+        set replica1_port [srv 0 port]
+        set replica1_log [srv 0 stdout]
+        start_server {} {
+            set replica2 [srv 0 client]
+            set replica2_host [srv 0 host]
+            set replica2_port [srv 0 port]
+            set replica2_log [srv 0 stdout]
+            start_server {} {
+                set master [srv 0 client]
+                set master_host [srv 0 host]
+                set master_port [srv 0 port]
+                $replica1 replicaof $master_host $master_port
+                $replica2 replicaof $master_host $master_port
+
+                wait_for_condition 50 100 {
+                    [string match {*master_link_status:up*} [$replica1 info replication]] &&
+                    [string match {*master_link_status:up*} [$replica2 info replication]]
+                } else {
+                    fail "Can't turn the instance into a replica"
+                }
+
+                set rd [redis_deferring_client -1]
+                $rd debug sleep 1
+                after 100
+
+                # when replica2 will wake up from the sleep it will find both disconnection
+                # from it's master and also a replicaof command at the same event loop
+                $master client kill type replica
+                $replica2 replicaof $replica1_host $replica1_port
+                $rd read
+
+                wait_for_condition 50 100 {
+                    [string match {*master_link_status:up*} [$replica2 info replication]]
+                } else {
+                    fail "role change failed."
+                }
+
+                # make sure psync succeeded, and there were no unexpected full syncs.
+                assert_equal [status $master sync_full] 2
+                assert_equal [status $replica1 sync_full] 0
+                assert_equal [status $replica2 sync_full] 0
+            }
+        }
+    }
+}

From 96bb0c9471ddc353ffb4f53e5efbd04d26b9d91f Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Sat, 16 May 2020 18:03:28 +0200
Subject: [PATCH 0383/1098] Improve the PSYNC2 test reliability.

---
 tests/integration/psync2.tcl | 48 +++++++++++++++++++++++++-----------
 1 file changed, 33 insertions(+), 15 deletions(-)

diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl
index 5fe29caba..d6b2d19d0 100644
--- a/tests/integration/psync2.tcl
+++ b/tests/integration/psync2.tcl
@@ -12,7 +12,7 @@ start_server {} {
 
     set no_exit 0                   ; # Do not exit at end of the test
 
-    set duration 20                 ; # Total test seconds
+    set duration 40                 ; # Total test seconds
 
     set genload 1                   ; # Load master with writes at every cycle
 
@@ -125,14 +125,16 @@ start_server {} {
             }
         }
 
-        # wait for all the slaves to be in sync with the master, due to pings, we have to re-sample the master constantly too
+        # wait for all the slaves to be in sync.
+        set masteroff [status $R($master_id) master_repl_offset]
         wait_for_condition 500 100 {
-            [status $R($master_id) master_repl_offset] == [status $R(0) master_repl_offset] &&
-            [status $R($master_id) master_repl_offset] == [status $R(1) master_repl_offset] &&
-            [status $R($master_id) master_repl_offset] == [status $R(2) master_repl_offset] &&
-            [status $R($master_id) master_repl_offset] == [status $R(3) master_repl_offset] &&
-            [status $R($master_id) master_repl_offset] == [status $R(4) master_repl_offset]
+            [status $R(0) master_repl_offset] >= $masteroff &&
+            [status $R(1) master_repl_offset] >= $masteroff &&
+            [status $R(2) master_repl_offset] >= $masteroff &&
+            [status $R(3) master_repl_offset] >= $masteroff &&
+            [status $R(4) master_repl_offset] >= $masteroff
         } else {
+            puts "Master ID is $master_id"
             for {set j 0} {$j < 5} {incr j} {
                 puts "$j: sync_full: [status $R($j) sync_full]"
                 puts "$j: id1      : [status $R($j) master_replid]:[status $R($j) master_repl_offset]"
@@ -140,17 +142,11 @@ start_server {} {
                 puts "$j: backlog  : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]"
                 puts "---"
             }
-            fail "Slaves are not in sync with the master after too long time."
+            fail "Replicas offsets didn't catch up with the master after too long time."
         }
 
-        # Put down the old master so that it cannot generate more
-        # replication stream, this way in the next master switch, the time at
-        # which we move slaves away is not important, each will have full
-        # history (otherwise PINGs will make certain slaves have more history),
-        # and sometimes a full resync will be needed.
-        $R($master_id) slaveof 127.0.0.1 0 ;# We use port zero to make it fail.
-
         if {$debug_msg} {
+            puts "Master ID is $master_id"
             for {set j 0} {$j < 5} {incr j} {
                 puts "$j: sync_full: [status $R($j) sync_full]"
                 puts "$j: id1      : [status $R($j) master_replid]:[status $R($j) master_repl_offset]"
@@ -168,6 +164,28 @@ start_server {} {
             assert {$sum == 4}
         }
 
+        # In absence of pings, are the instances really able to have
+        # the exact same offset?
+        $R($master_id) config set repl-ping-replica-period 3600
+        wait_for_condition 500 100 {
+            [status $R($master_id) master_repl_offset] == [status $R(0) master_repl_offset] &&
+            [status $R($master_id) master_repl_offset] == [status $R(1) master_repl_offset] &&
+            [status $R($master_id) master_repl_offset] == [status $R(2) master_repl_offset] &&
+            [status $R($master_id) master_repl_offset] == [status $R(3) master_repl_offset] &&
+            [status $R($master_id) master_repl_offset] == [status $R(4) master_repl_offset]
+        } else {
+            puts "Master ID is $master_id"
+            for {set j 0} {$j < 5} {incr j} {
+                puts "$j: sync_full: [status $R($j) sync_full]"
+                puts "$j: id1      : [status $R($j) master_replid]:[status $R($j) master_repl_offset]"
+                puts "$j: id2      : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]"
+                puts "$j: backlog  : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]"
+                puts "---"
+            }
+            fail "Replicas and master offsets were unable to match *exactly*."
+        }
+        $R($master_id) config set repl-ping-replica-period 10
+
         # Limit anyway the maximum number of cycles. This is useful when the
         # test is skipped via --only option of the test suite. In that case
         # we don't want to see many seconds of this test being just skipped.

From 75c11d7fec3098f0781154284dc851ae251262d1 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 18 May 2020 10:01:30 +0300
Subject: [PATCH 0384/1098] fix valgrind test failure in replication test

in b4416280c i added more keys to that test to make it run longer
but in valgrind this now means the test times out, give valgrind more
time.
---
 tests/integration/replication.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl
index e5079d9dd..7c03c4bc6 100644
--- a/tests/integration/replication.tcl
+++ b/tests/integration/replication.tcl
@@ -612,7 +612,7 @@ start_server {tags {"repl"}} {
                         # Wait that replicas acknowledge they are online so
                         # we are sure that DBSIZE and DEBUG DIGEST will not
                         # fail because of timing issues.
-                        wait_for_condition 50 100 {
+                        wait_for_condition 150 100 {
                             [lindex [$replica role] 3] eq {connected}
                         } else {
                             fail "replicas still not connected after some time"

From abff264000a9926bb7b96172fd49dae3b804c4e0 Mon Sep 17 00:00:00 2001
From: ShooterIT <wangyuancode@163.com>
Date: Mon, 18 May 2020 18:18:20 +0800
Subject: [PATCH 0385/1098] Redis Benchmark: generate random test data

The function of generating random data is designed by antirez. See #7196.
---
 src/redis-benchmark.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 2df41580b..19c72551c 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -1269,6 +1269,17 @@ static void updateClusterSlotsConfiguration() {
     pthread_mutex_unlock(&config.is_updating_slots_mutex);
 }
 
+/* Generate random data for redis benchmark. See #7196. */
+static void genBenchmarkRandomData(char *data, int count) {
+    static uint32_t state = 1234;
+    int i = 0;
+
+    while (count--) {
+        state = (state*1103515245+12345);
+        data[i++] = '0'+((state>>16)&63);
+    }
+}
+
 /* Returns number of consumed options. */
 int parseOptions(int argc, const char **argv) {
     int i;
@@ -1619,7 +1630,7 @@ int main(int argc, const char **argv) {
     /* Run default benchmark suite. */
     data = zmalloc(config.datasize+1);
     do {
-        memset(data,'x',config.datasize);
+        genBenchmarkRandomData(data, config.datasize);
         data[config.datasize] = '\0';
 
         if (test_is_selected("ping_inline") || test_is_selected("ping"))

From edc1f7b176af202794a23dbb679128d09f16740a Mon Sep 17 00:00:00 2001
From: hujie <hujiecs@qq.com>
Date: Tue, 19 May 2020 00:58:58 +0800
Subject: [PATCH 0386/1098] fix clear USER_FLAG_ALLCOMMANDS flag in acl

in ACLSetUserCommandBit, when the command bit overflows, no operation
is performed, so no need clear the USER_FLAG_ALLCOMMANDS flag.

in ACLSetUser, when adding subcommand, we don't need to call
ACLGetCommandID ahead since subcommand may be empty.
---
 src/acl.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index bcca116bb..c3a724c33 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -375,12 +375,13 @@ int ACLUserCanExecuteFutureCommands(user *u) {
  * to skip the command bit explicit test. */
 void ACLSetUserCommandBit(user *u, unsigned long id, int value) {
     uint64_t word, bit;
-    if (value == 0) u->flags &= ~USER_FLAG_ALLCOMMANDS;
     if (ACLGetCommandBitCoordinates(id,&word,&bit) == C_ERR) return;
-    if (value)
+    if (value) {
         u->allowed_commands[word] |= bit;
-    else
+    } else {
         u->allowed_commands[word] &= ~bit;
+        u->flags &= ~USER_FLAG_ALLCOMMANDS;
+    }
 }
 
 /* This is like ACLSetUserCommandBit(), but instead of setting the specified
@@ -845,7 +846,6 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
                 errno = ENOENT;
                 return C_ERR;
             }
-            unsigned long id = ACLGetCommandID(copy);
 
             /* The subcommand cannot be empty, so things like DEBUG|
              * are syntax errors of course. */
@@ -858,6 +858,7 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
             /* The command should not be set right now in the command
              * bitmap, because adding a subcommand of a fully added
              * command is probably an error on the user side. */
+            unsigned long id = ACLGetCommandID(copy);
             if (ACLGetUserCommandBit(u,id) == 1) {
                 zfree(copy);
                 errno = EBUSY;

From 07e1417ef01594bb007c63f0390ed63077b8c744 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Mon, 18 May 2020 22:10:57 -0400
Subject: [PATCH 0387/1098] Redis-Benchmark: avoid potentical memmory leaking

---
 src/redis-benchmark.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 34295147d..46c1853f7 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -303,7 +303,7 @@ fail:
     else fprintf(stderr, "%s\n", hostsocket);
     freeReplyObject(reply);
     redisFree(c);
-    zfree(cfg);
+    freeRedisConfig(cfg);
     return NULL;
 }
 static void freeRedisConfig(redisConfig *cfg) {

From 5d83e9e1de94fb0250073565c051e71b7648d42d Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 17 May 2020 15:10:25 +0300
Subject: [PATCH 0388/1098] improve DEBUG MALLCTL to be able to write to write
 only fields.

also support:
  debug mallctl-str thread.tcache.flush VOID
---
 src/debug.c | 34 +++++++++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/src/debug.c b/src/debug.c
index 587ff7c5d..d79226bf2 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -311,6 +311,13 @@ void mallctl_int(client *c, robj **argv, int argc) {
     size_t sz = sizeof(old);
     while (sz > 0) {
         if ((ret=je_mallctl(argv[0]->ptr, &old, &sz, argc > 1? &val: NULL, argc > 1?sz: 0))) {
+            if (ret == EPERM && argc > 1) {
+                /* if this option is write only, try just writing to it. */
+                if (!(ret=je_mallctl(argv[0]->ptr, NULL, 0, &val, sz))) {
+                    addReply(c, shared.ok);
+                    return;
+                }
+            }
             if (ret==EINVAL) {
                 /* size might be wrong, try a smaller one */
                 sz /= 2;
@@ -333,17 +340,30 @@ void mallctl_int(client *c, robj **argv, int argc) {
 }
 
 void mallctl_string(client *c, robj **argv, int argc) {
-    int ret;
+    int rret, wret;
     char *old;
     size_t sz = sizeof(old);
     /* for strings, it seems we need to first get the old value, before overriding it. */
-    if ((ret=je_mallctl(argv[0]->ptr, &old, &sz, NULL, 0))) {
-        addReplyErrorFormat(c,"%s", strerror(ret));
-        return;
+    if ((rret=je_mallctl(argv[0]->ptr, &old, &sz, NULL, 0))) {
+        /* return error unless this option is write only. */
+        if (!(rret == EPERM && argc > 1)) {
+            addReplyErrorFormat(c,"%s", strerror(rret));
+            return;
+        }
     }
-    addReplyBulkCString(c, old);
-    if(argc > 1)
-        je_mallctl(argv[0]->ptr, NULL, 0, &argv[1]->ptr, sizeof(char*));
+    if(argc > 1) {
+        char *val = argv[1]->ptr;
+        char **valref = &val;
+        if ((!strcmp(val,"VOID")))
+            valref = NULL, sz = 0;
+        wret = je_mallctl(argv[0]->ptr, NULL, 0, valref, sz);
+    }
+    if (!rret)
+        addReplyBulkCString(c, old);
+    else if (wret)
+        addReplyErrorFormat(c,"%s", strerror(wret));
+    else
+        addReply(c, shared.ok);
 }
 #endif
 

From 39d1e392121caeac07c3f784ca12524a5e28bc69 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 20 May 2020 15:14:23 +0300
Subject: [PATCH 0389/1098] TLS: Improve tls-protocols clarity in redis.conf.

---
 redis.conf | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/redis.conf b/redis.conf
index a469d8b61..5cea06a47 100644
--- a/redis.conf
+++ b/redis.conf
@@ -176,9 +176,10 @@ tcp-keepalive 300
 # tls-cluster yes
 
 # Explicitly specify TLS versions to support. Allowed values are case insensitive
-# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) 
+# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) or
+# any combination. To enable only TLSv1.2 and TLSv1.3, use:
 #
-# tls-protocols TLSv1.2
+# tls-protocols "TLSv1.2 TLSv1.3"
 
 # Configure allowed ciphers.  See the ciphers(1ssl) manpage for more information
 # about the syntax of this string.

From 88d71f479338c1a70fac15ea37f87315f9401f99 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 20 May 2020 14:08:40 +0300
Subject: [PATCH 0390/1098] fix a rare active defrag edge case bug leading to
 stagnation

There's a rare case which leads to stagnation in the defragger, causing
it to keep scanning the keyspace and do nothing (not moving any
allocation), this happens when all the allocator slabs of a certain bin
have the same % utilization, but the slab from which new allocations are
made have a lower utilization.

this commit fixes it by removing the current slab from the overall
average utilization of the bin, and also eliminate any precision loss in
the utilization calculation and move the decision about the defrag to
reside inside jemalloc.

and also add a test that consistently reproduce this issue.
---
 .../internal/jemalloc_internal_inlines_c.h    |  23 +++-
 deps/jemalloc/src/jemalloc.c                  |   8 +-
 src/defrag.c                                  |  13 +-
 tests/unit/memefficiency.tcl                  | 125 +++++++++++++++++-
 4 files changed, 146 insertions(+), 23 deletions(-)

diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h
index 290e5cf99..2685802b8 100644
--- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h
+++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h
@@ -216,7 +216,7 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra,
 }
 
 JEMALLOC_ALWAYS_INLINE int
-iget_defrag_hint(tsdn_t *tsdn, void* ptr, int *bin_util, int *run_util) {
+iget_defrag_hint(tsdn_t *tsdn, void* ptr) {
 	int defrag = 0;
 	rtree_ctx_t rtree_ctx_fallback;
 	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
@@ -232,11 +232,22 @@ iget_defrag_hint(tsdn_t *tsdn, void* ptr, int *bin_util, int *run_util) {
 		malloc_mutex_lock(tsdn, &bin->lock);
 		/* don't bother moving allocations from the slab currently used for new allocations */
 		if (slab != bin->slabcur) {
-			const bin_info_t *bin_info = &bin_infos[binind];
-			size_t availregs = bin_info->nregs * bin->stats.curslabs;
-			*bin_util = ((long long)bin->stats.curregs<<16) / availregs;
-			*run_util = ((long long)(bin_info->nregs - extent_nfree_get(slab))<<16) / bin_info->nregs;
-			defrag = 1;
+			int free_in_slab = extent_nfree_get(slab);
+			if (free_in_slab) {
+				const bin_info_t *bin_info = &bin_infos[binind];
+				int curslabs = bin->stats.curslabs;
+				size_t curregs = bin->stats.curregs;
+				if (bin->slabcur) {
+					/* remove slabcur from the overall utilization */
+					curregs -= bin_info->nregs - extent_nfree_get(bin->slabcur);
+					curslabs -= 1;
+				}
+				/* Compare the utilization ratio of the slab in question to the total average,
+				 * to avoid precision lost and division, we do that by extrapolating the usage
+				 * of the slab as if all slabs have the same usage. If this slab is less used 
+				 * than the average, we'll prefer to evict the data to hopefully more used ones */
+				defrag = (bin_info->nregs - free_in_slab) * curslabs <= curregs;
+			}
 		}
 		malloc_mutex_unlock(tsdn, &bin->lock);
 	}
diff --git a/deps/jemalloc/src/jemalloc.c b/deps/jemalloc/src/jemalloc.c
index 5b936cb48..585645a28 100644
--- a/deps/jemalloc/src/jemalloc.c
+++ b/deps/jemalloc/src/jemalloc.c
@@ -3326,12 +3326,10 @@ jemalloc_postfork_child(void) {
 /******************************************************************************/
 
 /* Helps the application decide if a pointer is worth re-allocating in order to reduce fragmentation.
- * returns 0 if the allocation is in the currently active run,
- * or when it is not causing any frag issue (large or huge bin)
- * returns the bin utilization and run utilization both in fixed point 16:16.
+ * returns 1 if the allocation should be moved, and 0 if the allocation be kept.
  * If the application decides to re-allocate it should use MALLOCX_TCACHE_NONE when doing so. */
 JEMALLOC_EXPORT int JEMALLOC_NOTHROW
-get_defrag_hint(void* ptr, int *bin_util, int *run_util) {
+get_defrag_hint(void* ptr) {
 	assert(ptr != NULL);
-	return iget_defrag_hint(TSDN_NULL, ptr, bin_util, run_util);
+	return iget_defrag_hint(TSDN_NULL, ptr);
 }
diff --git a/src/defrag.c b/src/defrag.c
index e729297a5..6e5296632 100644
--- a/src/defrag.c
+++ b/src/defrag.c
@@ -43,7 +43,7 @@
 
 /* this method was added to jemalloc in order to help us understand which
  * pointers are worthwhile moving and which aren't */
-int je_get_defrag_hint(void* ptr, int *bin_util, int *run_util);
+int je_get_defrag_hint(void* ptr);
 
 /* forward declarations*/
 void defragDictBucketCallback(void *privdata, dictEntry **bucketref);
@@ -55,18 +55,11 @@ dictEntry* replaceSateliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sd
  * when it returns a non-null value, the old pointer was already released
  * and should NOT be accessed. */
 void* activeDefragAlloc(void *ptr) {
-    int bin_util, run_util;
     size_t size;
     void *newptr;
-    if(!je_get_defrag_hint(ptr, &bin_util, &run_util)) {
-        server.stat_active_defrag_misses++;
-        return NULL;
-    }
-    /* if this run is more utilized than the average utilization in this bin
-     * (or it is full), skip it. This will eventually move all the allocations
-     * from relatively empty runs into relatively full runs. */
-    if (run_util > bin_util || run_util == 1<<16) {
+    if(!je_get_defrag_hint(ptr)) {
         server.stat_active_defrag_misses++;
+        size = zmalloc_size(ptr);
         return NULL;
     }
     /* move this allocation to a new allocation.
diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl
index 777693fdf..390bad192 100644
--- a/tests/unit/memefficiency.tcl
+++ b/tests/unit/memefficiency.tcl
@@ -95,6 +95,10 @@ start_server {tags {"defrag"}} {
                 }
                 if {$::verbose} {
                     puts "frag $frag"
+                    set misses [s active_defrag_misses]
+                    set hits [s active_defrag_hits]
+                    puts "hits: $hits"
+                    puts "misses: $misses"
                     puts "max latency $max_latency"
                     puts [r latency latest]
                     puts [r latency history active-defrag-cycle]
@@ -221,6 +225,10 @@ start_server {tags {"defrag"}} {
                 }
                 if {$::verbose} {
                     puts "frag $frag"
+                    set misses [s active_defrag_misses]
+                    set hits [s active_defrag_hits]
+                    puts "hits: $hits"
+                    puts "misses: $misses"
                     puts "max latency $max_latency"
                     puts [r latency latest]
                     puts [r latency history active-defrag-cycle]
@@ -256,11 +264,12 @@ start_server {tags {"defrag"}} {
             set expected_frag 1.7
             # add a mass of list nodes to two lists (allocations are interlaced)
             set val [string repeat A 100] ;# 5 items of 100 bytes puts us in the 640 bytes bin, which has 32 regs, so high potential for fragmentation
-            for {set j 0} {$j < 500000} {incr j} {
+            set elements 500000
+            for {set j 0} {$j < $elements} {incr j} {
                 $rd lpush biglist1 $val
                 $rd lpush biglist2 $val
             }
-            for {set j 0} {$j < 500000} {incr j} {
+            for {set j 0} {$j < $elements} {incr j} {
                 $rd read ; # Discard replies
                 $rd read ; # Discard replies
             }
@@ -302,6 +311,8 @@ start_server {tags {"defrag"}} {
 
                 # test the the fragmentation is lower
                 after 120 ;# serverCron only updates the info once in 100ms
+                set misses [s active_defrag_misses]
+                set hits [s active_defrag_hits]
                 set frag [s allocator_frag_ratio]
                 set max_latency 0
                 foreach event [r latency latest] {
@@ -312,6 +323,8 @@ start_server {tags {"defrag"}} {
                 }
                 if {$::verbose} {
                     puts "frag $frag"
+                    puts "misses: $misses"
+                    puts "hits: $hits"
                     puts "max latency $max_latency"
                     puts [r latency latest]
                     puts [r latency history active-defrag-cycle]
@@ -320,6 +333,10 @@ start_server {tags {"defrag"}} {
                 # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75,
                 # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher
                 assert {$max_latency <= 30}
+
+                # in extreme cases of stagnation, we see over 20m misses before the tests aborts with "defrag didn't stop",
+                # in normal cases we only see 100k misses out of 500k elements
+                assert {$misses < $elements}
             }
             # verify the data isn't corrupted or changed
             set newdigest [r debug digest]
@@ -327,6 +344,110 @@ start_server {tags {"defrag"}} {
             r save ;# saving an rdb iterates over all the data / pointers
             r del biglist1 ;# coverage for quicklistBookmarksClear
         } {1}
+
+        test "Active defrag edge case" {
+            # there was an edge case in defrag where all the slabs of a certain bin are exact the same
+            # % utilization, with the exception of the current slab from which new allocations are made
+            # if the current slab is lower in utilization the defragger would have ended up in stagnation,
+            # keept running and not move any allocation.
+            # this test is more consistent on a fresh server with no history
+            start_server {tags {"defrag"}} {
+                r flushdb
+                r config resetstat
+                r config set save "" ;# prevent bgsave from interfereing with save below
+                r config set hz 100
+                r config set activedefrag no
+                r config set active-defrag-max-scan-fields 1000
+                r config set active-defrag-threshold-lower 5
+                r config set active-defrag-cycle-min 65
+                r config set active-defrag-cycle-max 75
+                r config set active-defrag-ignore-bytes 1mb
+                r config set maxmemory 0
+                set expected_frag 1.3
+
+                r debug mallctl-str thread.tcache.flush VOID
+                # fill the first slab containin 32 regs of 640 bytes.
+                for {set j 0} {$j < 32} {incr j} {
+                    r setrange "_$j" 600 x
+                    r debug mallctl-str thread.tcache.flush VOID
+                }
+
+                # add a mass of keys with 600 bytes values, fill the bin of 640 bytes which has 32 regs per slab.
+                set rd [redis_deferring_client]
+                set keys 640000
+                for {set j 0} {$j < $keys} {incr j} {
+                    $rd setrange $j 600 x
+                }
+                for {set j 0} {$j < $keys} {incr j} {
+                    $rd read ; # Discard replies
+                }
+
+                # create some fragmentation of 50%
+                set sent 0
+                for {set j 0} {$j < $keys} {incr j 1} {
+                    $rd del $j
+                    incr sent
+                    incr j 1
+                }
+                for {set j 0} {$j < $sent} {incr j} {
+                    $rd read ; # Discard replies
+                }
+
+                # create higher fragmentation in the first slab
+                for {set j 10} {$j < 32} {incr j} {
+                    r del "_$j"
+                }
+
+                # start defrag
+                after 120 ;# serverCron only updates the info once in 100ms
+                set frag [s allocator_frag_ratio]
+                if {$::verbose} {
+                    puts "frag $frag"
+                }
+
+                assert {$frag >= $expected_frag}
+
+                set digest [r debug digest]
+                catch {r config set activedefrag yes} e
+                if {![string match {DISABLED*} $e]} {
+                    # wait for the active defrag to start working (decision once a second)
+                    wait_for_condition 50 100 {
+                        [s active_defrag_running] ne 0
+                    } else {
+                        fail "defrag not started."
+                    }
+
+                    # wait for the active defrag to stop working
+                    wait_for_condition 500 100 {
+                        [s active_defrag_running] eq 0
+                    } else {
+                        after 120 ;# serverCron only updates the info once in 100ms
+                        puts [r info memory]
+                        puts [r info stats]
+                        puts [r memory malloc-stats]
+                        fail "defrag didn't stop."
+                    }
+
+                    # test the the fragmentation is lower
+                    after 120 ;# serverCron only updates the info once in 100ms
+                    set misses [s active_defrag_misses]
+                    set hits [s active_defrag_hits]
+                    set frag [s allocator_frag_ratio]
+                    if {$::verbose} {
+                        puts "frag $frag"
+                        puts "hits: $hits"
+                        puts "misses: $misses"
+                    }
+                    assert {$frag < 1.1}
+                    assert {$misses < 10000000} ;# when defrag doesn't stop, we have some 30m misses, when it does, we have 2m misses
+                }
+
+                # verify the data isn't corrupted or changed
+                set newdigest [r debug digest]
+                assert {$digest eq $newdigest}
+                r save ;# saving an rdb iterates over all the data / pointers
+            }
+        }
     }
 }
 } ;# run_solo

From 4f3ff46a818b665b314d637f9aabf6ab602cf014 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Thu, 21 May 2020 13:57:29 +0800
Subject: [PATCH 0391/1098] Tracking: flag CLIENT_TRACKING_BROKEN_REDIR when
 redir broken

---
 src/tracking.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/tracking.c b/src/tracking.c
index cfde26fc9..eb4113131 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -206,6 +206,7 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen, int proto) {
     if (c->client_tracking_redirection) {
         client *redir = lookupClientByID(c->client_tracking_redirection);
         if (!redir) {
+            c->flags |= CLIENT_TRACKING_BROKEN_REDIR;
             /* We need to signal to the original connection that we
              * are unable to send invalidation messages to the redirected
              * connection, because the client no longer exist. */

From 9018ddc32a2246fe806eba96bf13bb461c8000fc Mon Sep 17 00:00:00 2001
From: ShooterIT <wangyuancode@163.com>
Date: Thu, 21 May 2020 21:00:21 +0800
Subject: [PATCH 0392/1098] Fix reply bytes calculation error

Fix #7275.
---
 src/networking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index 0a04c0489..e6c48ac77 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -488,7 +488,7 @@ void trimReplyUnusedTailSpace(client *c) {
         /* take over the allocation's internal fragmentation (at least for
          * memory usage tracking) */
         tail->size = zmalloc_usable(tail) - sizeof(clientReplyBlock);
-        c->reply_bytes += tail->size - old_size;
+        c->reply_bytes = c->reply_bytes + tail->size - old_size;
         listNodeValue(ln) = tail;
     }
 }

From 86f0e873c705f37b29af161c7a6216397d518542 Mon Sep 17 00:00:00 2001
From: ShooterIT <wangyuancode@163.com>
Date: Thu, 21 May 2020 21:45:35 +0800
Subject: [PATCH 0393/1098] Replace addDeferredMultiBulkLength with
 addReplyDeferredLen in comment

---
 src/networking.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 0a04c0489..f4461a744 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -268,7 +268,7 @@ void _addReplyProtoToList(client *c, const char *s, size_t len) {
     clientReplyBlock *tail = ln? listNodeValue(ln): NULL;
 
     /* Note that 'tail' may be NULL even if we have a tail node, becuase when
-     * addDeferredMultiBulkLength() is used, it sets a dummy node to NULL just
+     * addReplyDeferredLen() is used, it sets a dummy node to NULL just
      * fo fill it later, when the size of the bulk length is set. */
 
     /* Append to tail string when possible. */
@@ -473,7 +473,7 @@ void trimReplyUnusedTailSpace(client *c) {
     clientReplyBlock *tail = ln? listNodeValue(ln): NULL;
 
     /* Note that 'tail' may be NULL even if we have a tail node, becuase when
-     * addDeferredMultiBulkLength() is used */
+     * addReplyDeferredLen() is used */
     if (!tail) return;
 
     /* We only try to trim the space is relatively high (more than a 1/4 of the

From 5edb1beb636ba81b09c7f13853e517d09499495b Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Thu, 21 May 2020 17:30:36 -0400
Subject: [PATCH 0394/1098] fix server crash for STRALGO command

---
 src/t_string.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index d4eb04769..d46900a7a 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -519,7 +519,7 @@ void stralgoLCS(client *c) {
                 != C_OK) return;
             if (minmatchlen < 0) minmatchlen = 0;
             j++;
-        } else if (!strcasecmp(opt,"STRINGS")) {
+        } else if (!strcasecmp(opt,"STRINGS") && (c->argc-j) > 2) {
             if (a != NULL) {
                 addReplyError(c,"Either use STRINGS or KEYS");
                 return;
@@ -527,7 +527,7 @@ void stralgoLCS(client *c) {
             a = c->argv[j+1]->ptr;
             b = c->argv[j+2]->ptr;
             j += 2;
-        } else if (!strcasecmp(opt,"KEYS")) {
+        } else if (!strcasecmp(opt,"KEYS") && (c->argc-j) > 2) {
             if (a != NULL) {
                 addReplyError(c,"Either use STRINGS or KEYS");
                 return;

From 06a59fd44d1f8beda2b0868d018b3f2f3fb1e08e Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Thu, 21 May 2020 18:13:35 -0400
Subject: [PATCH 0395/1098] using moreargs variable

---
 src/t_string.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index d46900a7a..5306069bf 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -519,7 +519,7 @@ void stralgoLCS(client *c) {
                 != C_OK) return;
             if (minmatchlen < 0) minmatchlen = 0;
             j++;
-        } else if (!strcasecmp(opt,"STRINGS") && (c->argc-j) > 2) {
+        } else if (!strcasecmp(opt,"STRINGS") && moreargs > 1) {
             if (a != NULL) {
                 addReplyError(c,"Either use STRINGS or KEYS");
                 return;
@@ -527,7 +527,7 @@ void stralgoLCS(client *c) {
             a = c->argv[j+1]->ptr;
             b = c->argv[j+2]->ptr;
             j += 2;
-        } else if (!strcasecmp(opt,"KEYS") && (c->argc-j) > 2) {
+        } else if (!strcasecmp(opt,"KEYS") && moreargs > 1) {
             if (a != NULL) {
                 addReplyError(c,"Either use STRINGS or KEYS");
                 return;

From 5109f16b7742866251701d61a7a7cbe5fd2d261c Mon Sep 17 00:00:00 2001
From: Madelyn Olson <matolson@amazon.com>
Date: Thu, 21 May 2020 15:20:59 -0700
Subject: [PATCH 0396/1098] EAGAIN for tls during diskless load

---
 src/replication.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/replication.c b/src/replication.c
index b10635f25..6483d296c 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1526,6 +1526,10 @@ void readSyncBulkPayload(connection *conn) {
 
         nread = connRead(conn,buf,readlen);
         if (nread <= 0) {
+            if (connGetState(conn) == CONN_STATE_CONNECTED) {
+                /* equivalent to EAGAIN */
+                return;
+            }
             serverLog(LL_WARNING,"I/O error trying to sync with MASTER: %s",
                 (nread == -1) ? strerror(errno) : "connection lost");
             cancelReplicationHandshake();

From 42f5da5d2d373b1a016d725b2b7111288949a28e Mon Sep 17 00:00:00 2001
From: Qu Chen <quchen@amazon.com>
Date: Thu, 21 May 2020 18:39:38 -0700
Subject: [PATCH 0397/1098] Disconnect chained replicas when the replica
 performs PSYNC with the master always to avoid replication offset mismatch
 between master and chained replicas.

---
 src/replication.c                    | 13 ++++++--
 tests/integration/psync2-pingoff.tcl | 50 ++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index b10635f25..7eb161f7d 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2023,12 +2023,19 @@ int slaveTryPartialResynchronization(connection *conn, int read_reply) {
                  * new one. */
                 memcpy(server.replid,new,sizeof(server.replid));
                 memcpy(server.cached_master->replid,new,sizeof(server.replid));
-
-                /* Disconnect all the sub-slaves: they need to be notified. */
-                disconnectSlaves();
             }
         }
 
+        /* Disconnect all the sub-replicas: they need to be notified always because
+         * in case the master has last replicated some non-meaningful commands
+         * (e.g. PINGs), the primary replica will request the PSYNC offset for the
+         * last known meaningful command. This means the master will again replicate
+         * the non-meaningful commands. If the sub-replicas still remains connected,
+         * they will receive those commands a second time and increment the master
+         * replication offset to be higher than the master's offset forever.
+         */
+        disconnectSlaves();
+
         /* Setup the replication to continue. */
         sdsfree(reply);
         replicationResurrectCachedMaster(conn);
diff --git a/tests/integration/psync2-pingoff.tcl b/tests/integration/psync2-pingoff.tcl
index 420747d21..2c2303141 100644
--- a/tests/integration/psync2-pingoff.tcl
+++ b/tests/integration/psync2-pingoff.tcl
@@ -60,3 +60,53 @@ start_server {} {
         }
     }
 }}
+
+
+start_server {tags {"psync2"}} {
+start_server {} {
+start_server {} {
+
+    for {set j 0} {$j < 3} {incr j} {
+        set R($j) [srv [expr 0-$j] client]
+        set R_host($j) [srv [expr 0-$j] host]
+        set R_port($j) [srv [expr 0-$j] port]
+        $R($j) CONFIG SET repl-ping-replica-period 1
+    }
+
+    test "Chained replicas disconnect when replica re-connect with the same master" {
+        # Add a second replica as a chained replica of the current replica
+        $R(1) replicaof $R_host(0) $R_port(0)
+        $R(2) replicaof $R_host(1) $R_port(1)
+        wait_for_condition 50 1000 {
+            [status $R(2) master_link_status] == "up"
+        } else {
+            fail "Chained replica not replicating from its master"
+        }
+
+        # Do a write on the master, and wait for 3 seconds for the master to
+        # send some PINGs to its replica
+        $R(0) INCR counter2
+        after 2000
+        set sync_partial_master [status $R(0) sync_partial_ok]
+        set sync_partial_replica [status $R(1) sync_partial_ok]
+        $R(0) CONFIG SET repl-ping-replica-period 100
+
+        # Disconnect the master's direct replica
+        $R(0) client kill type replica
+        wait_for_condition 50 1000 {
+            [status $R(1) master_link_status] == "up" && 
+            [status $R(2) master_link_status] == "up" &&
+            [status $R(0) sync_partial_ok] == $sync_partial_master + 1 &&
+            [status $R(1) sync_partial_ok] == $sync_partial_replica + 1
+        } else {
+            fail "Disconnected replica failed to PSYNC with master"
+        }
+
+        # Verify that the replica and its replica's meaningful and real
+        # offsets match with the master
+        assert_equal [status $R(0) master_repl_offset] [status $R(1) master_repl_offset]
+        assert_equal [status $R(0) master_repl_offset] [status $R(2) master_repl_offset]
+        assert_equal [status $R(0) master_repl_meaningful_offset] [status $R(1) master_repl_meaningful_offset]
+        assert_equal [status $R(0) master_repl_meaningful_offset] [status $R(2) master_repl_meaningful_offset]
+    }
+}}}

From b407590cee4988a1cde3afe7ec9b8ff0243462ef Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 22 May 2020 15:59:02 +0200
Subject: [PATCH 0398/1098] Fix #7306 less aggressively.

Citing from the issue:

btw I suggest we change this fix to something else:
* We revert the fix.
* We add a call that disconnects chained replicas in the place where we trim the replica (that is a master i this case) offset.
This way we can avoid disconnections when there is no trimming of the backlog.

Note that we now want to disconnect replicas asynchronously in
disconnectSlaves(), because it's in general safer now that we can call
it from freeClient(). Otherwise for instance the command:

    CLIENT KILL TYPE master

May crash: clientCommand() starts running the linked of of clients,
looking for clients to kill. However it finds the master, kills it
calling freeClient(), but this in turn calls replicationCacheMaster()
that may also call disconnectSlaves() now. So the linked list iterator
of the clientCommand() will no longer be valid.
---
 src/networking.c  |  7 +++++--
 src/replication.c | 39 ++++++++++++++++++++++++---------------
 2 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index f3362aba4..a1948ce60 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1039,9 +1039,12 @@ static void freeClientArgv(client *c) {
  * when we resync with our own master and want to force all our slaves to
  * resync with us as well. */
 void disconnectSlaves(void) {
-    while (listLength(server.slaves)) {
+    listIter li;
+    listNode *ln;
+    listRewind(server.slaves,&li);
+    while((ln = listNext(&li))) {
         listNode *ln = listFirst(server.slaves);
-        freeClient((client*)ln->value);
+        freeClientAsync((client*)ln->value);
     }
 }
 
diff --git a/src/replication.c b/src/replication.c
index 7e981873f..603ef4895 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -39,7 +39,7 @@
 #include <sys/socket.h>
 #include <sys/stat.h>
 
-long long adjustMeaningfulReplOffset();
+long long adjustMeaningfulReplOffset(int *adjusted);
 void replicationDiscardCachedMaster(void);
 void replicationResurrectCachedMaster(connection *conn);
 void replicationSendAck(void);
@@ -2027,19 +2027,12 @@ int slaveTryPartialResynchronization(connection *conn, int read_reply) {
                  * new one. */
                 memcpy(server.replid,new,sizeof(server.replid));
                 memcpy(server.cached_master->replid,new,sizeof(server.replid));
+
+                /* Disconnect all the sub-slaves: they need to be notified. */
+                disconnectSlaves();
             }
         }
 
-        /* Disconnect all the sub-replicas: they need to be notified always because
-         * in case the master has last replicated some non-meaningful commands
-         * (e.g. PINGs), the primary replica will request the PSYNC offset for the
-         * last known meaningful command. This means the master will again replicate
-         * the non-meaningful commands. If the sub-replicas still remains connected,
-         * they will receive those commands a second time and increment the master
-         * replication offset to be higher than the master's offset forever.
-         */
-        disconnectSlaves();
-
         /* Setup the replication to continue. */
         sdsfree(reply);
         replicationResurrectCachedMaster(conn);
@@ -2708,7 +2701,8 @@ void replicationCacheMaster(client *c) {
 
     /* Adjust reploff and read_reploff to the last meaningful offset we
      * executed. This is the offset the replica will use for future PSYNC. */
-    server.master->reploff = adjustMeaningfulReplOffset();
+    int offset_adjusted;
+    server.master->reploff = adjustMeaningfulReplOffset(&offset_adjusted);
     server.master->read_reploff = server.master->reploff;
     if (c->flags & CLIENT_MULTI) discardTransaction(c);
     listEmpty(c->reply);
@@ -2731,6 +2725,13 @@ void replicationCacheMaster(client *c) {
      * so make sure to adjust the replication state. This function will
      * also set server.master to NULL. */
     replicationHandleMasterDisconnection();
+
+    /* If we trimmed this replica backlog, we need to disconnect our chained
+     * replicas (if any), otherwise they may have the PINGs we removed
+     * from the stream and their offset would no longer match: upon
+     * disconnection they will also trim the final PINGs and will be able
+     * to incrementally sync without issues. */
+    if (offset_adjusted) disconnectSlaves();
 }
 
 /* If the "meaningful" offset, that is the offset without the final PINGs
@@ -2740,8 +2741,13 @@ void replicationCacheMaster(client *c) {
  * offset because of the PINGs and will not be able to incrementally
  * PSYNC with the new master.
  * This function trims the replication backlog when needed, and returns
- * the offset to be used for future partial sync. */
-long long adjustMeaningfulReplOffset() {
+ * the offset to be used for future partial sync.
+ *
+ * If the integer 'adjusted' was passed by reference, it is set to 1
+ * if the function call actually modified the offset and the replication
+ * backlog, otherwise it is set to 0. It can be NULL if the caller is
+ * not interested in getting this info. */
+long long adjustMeaningfulReplOffset(int *adjusted) {
     if (server.master_repl_offset > server.master_repl_meaningful_offset) {
         long long delta = server.master_repl_offset -
                           server.master_repl_meaningful_offset;
@@ -2761,6 +2767,9 @@ long long adjustMeaningfulReplOffset() {
                 (server.repl_backlog_idx + (server.repl_backlog_size - delta)) %
                 server.repl_backlog_size;
         }
+        if (adjusted) *adjusted = 1;
+    } else {
+        if (adjusted) *adjusted = 0;
     }
     return server.master_repl_offset;
 }
@@ -2784,7 +2793,7 @@ void replicationCacheMasterUsingMyself(void) {
      * by replicationCreateMasterClient(). We'll later set the created
      * master as server.cached_master, so the replica will use such
      * offset for PSYNC. */
-    server.master_initial_offset = adjustMeaningfulReplOffset();
+    server.master_initial_offset = adjustMeaningfulReplOffset(NULL);
 
     /* The master client we create can be set to any DBID, because
      * the new master will start its replication stream with SELECT. */

From adc5df1bc3289cb1d786e0ef515ab9acde03bf52 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 22 May 2020 19:29:09 +0200
Subject: [PATCH 0399/1098] Make disconnectSlaves() synchronous in the base
 case.

Otherwise we run into that:

Backtrace:
src/redis-server 127.0.0.1:21322(logStackTrace+0x45)[0x479035]
src/redis-server 127.0.0.1:21322(sigsegvHandler+0xb9)[0x4797f9]
/lib/x86_64-linux-gnu/libpthread.so.0(+0x11390)[0x7fd373c5e390]
src/redis-server 127.0.0.1:21322(_serverAssert+0x6a)[0x47660a]
src/redis-server 127.0.0.1:21322(freeReplicationBacklog+0x42)[0x451282]
src/redis-server 127.0.0.1:21322[0x4552d4]
src/redis-server 127.0.0.1:21322[0x4c5593]
src/redis-server 127.0.0.1:21322(aeProcessEvents+0x2e6)[0x42e786]
src/redis-server 127.0.0.1:21322(aeMain+0x1d)[0x42eb0d]
src/redis-server 127.0.0.1:21322(main+0x4c5)[0x42b145]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf0)[0x7fd3738a3830]
src/redis-server 127.0.0.1:21322(_start+0x29)[0x42b409]

Since we disconnect all the replicas and free the replication backlog in
certain replication paths, and the code that will free the replication
backlog expects that no replica is connected.

However we still need to free the replicas asynchronously in certain
cases, as documented in the top comment of disconnectSlaves().
---
 src/networking.c  | 17 ++++++++++++++---
 src/replication.c | 10 +++++-----
 src/server.h      |  2 +-
 3 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index a1948ce60..364654642 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1037,14 +1037,25 @@ static void freeClientArgv(client *c) {
 
 /* Close all the slaves connections. This is useful in chained replication
  * when we resync with our own master and want to force all our slaves to
- * resync with us as well. */
-void disconnectSlaves(void) {
+ * resync with us as well.
+ *
+ * If 'async' is non-zero we free the clients asynchronously. This is needed
+ * when we call this function from a context where in the chain of the
+ * callers somebody is iterating the list of clients. For instance when
+ * CLIENT KILL TYPE master is called, caching the master client may
+ * adjust the meaningful offset of replication, and in turn call
+ * discionectSlaves(). Since CLIENT KILL iterates the clients this is
+ * not safe. */
+void disconnectSlaves(int async) {
     listIter li;
     listNode *ln;
     listRewind(server.slaves,&li);
     while((ln = listNext(&li))) {
         listNode *ln = listFirst(server.slaves);
-        freeClientAsync((client*)ln->value);
+        if (async)
+            freeClientAsync((client*)ln->value);
+        else
+            freeClient((client*)ln->value);
     }
 }
 
diff --git a/src/replication.c b/src/replication.c
index 77422344a..e1c65d48b 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2076,7 +2076,7 @@ int slaveTryPartialResynchronization(connection *conn, int read_reply) {
                 memcpy(server.cached_master->replid,new,sizeof(server.replid));
 
                 /* Disconnect all the sub-slaves: they need to be notified. */
-                disconnectSlaves();
+                disconnectSlaves(0);
             }
         }
 
@@ -2349,7 +2349,7 @@ void syncWithMaster(connection *conn) {
      * as well, if we have any sub-slaves. The master may transfer us an
      * entirely different data set and we have no way to incrementally feed
      * our slaves after that. */
-    disconnectSlaves(); /* Force our slaves to resync with us as well. */
+    disconnectSlaves(0); /* Force our slaves to resync with us as well. */
     freeReplicationBacklog(); /* Don't allow our chained slaves to PSYNC. */
 
     /* Fall back to SYNC if needed. Otherwise psync_result == PSYNC_FULLRESYNC
@@ -2496,7 +2496,7 @@ void replicationSetMaster(char *ip, int port) {
 
     /* Force our slaves to resync with us as well. They may hopefully be able
      * to partially resync with us, but we can notify the replid change. */
-    disconnectSlaves();
+    disconnectSlaves(0);
     cancelReplicationHandshake();
     /* Before destroying our master state, create a cached master using
      * our own parameters, to later PSYNC with the new master. */
@@ -2543,7 +2543,7 @@ void replicationUnsetMaster(void) {
      * of the replication ID change (see shiftReplicationId() call). However
      * the slaves will be able to partially resync with us, so it will be
      * a very fast reconnection. */
-    disconnectSlaves();
+    disconnectSlaves(0);
     server.repl_state = REPL_STATE_NONE;
 
     /* We need to make sure the new master will start the replication stream
@@ -2778,7 +2778,7 @@ void replicationCacheMaster(client *c) {
      * from the stream and their offset would no longer match: upon
      * disconnection they will also trim the final PINGs and will be able
      * to incrementally sync without issues. */
-    if (offset_adjusted) disconnectSlaves();
+    if (offset_adjusted) disconnectSlaves(1);
 }
 
 /* If the "meaningful" offset, that is the offset without the final PINGs
diff --git a/src/server.h b/src/server.h
index f835bf5e9..2d17d69c8 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1660,7 +1660,7 @@ int getClientType(client *c);
 int getClientTypeByName(char *name);
 char *getClientTypeName(int class);
 void flushSlavesOutputBuffers(void);
-void disconnectSlaves(void);
+void disconnectSlaves(int async);
 int listenToPort(int port, int *fds, int *count);
 void pauseClients(mstime_t duration);
 int clientsArePaused(void);

From c1da9420c720ba27c61a6b6317d57463b7a726d8 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 24 May 2020 08:00:12 +0300
Subject: [PATCH 0400/1098] add CI for 32bit build

---
 .github/workflows/ci.yml    | 18 ++++++++++++++++++
 .github/workflows/daily.yml | 16 ++++++++++++++++
 2 files changed, 34 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 551fb2d91..439e3f3df 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3,6 +3,7 @@ name: CI
 on: [push, pull_request]
 
 jobs:
+
   test-ubuntu-latest:
     runs-on: ubuntu-latest
     steps:
@@ -29,3 +30,20 @@ jobs:
     - uses: actions/checkout@v1
     - name: make
       run: make
+
+  biuld-32bit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: make
+      run: |
+        sudo apt-get update && sudo apt-get install libc6-dev-i386
+        make 32bit
+
+  build-libc-malloc:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: make
+      run: make MALLOC=libc
+
diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index b6a9abb68..828268c70 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -5,6 +5,7 @@ on:
     - cron: '0 7 * * *'
 
 jobs:
+
   test-jemalloc:
     runs-on: ubuntu-latest
     timeout-minutes: 1200
@@ -33,6 +34,21 @@ jobs:
     - name: module api test
       run: ./runtest-moduleapi --verbose
 
+  test-32bit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: make
+      run: |
+        sudo apt-get update && sudo apt-get install libc6-dev-i386
+        make 32bit
+    - name: test
+      run: |
+        sudo apt-get install tcl8.5
+        ./runtest --accurate --verbose
+    - name: module api test
+      run: ./runtest-moduleapi --verbose
+
   test-valgrind:
     runs-on: ubuntu-latest
     timeout-minutes: 14400

From eec769be59d2100748bbb45a0e97e7834f490d4a Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Mon, 25 May 2020 11:17:51 +0800
Subject: [PATCH 0401/1098] PSYNC2: second_replid_offset should be real
 meaningful offset

After adjustMeaningfulReplOffset(), all the other related variable
should be updated, including server.second_replid_offset.

Or the old version redis like 5.0 may receive wrong data from
replication stream, cause redis 5.0 can sync with redis 6.0,
but doesn't know meaningful offset.
---
 src/replication.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index e1c65d48b..d45a15783 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2531,14 +2531,14 @@ void replicationUnsetMaster(void) {
 
     sdsfree(server.masterhost);
     server.masterhost = NULL;
+    if (server.master) freeClient(server.master);
+    replicationDiscardCachedMaster();
+    cancelReplicationHandshake();
     /* When a slave is turned into a master, the current replication ID
      * (that was inherited from the master at synchronization time) is
      * used as secondary ID up to the current offset, and a new replication
      * ID is created to continue with a new replication history. */
     shiftReplicationId();
-    if (server.master) freeClient(server.master);
-    replicationDiscardCachedMaster();
-    cancelReplicationHandshake();
     /* Disconnecting all the slaves is required: we need to inform slaves
      * of the replication ID change (see shiftReplicationId() call). However
      * the slaves will be able to partially resync with us, so it will be

From 92a3ff6168864673e806075528574d8b70fa20b0 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 25 May 2020 11:47:38 +0200
Subject: [PATCH 0402/1098] Clarify what is happening in PR #7320.

---
 src/replication.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/replication.c b/src/replication.c
index d45a15783..3ebe451a2 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2537,7 +2537,11 @@ void replicationUnsetMaster(void) {
     /* When a slave is turned into a master, the current replication ID
      * (that was inherited from the master at synchronization time) is
      * used as secondary ID up to the current offset, and a new replication
-     * ID is created to continue with a new replication history. */
+     * ID is created to continue with a new replication history.
+     *
+     * NOTE: this function MUST be called after we call
+     * freeClient(server.master), since there we adjust the replication
+     * offset trimming the final PINGs. See Github issue #7320. */
     shiftReplicationId();
     /* Disconnecting all the slaves is required: we need to inform slaves
      * of the replication ID change (see shiftReplicationId() call). However

From 091fb64681b4670d4103edf055b82c8c4e48eef9 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 25 May 2020 18:37:05 +0200
Subject: [PATCH 0403/1098] Test: PSYNC2 test can now show server logs.

---
 tests/integration/psync2.tcl | 113 +++++++++++++++++++++++++++--------
 1 file changed, 88 insertions(+), 25 deletions(-)

diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl
index d6b2d19d0..29a880f99 100644
--- a/tests/integration/psync2.tcl
+++ b/tests/integration/psync2.tcl
@@ -1,3 +1,75 @@
+
+proc show_cluster_status {} {
+    uplevel 1 {
+        # The following is the regexp we use to match the log line
+        # time info. Logs are in the following form:
+        #
+        # 11296:M 25 May 2020 17:37:14.652 # Server initialized
+        set log_regexp {^[0-9]+:[A-Z] [0-9]+ [A-z]+ [0-9]+ ([0-9:.]+) .*}
+        set repl_regexp {(master|repl|sync|backlog|meaningful|offset)}
+
+        puts "Master ID is $master_id"
+        for {set j 0} {$j < 5} {incr j} {
+            puts "$j: sync_full: [status $R($j) sync_full]"
+            puts "$j: id1      : [status $R($j) master_replid]:[status $R($j) master_repl_offset]"
+            puts "$j: id2      : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]"
+            puts "$j: backlog  : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]"
+            puts "$j: x var is : [$R($j) GET x]"
+            puts "---"
+        }
+
+        # Show the replication logs of every instance, interleaving
+        # them by the log date.
+        #
+        # First: load the lines as lists for each instance.
+        array set log {}
+        for {set j 0} {$j < 5} {incr j} {
+            set fd [open $R_log($j)]
+            while {[gets $fd l] >= 0} {
+                if {[regexp $log_regexp $l] &&
+                    [regexp -nocase $repl_regexp $l]} {
+                    lappend log($j) $l
+                }
+            }
+            close $fd
+        }
+
+        # To interleave the lines, at every step consume the element of
+        # the list with the lowest time and remove it. Do it until
+        # all the lists are empty.
+        #
+        # regexp {^[0-9]+:[A-Z] [0-9]+ [A-z]+ [0-9]+ ([0-9:.]+) .*} $l - logdate
+        while 1 {
+            # Find the log with smallest time.
+            set empty 0
+            set best 0
+            set bestdate {}
+            for {set j 0} {$j < 5} {incr j} {
+                if {[llength $log($j)] == 0} {
+                    incr empty
+                    continue
+                }
+                regexp $log_regexp [lindex $log($j) 0] - date
+                if {$bestdate eq {}} {
+                    set best $j
+                    set bestdate $date
+                } else {
+                    if {[string compare $bestdate $date] > 0} {
+                        set best $j
+                        set bestdate $date
+                    }
+                }
+            }
+            if {$empty == 5} break ; # Our exit condition: no more logs
+
+            # Emit the one with the smallest time (that is the first
+            # event in the time line).
+            puts "\[$best port $R_port($best)\] [lindex $log($best) 0]"
+            set log($best) [lrange $log($best) 1 end]
+        }
+    }
+}
+
 start_server {tags {"psync2"}} {
 start_server {} {
 start_server {} {
@@ -28,6 +100,7 @@ start_server {} {
         set R($j) [srv [expr 0-$j] client]
         set R_host($j) [srv [expr 0-$j] host]
         set R_port($j) [srv [expr 0-$j] port]
+        set R_log($j) [srv [expr 0-$j] stdout]
         if {$debug_msg} {puts "Log file: [srv [expr 0-$j] stdout]"}
     }
 
@@ -74,6 +147,7 @@ start_server {} {
             [status $R([expr {($master_id+3)%5}]) master_link_status] == "up" &&
             [status $R([expr {($master_id+4)%5}]) master_link_status] == "up"
         } else {
+            show_cluster_status
             fail "Replica not reconnecting"
         }
 
@@ -85,6 +159,7 @@ start_server {} {
                 wait_for_condition 50 1000 {
                     [$R($j) get x] == $counter_value
                 } else {
+                    show_cluster_status
                     fail "Instance #$j x variable is inconsistent"
                 }
             }
@@ -120,6 +195,7 @@ start_server {} {
                 wait_for_condition 50 1000 {
                     [$R($j) get x] == $counter_value
                 } else {
+                    show_cluster_status
                     fail "Instance #$j x variable is inconsistent"
                 }
             }
@@ -134,26 +210,12 @@ start_server {} {
             [status $R(3) master_repl_offset] >= $masteroff &&
             [status $R(4) master_repl_offset] >= $masteroff
         } else {
-            puts "Master ID is $master_id"
-            for {set j 0} {$j < 5} {incr j} {
-                puts "$j: sync_full: [status $R($j) sync_full]"
-                puts "$j: id1      : [status $R($j) master_replid]:[status $R($j) master_repl_offset]"
-                puts "$j: id2      : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]"
-                puts "$j: backlog  : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]"
-                puts "---"
-            }
+            show_cluster_status
             fail "Replicas offsets didn't catch up with the master after too long time."
         }
 
         if {$debug_msg} {
-            puts "Master ID is $master_id"
-            for {set j 0} {$j < 5} {incr j} {
-                puts "$j: sync_full: [status $R($j) sync_full]"
-                puts "$j: id1      : [status $R($j) master_replid]:[status $R($j) master_repl_offset]"
-                puts "$j: id2      : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]"
-                puts "$j: backlog  : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]"
-                puts "---"
-            }
+            show_cluster_status
         }
 
         test "PSYNC2: total sum of full synchronizations is exactly 4" {
@@ -161,7 +223,10 @@ start_server {} {
             for {set j 0} {$j < 5} {incr j} {
                 incr sum [status $R($j) sync_full]
             }
-            assert {$sum == 4}
+            if {$sum != 4} {
+                show_cluster_status
+                assert {$sum == 4}
+            }
         }
 
         # In absence of pings, are the instances really able to have
@@ -174,14 +239,7 @@ start_server {} {
             [status $R($master_id) master_repl_offset] == [status $R(3) master_repl_offset] &&
             [status $R($master_id) master_repl_offset] == [status $R(4) master_repl_offset]
         } else {
-            puts "Master ID is $master_id"
-            for {set j 0} {$j < 5} {incr j} {
-                puts "$j: sync_full: [status $R($j) sync_full]"
-                puts "$j: id1      : [status $R($j) master_replid]:[status $R($j) master_repl_offset]"
-                puts "$j: id2      : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]"
-                puts "$j: backlog  : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]"
-                puts "---"
-            }
+            show_cluster_status
             fail "Replicas and master offsets were unable to match *exactly*."
         }
         $R($master_id) config set repl-ping-replica-period 10
@@ -210,6 +268,7 @@ start_server {} {
             [status $R([expr {($master_id+3)%5}]) master_link_status] == "up" &&
             [status $R([expr {($master_id+4)%5}]) master_link_status] == "up"
         } else {
+            show_cluster_status
             fail "Replica not reconnecting"
         }
     }
@@ -233,6 +292,7 @@ start_server {} {
             puts "prev sync_partial_ok: $sync_partial"
             puts "prev sync_partial_err: $sync_partial_err"
             puts [$R($master_id) info stats]
+            show_cluster_status
             fail "Replica didn't partial sync"
         }
         set new_sync_count [status $R($master_id) sync_full]
@@ -254,6 +314,7 @@ start_server {} {
         wait_for_condition 50 1000 {
             [$R($master_id) debug digest] == [$R($slave_id) debug digest]
         } else {
+            show_cluster_status
             fail "Replica not reconnecting"
         }
 
@@ -288,6 +349,7 @@ start_server {} {
         wait_for_condition 50 1000 {
             [status $R($master_id) connected_slaves] == 4
         } else {
+            show_cluster_status
             fail "Replica not reconnecting"
         }
         set new_sync_count [status $R($master_id) sync_full]
@@ -298,6 +360,7 @@ start_server {} {
         wait_for_condition 50 1000 {
             [$R($master_id) debug digest] == [$R($slave_id) debug digest]
         } else {
+            show_cluster_status
             fail "Debug digest mismatch between master and replica in post-restart handshake"
         }
     }

From e258a1c0877d825c1675251620f817f5333c1446 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 26 May 2020 11:00:48 +0300
Subject: [PATCH 0404/1098] tests: each test client work on a distinct port
 range

apparently when running tests in parallel (the default of --clients 16),
there's a chance for two tests to use the same port.
specifically, one test might shutdown a master and still have the
replica up, and then another test will re-use the port number of master
for another master, and then that replica will connect to the master of
the other test.

this can cause a master to count too many full syncs and fail a test if
we run the tests with --single integration/psync2 --loop --stop

see Probmem 2 in #7314
---
 tests/instances.tcl      |  3 ++-
 tests/support/server.tcl | 21 ++++++++++-----------
 tests/support/util.tcl   |  8 ++++----
 tests/test_helper.tcl    | 30 +++++++++++++++++++++---------
 tests/unit/other.tcl     |  4 ++--
 5 files changed, 39 insertions(+), 27 deletions(-)

diff --git a/tests/instances.tcl b/tests/instances.tcl
index 0a0cbab12..3a4fadca0 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -25,6 +25,7 @@ set ::sentinel_instances {}
 set ::redis_instances {}
 set ::sentinel_base_port 20000
 set ::redis_base_port 30000
+set ::redis_port_count 1024
 set ::pids {} ; # We kill everything at exit
 set ::dirs {} ; # We remove all the temp dirs at exit
 set ::run_matching {} ; # If non empty, only tests matching pattern are run.
@@ -57,7 +58,7 @@ proc exec_instance {type cfgfile} {
 # Spawn a redis or sentinel instance, depending on 'type'.
 proc spawn_instance {type base_port count {conf {}}} {
     for {set j 0} {$j < $count} {incr j} {
-        set port [find_available_port $base_port]
+        set port [find_available_port $base_port $::redis_port_count]
         incr base_port
         puts "Starting $type #$j at port $port"
 
diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index d086366dc..146ebc72c 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -214,14 +214,14 @@ proc start_server {options {code undefined}} {
     dict set config dir [tmpdir server]
 
     # start every server on a different port
-    set ::port [find_available_port [expr {$::port+1}]]
+    set port [find_available_port $::baseport $::portcount]
     if {$::tls} {
         dict set config "port" 0
-        dict set config "tls-port" $::port
+        dict set config "tls-port" $port
         dict set config "tls-cluster" "yes"
         dict set config "tls-replication" "yes"
     } else {
-        dict set config port $::port
+        dict set config port $port
     }
 
     set unixsocket [file normalize [format "%s/%s" [dict get $config "dir"] "socket"]]
@@ -243,10 +243,10 @@ proc start_server {options {code undefined}} {
     set server_started 0
     while {$server_started == 0} {
         if {$::verbose} {
-            puts -nonewline "=== ($tags) Starting server ${::host}:${::port} "
+            puts -nonewline "=== ($tags) Starting server ${::host}:${port} "
         }
 
-        send_data_packet $::test_server_fd "server-spawning" "port $::port"
+        send_data_packet $::test_server_fd "server-spawning" "port $port"
 
         if {$::valgrind} {
             set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &]
@@ -291,19 +291,19 @@ proc start_server {options {code undefined}} {
         # for availability. Other test clients may grab the port before we
         # are able to do it for example.
         if {$port_busy} {
-            puts "Port $::port was already busy, trying another port..."
-            set ::port [find_available_port [expr {$::port+1}]]
+            puts "Port $port was already busy, trying another port..."
+            set port [find_available_port $::baseport $::portcount]
             if {$::tls} {
-                dict set config "tls-port" $::port
+                dict set config "tls-port" $port
             } else {
-                dict set config port $::port
+                dict set config port $port
             }
             create_server_config_file $config_file $config
             continue; # Try again
         }
 
         if {$code ne "undefined"} {
-            set serverisup [server_is_up $::host $::port $retrynum]
+            set serverisup [server_is_up $::host $port $retrynum]
         } else {
             set serverisup 1
         }
@@ -324,7 +324,6 @@ proc start_server {options {code undefined}} {
     # setup properties to be able to initialize a client object
     set port_param [expr $::tls ? {"tls-port"} : {"port"}]
     set host $::host
-    set port $::port
     if {[dict exists $config bind]} { set host [dict get $config bind] }
     if {[dict exists $config $port_param]} { set port [dict get $config $port_param] }
 
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index 7ecf5b79c..a5ded67a3 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -344,8 +344,8 @@ proc roundFloat f {
     format "%.10g" $f
 }
 
-proc find_available_port start {
-    for {set j $start} {$j < $start+1024} {incr j} {
+proc find_available_port {start count} {
+    for {set j $start} {$j < $start+$count} {incr j} {
         if {[catch {set fd1 [socket 127.0.0.1 $j]}] &&
             [catch {set fd2 [socket 127.0.0.1 [expr $j+10000]]}]} {
             return $j
@@ -356,8 +356,8 @@ proc find_available_port start {
             }
         }
     }
-    if {$j == $start+1024} {
-        error "Can't find a non busy port in the $start-[expr {$start+1023}] range."
+    if {$j == $start+$count} {
+        error "Can't find a non busy port in the $start-[expr {$start+$count-1}] range."
     }
 }
 
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index de0a64728..7fc0e6f00 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -70,7 +70,9 @@ set ::all_tests {
 set ::next_test 0
 
 set ::host 127.0.0.1
-set ::port 21111
+set ::port 6379; # port for external server
+set ::baseport 21111; # initial port for spawned redis servers
+set ::portcount 8000; # we don't wanna use more than 10000 to avoid collision with cluster bus ports
 set ::traceleaks 0
 set ::valgrind 0
 set ::tls 0
@@ -228,26 +230,26 @@ proc test_server_main {} {
     set tclsh [info nameofexecutable]
     # Open a listening socket, trying different ports in order to find a
     # non busy one.
-    set port [find_available_port 11111]
+    set clientport [find_available_port 11111 32]
     if {!$::quiet} {
-        puts "Starting test server at port $port"
+        puts "Starting test server at port $clientport"
     }
-    socket -server accept_test_clients  -myaddr 127.0.0.1 $port
+    socket -server accept_test_clients  -myaddr 127.0.0.1 $clientport
 
     # Start the client instances
     set ::clients_pids {}
     if {$::external} {
         set p [exec $tclsh [info script] {*}$::argv \
-            --client $port --port $::port &]
+            --client $clientport &]
         lappend ::clients_pids $p
     } else {
-        set start_port [expr {$::port+100}]
+        set start_port $::baseport
+        set port_count [expr {$::portcount / $::numclients}]
         for {set j 0} {$j < $::numclients} {incr j} {
-            set start_port [find_available_port $start_port]
             set p [exec $tclsh [info script] {*}$::argv \
-                --client $port --port $start_port &]
+                --client $clientport --baseport $start_port --portcount $port_count &]
             lappend ::clients_pids $p
-            incr start_port 10
+            incr start_port $port_count
         }
     }
 
@@ -510,6 +512,10 @@ proc print_help_screen {} {
         "--loop             Execute the specified set of tests forever."
         "--wait-server      Wait after server is started (so that you can attach a debugger)."
         "--tls              Run tests in TLS mode."
+        "--host <addr>      Run tests against an external host."
+        "--port <port>      TCP port to use against external host."
+        "--baseport <port>  Initial port number for spawned redis servers."
+        "--portcount <num>  Port range for spawned redis servers."
         "--help             Print this help screen."
     } "\n"]
 }
@@ -560,6 +566,12 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
     } elseif {$opt eq {--port}} {
         set ::port $arg
         incr j
+    } elseif {$opt eq {--baseport}} {
+        set ::baseport $arg
+        incr j
+    } elseif {$opt eq {--portcount}} {
+        set ::portcount $arg
+        incr j
     } elseif {$opt eq {--accurate}} {
         set ::accurate 1
     } elseif {$opt eq {--force-failure}} {
diff --git a/tests/unit/other.tcl b/tests/unit/other.tcl
index 7720c055a..20a32e795 100644
--- a/tests/unit/other.tcl
+++ b/tests/unit/other.tcl
@@ -167,9 +167,9 @@ start_server {tags {"other"}} {
     tags {protocol} {
         test {PIPELINING stresser (also a regression for the old epoll bug)} {
             if {$::tls} {
-                set fd2 [::tls::socket $::host $::port]
+                set fd2 [::tls::socket [srv host] [srv port]]
             } else {
-                set fd2 [socket $::host $::port]
+                set fd2 [socket [srv host] [srv port]]
             }
             fconfigure $fd2 -encoding binary -translation binary
             puts -nonewline $fd2 "SELECT 9\r\n"

From b9abecfc4c97db01fa6f09180c68a92ea5974ac1 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 26 May 2020 13:50:59 +0300
Subject: [PATCH 0405/1098] avoid using sendfile if tls-replication is enabled

this obviously broke the tests, but went unnoticed so far since tls
wasn't often tested.
---
 src/replication.c | 65 ++++++++++++++++++++++++++---------------------
 1 file changed, 36 insertions(+), 29 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index e1c65d48b..e732788e7 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1032,39 +1032,46 @@ void sendBulkToSlave(connection *conn) {
         }
     }
 
-    /* If the preamble was already transferred, send the RDB bulk data. */
+    /* If the preamble was already transferred, send the RDB bulk data.
+     * try to use sendfile system call if supported, unless tls is enabled.
+     * fallback to normal read+write otherwise. */
+    nwritten = 0;
 #if HAVE_SENDFILE
-    if ((nwritten = redis_sendfile(conn->fd,slave->repldbfd,
-        slave->repldboff,PROTO_IOBUF_LEN)) == -1)
-    {
-        if (errno != EAGAIN) {
-            serverLog(LL_WARNING,"Sendfile error sending DB to replica: %s",
-                strerror(errno));
-            freeClient(slave);
+    if (!server.tls_replication) {
+        if ((nwritten = redis_sendfile(conn->fd,slave->repldbfd,
+            slave->repldboff,PROTO_IOBUF_LEN)) == -1)
+        {
+            if (errno != EAGAIN) {
+                serverLog(LL_WARNING,"Sendfile error sending DB to replica: %s",
+                    strerror(errno));
+                freeClient(slave);
+            }
+            return;
         }
-        return;
-    }
-#else
-    ssize_t buflen;
-    char buf[PROTO_IOBUF_LEN];
-
-    lseek(slave->repldbfd,slave->repldboff,SEEK_SET);
-    buflen = read(slave->repldbfd,buf,PROTO_IOBUF_LEN);
-    if (buflen <= 0) {
-        serverLog(LL_WARNING,"Read error sending DB to replica: %s",
-            (buflen == 0) ? "premature EOF" : strerror(errno));
-        freeClient(slave);
-        return;
-    }
-    if ((nwritten = connWrite(conn,buf,buflen)) == -1) {
-        if (connGetState(conn) != CONN_STATE_CONNECTED) {
-            serverLog(LL_WARNING,"Write error sending DB to replica: %s",
-                connGetLastError(conn));
-            freeClient(slave);
-        }
-        return;
     }
 #endif
+    if (!nwritten) {
+        ssize_t buflen;
+        char buf[PROTO_IOBUF_LEN];
+
+        lseek(slave->repldbfd,slave->repldboff,SEEK_SET);
+        buflen = read(slave->repldbfd,buf,PROTO_IOBUF_LEN);
+        if (buflen <= 0) {
+            serverLog(LL_WARNING,"Read error sending DB to replica: %s",
+                (buflen == 0) ? "premature EOF" : strerror(errno));
+            freeClient(slave);
+            return;
+        }
+        if ((nwritten = connWrite(conn,buf,buflen)) == -1) {
+            if (connGetState(conn) != CONN_STATE_CONNECTED) {
+                serverLog(LL_WARNING,"Write error sending DB to replica: %s",
+                    connGetLastError(conn));
+                freeClient(slave);
+            }
+            return;
+        }
+    }
+
     slave->repldboff += nwritten;
     server.stat_net_output_bytes += nwritten;
     if (slave->repldboff == slave->repldbsize) {

From 1965c0099e5f1c6d0714d326cea988dff8d481c6 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 25 May 2020 17:25:23 +0300
Subject: [PATCH 0406/1098] daily CI test with tls

---
 .github/workflows/daily.yml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 828268c70..c22d49895 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -49,6 +49,21 @@ jobs:
     - name: module api test
       run: ./runtest-moduleapi --verbose
 
+  test-tls:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: make
+      run: |
+        make BUILD_TLS=yes
+    - name: test
+      run: |
+        sudo apt-get install tcl8.5 tcl-tls
+        ./utils/gen-test-certs.sh
+        ./runtest --accurate --verbose --tls
+    - name: module api test
+      run: ./runtest-moduleapi --verbose --tls
+
   test-valgrind:
     runs-on: ubuntu-latest
     timeout-minutes: 14400

From 7d359392060d36dd05e2e9e73b65b0dbcbbb42ed Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 26 May 2020 23:55:18 +0200
Subject: [PATCH 0407/1098] Replication: log backlog creation event.

---
 src/replication.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/replication.c b/src/replication.c
index 3ebe451a2..07dc50edb 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -748,6 +748,9 @@ void syncCommand(client *c) {
         changeReplicationId();
         clearReplicationId2();
         createReplicationBacklog();
+        serverLog(LL_NOTICE,"Replication backlog created, my new "
+                            "replication IDs are '%s' and '%s'",
+                            server.replid, server.replid2);
     }
 
     /* CASE 1: BGSAVE is in progress, with disk target. */

From 3984dc6539e35cbd5fd2fdd4fda3e56f35244beb Mon Sep 17 00:00:00 2001
From: Liu Zhen <zhenliu215252@sohu-inc.com>
Date: Wed, 27 May 2020 11:55:23 +0800
Subject: [PATCH 0408/1098] fix clusters mixing accidentally by gossip

`clusterStartHandshake` will start hand handshake
and eventually send CLUSTER MEET message, which is strictly prohibited
in the REDIS CLUSTER SPEC.
Only system administrator can initiate CLUSTER MEET message.
Futher, according to the SPEC, rather than IP/PORT pairs, only nodeid
can be trusted.
---
 src/cluster.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index a2fab323a..24b14d1dc 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -1463,7 +1463,10 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) {
             }
         } else {
             /* If it's not in NOADDR state and we don't have it, we
-             * start a handshake process against this IP/PORT pairs.
+             * add it to our trusted dict with exact nodeid and flag.
+             * Note that we cannot simply start a handshake against
+             * this IP/PORT pairs, since IP/PORT can be reused already,
+             * otherwise we risk joining another cluster.
              *
              * Note that we require that the sender of this gossip message
              * is a well known node in our cluster, otherwise we risk
@@ -1472,7 +1475,12 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) {
                 !(flags & CLUSTER_NODE_NOADDR) &&
                 !clusterBlacklistExists(g->nodename))
             {
-                clusterStartHandshake(g->ip,ntohs(g->port),ntohs(g->cport));
+                clusterNode *node;
+                node = createClusterNode(g->nodename, flags);
+                memcpy(node->ip,g->ip,NET_IP_STR_LEN);
+                node->port = ntohs(g->port);
+                node->cport = ntohs(g->cport);
+                clusterAddNode(node);
             }
         }
 

From 325409a011d0155513a71b92dc09cac06b90b261 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 27 May 2020 11:45:49 +0200
Subject: [PATCH 0409/1098] Set a protocol error if master use the inline
 protocol.

We want to react a bit more aggressively if we sense that the master is
sending us some corrupted stream. By setting the protocol error we both
ensure that the replica will disconnect, and avoid caching the master so
that a full SYNC will be required. This is protective against
replication bugs.
---
 src/networking.c | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 364654642..bb682db4c 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1553,6 +1553,19 @@ int processInlineBuffer(client *c) {
     if (querylen == 0 && getClientType(c) == CLIENT_TYPE_SLAVE)
         c->repl_ack_time = server.unixtime;
 
+    /* Masters should never send us inline protocol to run actual
+     * commands. If this happens, it is likely due to a bug in Redis where
+     * we got some desynchronization in the protocol, for example
+     * beause of a PSYNC gone bad.
+     *
+     * However the is an exception: masters may send us just a newline
+     * to keep the connection active. */
+    if (querylen != 0 && c->flags & CLIENT_MASTER) {
+        serverLog(LL_WARNING,"WARNING: Receiving inline protocol from master, master stream corruption? Closing the master connection and discarding the cached master.");
+        setProtocolError("Master using the inline protocol. Desync?",c);
+        return C_ERR;
+    }
+
     /* Move querybuffer position to the next query in the buffer. */
     c->qb_pos += querylen+linefeed_chars;
 
@@ -1576,7 +1589,7 @@ int processInlineBuffer(client *c) {
  * CLIENT_PROTOCOL_ERROR. */
 #define PROTO_DUMP_LEN 128
 static void setProtocolError(const char *errstr, client *c) {
-    if (server.verbosity <= LL_VERBOSE) {
+    if (server.verbosity <= LL_VERBOSE || c->flags & CLIENT_MASTER) {
         sds client = catClientInfoString(sdsempty(),c);
 
         /* Sample some protocol to given an idea about what was inside. */
@@ -1595,7 +1608,9 @@ static void setProtocolError(const char *errstr, client *c) {
         }
 
         /* Log all the client and protocol info. */
-        serverLog(LL_VERBOSE,
+        int loglevel = (c->flags & CLIENT_MASTER) ? LL_WARNING :
+                                                    LL_VERBOSE;
+        serverLog(loglevel,
             "Protocol error (%s) from client: %s. %s", errstr, client, buf);
         sdsfree(client);
     }

From 22472fe5a1e59e4c3f589a9ea38d17a2e1f04a96 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 27 May 2020 12:06:33 +0200
Subject: [PATCH 0410/1098] Remove the meaningful offset feature.

After a closer look, the Redis core devleopers all believe that this was
too fragile, caused many bugs that we didn't expect and that were very
hard to track. Better to find an alternative solution that is simpler.
---
 src/networking.c  | 23 ++-------------
 src/replication.c | 73 ++++-------------------------------------------
 src/server.c      |  4 ---
 src/server.h      |  3 +-
 4 files changed, 10 insertions(+), 93 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index bb682db4c..7a5f1d7b9 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1037,25 +1037,14 @@ static void freeClientArgv(client *c) {
 
 /* Close all the slaves connections. This is useful in chained replication
  * when we resync with our own master and want to force all our slaves to
- * resync with us as well.
- *
- * If 'async' is non-zero we free the clients asynchronously. This is needed
- * when we call this function from a context where in the chain of the
- * callers somebody is iterating the list of clients. For instance when
- * CLIENT KILL TYPE master is called, caching the master client may
- * adjust the meaningful offset of replication, and in turn call
- * discionectSlaves(). Since CLIENT KILL iterates the clients this is
- * not safe. */
-void disconnectSlaves(int async) {
+ * resync with us as well. */
+void disconnectSlaves(void) {
     listIter li;
     listNode *ln;
     listRewind(server.slaves,&li);
     while((ln = listNext(&li))) {
         listNode *ln = listFirst(server.slaves);
-        if (async)
-            freeClientAsync((client*)ln->value);
-        else
-            freeClient((client*)ln->value);
+        freeClient((client*)ln->value);
     }
 }
 
@@ -1769,7 +1758,6 @@ int processMultibulkBuffer(client *c) {
  * 2. In the case of master clients, the replication offset is updated.
  * 3. Propagate commands we got from our master to replicas down the line. */
 void commandProcessed(client *c) {
-    int cmd_is_ping = c->cmd && c->cmd->proc == pingCommand;
     long long prev_offset = c->reploff;
     if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) {
         /* Update the applied replication offset of our master. */
@@ -1794,16 +1782,11 @@ void commandProcessed(client *c) {
      * sub-replicas and to the replication backlog. */
     if (c->flags & CLIENT_MASTER) {
         long long applied = c->reploff - prev_offset;
-        long long prev_master_repl_meaningful_offset = server.master_repl_meaningful_offset;
         if (applied) {
             replicationFeedSlavesFromMasterStream(server.slaves,
                     c->pending_querybuf, applied);
             sdsrange(c->pending_querybuf,applied,-1);
         }
-        /* The server.master_repl_meaningful_offset variable represents
-         * the offset of the replication stream without the pending PINGs. */
-        if (cmd_is_ping)
-            server.master_repl_meaningful_offset = prev_master_repl_meaningful_offset;
     }
 }
 
diff --git a/src/replication.c b/src/replication.c
index 2b21b02d8..0484ec8a5 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -39,7 +39,6 @@
 #include <sys/socket.h>
 #include <sys/stat.h>
 
-long long adjustMeaningfulReplOffset(int *adjusted);
 void replicationDiscardCachedMaster(void);
 void replicationResurrectCachedMaster(connection *conn);
 void replicationSendAck(void);
@@ -163,7 +162,6 @@ void feedReplicationBacklog(void *ptr, size_t len) {
     unsigned char *p = ptr;
 
     server.master_repl_offset += len;
-    server.master_repl_meaningful_offset = server.master_repl_offset;
 
     /* This is a circular buffer, so write as much data we can at every
      * iteration and rewind the "idx" index if we reach the limit. */
@@ -1831,7 +1829,6 @@ void readSyncBulkPayload(connection *conn) {
      * we are starting a new history. */
     memcpy(server.replid,server.master->replid,sizeof(server.replid));
     server.master_repl_offset = server.master->reploff;
-    server.master_repl_meaningful_offset = server.master->reploff;
     clearReplicationId2();
 
     /* Let's create the replication backlog if needed. Slaves need to
@@ -2086,7 +2083,7 @@ int slaveTryPartialResynchronization(connection *conn, int read_reply) {
                 memcpy(server.cached_master->replid,new,sizeof(server.replid));
 
                 /* Disconnect all the sub-slaves: they need to be notified. */
-                disconnectSlaves(0);
+                disconnectSlaves();
             }
         }
 
@@ -2359,7 +2356,7 @@ void syncWithMaster(connection *conn) {
      * as well, if we have any sub-slaves. The master may transfer us an
      * entirely different data set and we have no way to incrementally feed
      * our slaves after that. */
-    disconnectSlaves(0); /* Force our slaves to resync with us as well. */
+    disconnectSlaves(); /* Force our slaves to resync with us as well. */
     freeReplicationBacklog(); /* Don't allow our chained slaves to PSYNC. */
 
     /* Fall back to SYNC if needed. Otherwise psync_result == PSYNC_FULLRESYNC
@@ -2506,7 +2503,7 @@ void replicationSetMaster(char *ip, int port) {
 
     /* Force our slaves to resync with us as well. They may hopefully be able
      * to partially resync with us, but we can notify the replid change. */
-    disconnectSlaves(0);
+    disconnectSlaves();
     cancelReplicationHandshake();
     /* Before destroying our master state, create a cached master using
      * our own parameters, to later PSYNC with the new master. */
@@ -2557,7 +2554,7 @@ void replicationUnsetMaster(void) {
      * of the replication ID change (see shiftReplicationId() call). However
      * the slaves will be able to partially resync with us, so it will be
      * a very fast reconnection. */
-    disconnectSlaves(0);
+    disconnectSlaves();
     server.repl_state = REPL_STATE_NONE;
 
     /* We need to make sure the new master will start the replication stream
@@ -2760,10 +2757,7 @@ void replicationCacheMaster(client *c) {
     sdsclear(server.master->querybuf);
     sdsclear(server.master->pending_querybuf);
 
-    /* Adjust reploff and read_reploff to the last meaningful offset we
-     * executed. This is the offset the replica will use for future PSYNC. */
-    int offset_adjusted;
-    server.master->reploff = adjustMeaningfulReplOffset(&offset_adjusted);
+    server.master->reploff = server.master_repl_offset;
     server.master->read_reploff = server.master->reploff;
     if (c->flags & CLIENT_MULTI) discardTransaction(c);
     listEmpty(c->reply);
@@ -2786,53 +2780,6 @@ void replicationCacheMaster(client *c) {
      * so make sure to adjust the replication state. This function will
      * also set server.master to NULL. */
     replicationHandleMasterDisconnection();
-
-    /* If we trimmed this replica backlog, we need to disconnect our chained
-     * replicas (if any), otherwise they may have the PINGs we removed
-     * from the stream and their offset would no longer match: upon
-     * disconnection they will also trim the final PINGs and will be able
-     * to incrementally sync without issues. */
-    if (offset_adjusted) disconnectSlaves(1);
-}
-
-/* If the "meaningful" offset, that is the offset without the final PINGs
- * in the stream, is different than the last offset, use it instead:
- * often when the master is no longer reachable, replicas will never
- * receive the PINGs, however the master will end with an incremented
- * offset because of the PINGs and will not be able to incrementally
- * PSYNC with the new master.
- * This function trims the replication backlog when needed, and returns
- * the offset to be used for future partial sync.
- *
- * If the integer 'adjusted' was passed by reference, it is set to 1
- * if the function call actually modified the offset and the replication
- * backlog, otherwise it is set to 0. It can be NULL if the caller is
- * not interested in getting this info. */
-long long adjustMeaningfulReplOffset(int *adjusted) {
-    if (server.master_repl_offset > server.master_repl_meaningful_offset) {
-        long long delta = server.master_repl_offset -
-                          server.master_repl_meaningful_offset;
-        serverLog(LL_NOTICE,
-            "Using the meaningful offset %lld instead of %lld to exclude "
-            "the final PINGs (%lld bytes difference)",
-                server.master_repl_meaningful_offset,
-                server.master_repl_offset,
-                delta);
-        server.master_repl_offset = server.master_repl_meaningful_offset;
-        if (server.repl_backlog_histlen <= delta) {
-            server.repl_backlog_histlen = 0;
-            server.repl_backlog_idx = 0;
-        } else {
-            server.repl_backlog_histlen -= delta;
-            server.repl_backlog_idx =
-                (server.repl_backlog_idx + (server.repl_backlog_size - delta)) %
-                server.repl_backlog_size;
-        }
-        if (adjusted) *adjusted = 1;
-    } else {
-        if (adjusted) *adjusted = 0;
-    }
-    return server.master_repl_offset;
 }
 
 /* This function is called when a master is turend into a slave, in order to
@@ -2854,7 +2801,7 @@ void replicationCacheMasterUsingMyself(void) {
      * by replicationCreateMasterClient(). We'll later set the created
      * master as server.cached_master, so the replica will use such
      * offset for PSYNC. */
-    server.master_initial_offset = adjustMeaningfulReplOffset(NULL);
+    server.master_initial_offset = server.master_repl_offset;
 
     /* The master client we create can be set to any DBID, because
      * the new master will start its replication stream with SELECT. */
@@ -3246,18 +3193,10 @@ void replicationCron(void) {
             clientsArePaused();
 
         if (!manual_failover_in_progress) {
-            long long before_ping = server.master_repl_meaningful_offset;
             ping_argv[0] = createStringObject("PING",4);
             replicationFeedSlaves(server.slaves, server.slaveseldb,
                 ping_argv, 1);
             decrRefCount(ping_argv[0]);
-            /* The server.master_repl_meaningful_offset variable represents
-             * the offset of the replication stream without the pending PINGs.
-             * This is useful to set the right replication offset for PSYNC
-             * when the master is turned into a replica. Otherwise pending
-             * PINGs may not allow it to perform an incremental sync with the
-             * new master. */
-            server.master_repl_meaningful_offset = before_ping;
         }
     }
 
diff --git a/src/server.c b/src/server.c
index 5bc4666ee..b7a6a928f 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2394,7 +2394,6 @@ void initServerConfig(void) {
     server.repl_syncio_timeout = CONFIG_REPL_SYNCIO_TIMEOUT;
     server.repl_down_since = 0; /* Never connected, repl is down since EVER. */
     server.master_repl_offset = 0;
-    server.master_repl_meaningful_offset = 0;
 
     /* Replication partial resync backlog */
     server.repl_backlog = NULL;
@@ -4471,7 +4470,6 @@ sds genRedisInfoString(const char *section) {
             "master_replid:%s\r\n"
             "master_replid2:%s\r\n"
             "master_repl_offset:%lld\r\n"
-            "master_repl_meaningful_offset:%lld\r\n"
             "second_repl_offset:%lld\r\n"
             "repl_backlog_active:%d\r\n"
             "repl_backlog_size:%lld\r\n"
@@ -4480,7 +4478,6 @@ sds genRedisInfoString(const char *section) {
             server.replid,
             server.replid2,
             server.master_repl_offset,
-            server.master_repl_meaningful_offset,
             server.second_replid_offset,
             server.repl_backlog != NULL,
             server.repl_backlog_size,
@@ -4858,7 +4855,6 @@ void loadDataFromDisk(void) {
             {
                 memcpy(server.replid,rsi.repl_id,sizeof(server.replid));
                 server.master_repl_offset = rsi.repl_offset;
-                server.master_repl_meaningful_offset = rsi.repl_offset;
                 /* If we are a slave, create a cached master from this
                  * information, in order to allow partial resynchronizations
                  * with masters. */
diff --git a/src/server.h b/src/server.h
index 2d17d69c8..0c0b4d052 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1261,7 +1261,6 @@ struct redisServer {
     char replid[CONFIG_RUN_ID_SIZE+1];  /* My current replication ID. */
     char replid2[CONFIG_RUN_ID_SIZE+1]; /* replid inherited from master*/
     long long master_repl_offset;   /* My current replication offset */
-    long long master_repl_meaningful_offset; /* Offset minus latest PINGs. */
     long long second_replid_offset; /* Accept offsets up to this for replid2. */
     int slaveseldb;                 /* Last SELECTed DB in replication output */
     int repl_ping_slave_period;     /* Master pings the slave every N seconds */
@@ -1660,7 +1659,7 @@ int getClientType(client *c);
 int getClientTypeByName(char *name);
 char *getClientTypeName(int class);
 void flushSlavesOutputBuffers(void);
-void disconnectSlaves(int async);
+void disconnectSlaves(void);
 int listenToPort(int port, int *fds, int *count);
 void pauseClients(mstime_t duration);
 int clientsArePaused(void);

From 32d0df0c1fb5fdd39defc6d85887cca73d062f47 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 27 May 2020 12:47:34 +0200
Subject: [PATCH 0411/1098] Remove the PSYNC2 meaningful offset test.

---
 tests/integration/psync2-pingoff.tcl | 112 ---------------------------
 tests/test_helper.tcl                |   1 -
 2 files changed, 113 deletions(-)
 delete mode 100644 tests/integration/psync2-pingoff.tcl

diff --git a/tests/integration/psync2-pingoff.tcl b/tests/integration/psync2-pingoff.tcl
deleted file mode 100644
index 2c2303141..000000000
--- a/tests/integration/psync2-pingoff.tcl
+++ /dev/null
@@ -1,112 +0,0 @@
-# Test the meaningful offset implementation to make sure masters
-# are able to PSYNC with replicas even if the replication stream
-# has pending PINGs at the end.
-
-start_server {tags {"psync2"}} {
-start_server {} {
-    # Config
-    set debug_msg 0                 ; # Enable additional debug messages
-
-    for {set j 0} {$j < 2} {incr j} {
-        set R($j) [srv [expr 0-$j] client]
-        set R_host($j) [srv [expr 0-$j] host]
-        set R_port($j) [srv [expr 0-$j] port]
-        $R($j) CONFIG SET repl-ping-replica-period 1
-        if {$debug_msg} {puts "Log file: [srv [expr 0-$j] stdout]"}
-    }
-
-    # Setup replication
-    test "PSYNC2 meaningful offset: setup" {
-        $R(1) replicaof $R_host(0) $R_port(0)
-        $R(0) set foo bar
-        wait_for_condition 50 1000 {
-            [status $R(1) master_link_status] == "up" &&
-            [$R(0) dbsize] == 1 && [$R(1) dbsize] == 1
-        } else {
-            fail "Replicas not replicating from master"
-        }
-    }
-
-    test "PSYNC2 meaningful offset: write and wait replication" {
-        $R(0) INCR counter
-        $R(0) INCR counter
-        $R(0) INCR counter
-        wait_for_condition 50 1000 {
-            [$R(0) GET counter] eq [$R(1) GET counter]
-        } else {
-            fail "Master and replica don't agree about counter"
-        }
-    }
-
-    # In this test we'll make sure the replica will get stuck, but with
-    # an active connection: this way the master will continue to send PINGs
-    # every second (we modified the PING period earlier)
-    test "PSYNC2 meaningful offset: pause replica and promote it" {
-        $R(1) MULTI
-        $R(1) DEBUG SLEEP 5
-        $R(1) SLAVEOF NO ONE
-        $R(1) EXEC
-        $R(1) ping ; # Wait for it to return back available
-    }
-
-    test "Make the old master a replica of the new one and check conditions" {
-        set sync_partial [status $R(1) sync_partial_ok]
-        assert {$sync_partial == 0}
-        $R(0) REPLICAOF $R_host(1) $R_port(1)
-        wait_for_condition 50 1000 {
-            [status $R(1) sync_partial_ok] == 1
-        } else {
-            fail "The new master was not able to partial sync"
-        }
-    }
-}}
-
-
-start_server {tags {"psync2"}} {
-start_server {} {
-start_server {} {
-
-    for {set j 0} {$j < 3} {incr j} {
-        set R($j) [srv [expr 0-$j] client]
-        set R_host($j) [srv [expr 0-$j] host]
-        set R_port($j) [srv [expr 0-$j] port]
-        $R($j) CONFIG SET repl-ping-replica-period 1
-    }
-
-    test "Chained replicas disconnect when replica re-connect with the same master" {
-        # Add a second replica as a chained replica of the current replica
-        $R(1) replicaof $R_host(0) $R_port(0)
-        $R(2) replicaof $R_host(1) $R_port(1)
-        wait_for_condition 50 1000 {
-            [status $R(2) master_link_status] == "up"
-        } else {
-            fail "Chained replica not replicating from its master"
-        }
-
-        # Do a write on the master, and wait for 3 seconds for the master to
-        # send some PINGs to its replica
-        $R(0) INCR counter2
-        after 2000
-        set sync_partial_master [status $R(0) sync_partial_ok]
-        set sync_partial_replica [status $R(1) sync_partial_ok]
-        $R(0) CONFIG SET repl-ping-replica-period 100
-
-        # Disconnect the master's direct replica
-        $R(0) client kill type replica
-        wait_for_condition 50 1000 {
-            [status $R(1) master_link_status] == "up" && 
-            [status $R(2) master_link_status] == "up" &&
-            [status $R(0) sync_partial_ok] == $sync_partial_master + 1 &&
-            [status $R(1) sync_partial_ok] == $sync_partial_replica + 1
-        } else {
-            fail "Disconnected replica failed to PSYNC with master"
-        }
-
-        # Verify that the replica and its replica's meaningful and real
-        # offsets match with the master
-        assert_equal [status $R(0) master_repl_offset] [status $R(1) master_repl_offset]
-        assert_equal [status $R(0) master_repl_offset] [status $R(2) master_repl_offset]
-        assert_equal [status $R(0) master_repl_meaningful_offset] [status $R(1) master_repl_meaningful_offset]
-        assert_equal [status $R(0) master_repl_meaningful_offset] [status $R(2) master_repl_meaningful_offset]
-    }
-}}}
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index de0a64728..05e45b999 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -47,7 +47,6 @@ set ::all_tests {
     integration/logging
     integration/psync2
     integration/psync2-reg
-    integration/psync2-pingoff
     unit/pubsub
     unit/slowlog
     unit/scripting

From 484cfc3d7603277e8c5761dd571962dd0201d873 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 27 May 2020 12:50:02 +0200
Subject: [PATCH 0412/1098] Another meaningful offset test removed.

---
 tests/integration/psync2.tcl | 100 -----------------------------------
 1 file changed, 100 deletions(-)

diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl
index 29a880f99..7c50f620e 100644
--- a/tests/integration/psync2.tcl
+++ b/tests/integration/psync2.tcl
@@ -370,103 +370,3 @@ start_server {} {
     }
 
 }}}}}
-
-start_server {tags {"psync2"}} {
-start_server {} {
-start_server {} {
-start_server {} {
-start_server {} {
-    test {pings at the end of replication stream are ignored for psync} {
-        set master [srv -4 client]
-        set master_host [srv -4 host]
-        set master_port [srv -4 port]
-        set replica1 [srv -3 client]
-        set replica2 [srv -2 client]
-        set replica3 [srv -1 client]
-        set replica4 [srv -0 client]
-
-        $replica1 replicaof $master_host $master_port
-        $replica2 replicaof $master_host $master_port
-        $replica3 replicaof $master_host $master_port
-        $replica4 replicaof $master_host $master_port
-        wait_for_condition 50 1000 {
-            [status $master connected_slaves] == 4
-        } else {
-            fail "replicas didn't connect"
-        }
-
-        $master incr x
-        wait_for_condition 50 1000 {
-            [$replica1 get x] == 1 && [$replica2 get x] == 1 &&
-            [$replica3 get x] == 1 && [$replica4 get x] == 1
-        } else {
-            fail "replicas didn't get incr"
-        }
-
-        # disconnect replica1 and replica2
-        # and wait for the master to send a ping to replica3 and replica4
-        $replica1 replicaof no one
-        $replica2 replicaof 127.0.0.1 1 ;# we can't promote it to master since that will cycle the replication id
-        $master config set repl-ping-replica-period 1
-        after 1500
-
-        # make everyone sync from the replica1 that didn't get the last ping from the old master
-        # replica4 will keep syncing from the old master which now syncs from replica1
-        # and replica2 will re-connect to the old master (which went back in time)
-        set new_master_host [srv -3 host]
-        set new_master_port [srv -3 port]
-        $replica3 replicaof $new_master_host $new_master_port
-        $master replicaof $new_master_host $new_master_port
-        $replica2 replicaof $master_host $master_port
-        wait_for_condition 50 1000 {
-            [status $replica2 master_link_status] == "up" &&
-            [status $replica3 master_link_status] == "up" &&
-            [status $replica4 master_link_status] == "up" &&
-            [status $master master_link_status] == "up"
-        } else {
-            fail "replicas didn't connect"
-        }
-
-        # make sure replication is still alive and kicking
-        $replica1 incr x
-        wait_for_condition 50 1000 {
-            [$replica2 get x] == 2 &&
-            [$replica3 get x] == 2 &&
-            [$replica4 get x] == 2 &&
-            [$master get x] == 2
-        } else {
-            fail "replicas didn't get incr"
-        }
-
-        # make sure there are full syncs other than the initial ones
-        assert_equal [status $master sync_full] 4
-        assert_equal [status $replica1 sync_full] 0
-        assert_equal [status $replica2 sync_full] 0
-        assert_equal [status $replica3 sync_full] 0
-        assert_equal [status $replica4 sync_full] 0
-
-        # force psync
-        $master client kill type master
-        $replica2 client kill type master
-        $replica3 client kill type master
-        $replica4 client kill type master
-
-        # make sure replication is still alive and kicking
-        $replica1 incr x
-        wait_for_condition 50 1000 {
-            [$replica2 get x] == 3 &&
-            [$replica3 get x] == 3 &&
-            [$replica4 get x] == 3 &&
-            [$master get x] == 3
-        } else {
-            fail "replicas didn't get incr"
-        }
-
-        # make sure there are full syncs other than the initial ones
-        assert_equal [status $master sync_full] 4
-        assert_equal [status $replica1 sync_full] 0
-        assert_equal [status $replica2 sync_full] 0
-        assert_equal [status $replica3 sync_full] 0
-        assert_equal [status $replica4 sync_full] 0
-}
-}}}}}

From 151b12a80fe60e671604433876af4fd2a1f7084d Mon Sep 17 00:00:00 2001
From: Kevin Fwu <kfwu@litespeedtech.com>
Date: Wed, 27 May 2020 08:53:29 -0400
Subject: [PATCH 0413/1098] Fix TLS certificate loading for chained
 certificates.

This impacts client verification for chained certificates (such as Lets
Encrypt certificates). Client Verify requires the full chain in order to
properly verify the certificate.
---
 src/tls.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tls.c b/src/tls.c
index 28a74df9a..a62f2284e 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -217,7 +217,7 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
     SSL_CTX_set_ecdh_auto(ctx, 1);
 #endif
 
-    if (SSL_CTX_use_certificate_file(ctx, ctx_config->cert_file, SSL_FILETYPE_PEM) <= 0) {
+    if (SSL_CTX_use_certificate_chain_file(ctx, ctx_config->cert_file) <= 0) {
         ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf));
         serverLog(LL_WARNING, "Failed to load certificate: %s: %s", ctx_config->cert_file, errbuf);
         goto error;

From 1cf33a46d588a8e2591665cfcf1c121cbe0f18d5 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 27 May 2020 16:12:30 +0300
Subject: [PATCH 0414/1098] tests: find_available_port start search from next
 port

i.e. don't start the search from scratch hitting the used ones again.
this will also reduce the likelihood of collisions (if there are any
left) by increasing the time until we re-use a port we did use in the
past.
---
 tests/support/util.tcl | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index a5ded67a3..8bec95374 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -344,21 +344,26 @@ proc roundFloat f {
     format "%.10g" $f
 }
 
+set ::last_port_attempted 0
 proc find_available_port {start count} {
-    for {set j $start} {$j < $start+$count} {incr j} {
-        if {[catch {set fd1 [socket 127.0.0.1 $j]}] &&
-            [catch {set fd2 [socket 127.0.0.1 [expr $j+10000]]}]} {
-            return $j
+    set port [expr $::last_port_attempted + 1]
+    for {set attempts 0} {$attempts < $count} {incr attempts} {
+        if {$port < $start || $port >= $start+$count} {
+            set port $start
+        }
+        if {[catch {set fd1 [socket 127.0.0.1 $port]}] &&
+            [catch {set fd2 [socket 127.0.0.1 [expr $port+10000]]}]} {
+            set ::last_port_attempted $port
+            return $port
         } else {
             catch {
                 close $fd1
                 close $fd2
             }
         }
+        incr port
     }
-    if {$j == $start+$count} {
-        error "Can't find a non busy port in the $start-[expr {$start+$count-1}] range."
-    }
+    error "Can't find a non busy port in the $start-[expr {$start+$count-1}] range."
 }
 
 # Test if TERM looks like to support colors

From 484af8ed537c3ec33b4e1f10d851c2dee0ae8686 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 27 May 2020 17:08:51 +0200
Subject: [PATCH 0415/1098] Drop useless line from replicationCacheMaster().

---
 src/replication.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index 0484ec8a5..110f0e82d 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2756,8 +2756,6 @@ void replicationCacheMaster(client *c) {
      * pending outputs to the master. */
     sdsclear(server.master->querybuf);
     sdsclear(server.master->pending_querybuf);
-
-    server.master->reploff = server.master_repl_offset;
     server.master->read_reploff = server.master->reploff;
     if (c->flags & CLIENT_MULTI) discardTransaction(c);
     listEmpty(c->reply);

From 15bcb813d47dbfec343c7ed9ad328319c0bac989 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 27 May 2020 18:09:09 +0300
Subject: [PATCH 0416/1098] 32bit CI needs to build modules correctly

---
 .github/workflows/daily.yml | 4 +++-
 tests/modules/Makefile      | 5 ++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index c22d49895..acc4dd33a 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -47,7 +47,9 @@ jobs:
         sudo apt-get install tcl8.5
         ./runtest --accurate --verbose
     - name: module api test
-      run: ./runtest-moduleapi --verbose
+      run: |
+        make -C tests/modules 32bit # the script below doesn't have an argument, we must build manually ahead of time
+        ./runtest-moduleapi --verbose
 
   test-tls:
     runs-on: ubuntu-latest
diff --git a/tests/modules/Makefile b/tests/modules/Makefile
index 363231a87..39b8e6efa 100644
--- a/tests/modules/Makefile
+++ b/tests/modules/Makefile
@@ -28,11 +28,14 @@ TEST_MODULES = \
 
 all: $(TEST_MODULES)
 
+32bit:
+	$(MAKE) CFLAGS="-m32" LDFLAGS="-melf_i386"
+
 %.xo: %.c ../../src/redismodule.h
 	$(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@
 
 %.so: %.xo
-	$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc
+	$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LDFLAGS) $(LIBS) -lc
 
 .PHONY: clean
 

From 90f3856fd5a1c7d30bf4450772923bf4a16993b6 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 28 May 2020 08:21:24 +0300
Subject: [PATCH 0417/1098] revive meaningful offset tests

---
 tests/integration/psync2-pingoff.tcl | 212 +++++++++++++++++++++++++++
 tests/test_helper.tcl                |   1 +
 2 files changed, 213 insertions(+)
 create mode 100644 tests/integration/psync2-pingoff.tcl

diff --git a/tests/integration/psync2-pingoff.tcl b/tests/integration/psync2-pingoff.tcl
new file mode 100644
index 000000000..5d8459a92
--- /dev/null
+++ b/tests/integration/psync2-pingoff.tcl
@@ -0,0 +1,212 @@
+# Test the meaningful offset implementation to make sure masters
+# are able to PSYNC with replicas even if the replication stream
+# has pending PINGs at the end.
+
+start_server {tags {"psync2"}} {
+start_server {} {
+    # Config
+    set debug_msg 0                 ; # Enable additional debug messages
+
+    for {set j 0} {$j < 2} {incr j} {
+        set R($j) [srv [expr 0-$j] client]
+        set R_host($j) [srv [expr 0-$j] host]
+        set R_port($j) [srv [expr 0-$j] port]
+        $R($j) CONFIG SET repl-ping-replica-period 1
+        if {$debug_msg} {puts "Log file: [srv [expr 0-$j] stdout]"}
+    }
+
+    # Setup replication
+    test "PSYNC2 meaningful offset: setup" {
+        $R(1) replicaof $R_host(0) $R_port(0)
+        $R(0) set foo bar
+        wait_for_condition 50 1000 {
+            [status $R(1) master_link_status] == "up" &&
+            [$R(0) dbsize] == 1 && [$R(1) dbsize] == 1
+        } else {
+            fail "Replicas not replicating from master"
+        }
+    }
+
+    test "PSYNC2 meaningful offset: write and wait replication" {
+        $R(0) INCR counter
+        $R(0) INCR counter
+        $R(0) INCR counter
+        wait_for_condition 50 1000 {
+            [$R(0) GET counter] eq [$R(1) GET counter]
+        } else {
+            fail "Master and replica don't agree about counter"
+        }
+    }
+
+    # In this test we'll make sure the replica will get stuck, but with
+    # an active connection: this way the master will continue to send PINGs
+    # every second (we modified the PING period earlier)
+    test "PSYNC2 meaningful offset: pause replica and promote it" {
+        $R(1) MULTI
+        $R(1) DEBUG SLEEP 5
+        $R(1) SLAVEOF NO ONE
+        $R(1) EXEC
+        $R(1) ping ; # Wait for it to return back available
+    }
+
+    test "Make the old master a replica of the new one and check conditions" {
+        set sync_partial [status $R(1) sync_partial_ok]
+        assert {$sync_partial == 0}
+        $R(0) REPLICAOF $R_host(1) $R_port(1)
+        wait_for_condition 50 1000 {
+            [status $R(1) sync_partial_ok] == 1
+        } else {
+            fail "The new master was not able to partial sync"
+        }
+    }
+}}
+
+
+start_server {tags {"psync2"}} {
+start_server {} {
+start_server {} {
+start_server {} {
+start_server {} {
+    test {pings at the end of replication stream are ignored for psync} {
+        set master [srv -4 client]
+        set master_host [srv -4 host]
+        set master_port [srv -4 port]
+        set replica1 [srv -3 client]
+        set replica2 [srv -2 client]
+        set replica3 [srv -1 client]
+        set replica4 [srv -0 client]
+
+        $replica1 replicaof $master_host $master_port
+        $replica2 replicaof $master_host $master_port
+        $replica3 replicaof $master_host $master_port
+        $replica4 replicaof $master_host $master_port
+        wait_for_condition 50 1000 {
+            [status $master connected_slaves] == 4
+        } else {
+            fail "replicas didn't connect"
+        }
+
+        $master incr x
+        wait_for_condition 50 1000 {
+            [$replica1 get x] == 1 && [$replica2 get x] == 1 &&
+            [$replica3 get x] == 1 && [$replica4 get x] == 1
+        } else {
+            fail "replicas didn't get incr"
+        }
+
+        # disconnect replica1 and replica2
+        # and wait for the master to send a ping to replica3 and replica4
+        $replica1 replicaof no one
+        $replica2 replicaof 127.0.0.1 1 ;# we can't promote it to master since that will cycle the replication id
+        $master config set repl-ping-replica-period 1
+        after 1500
+
+        # make everyone sync from the replica1 that didn't get the last ping from the old master
+        # replica4 will keep syncing from the old master which now syncs from replica1
+        # and replica2 will re-connect to the old master (which went back in time)
+        set new_master_host [srv -3 host]
+        set new_master_port [srv -3 port]
+        $replica3 replicaof $new_master_host $new_master_port
+        $master replicaof $new_master_host $new_master_port
+        $replica2 replicaof $master_host $master_port
+        wait_for_condition 50 1000 {
+            [status $replica2 master_link_status] == "up" &&
+            [status $replica3 master_link_status] == "up" &&
+            [status $replica4 master_link_status] == "up" &&
+            [status $master master_link_status] == "up"
+        } else {
+            fail "replicas didn't connect"
+        }
+
+        # make sure replication is still alive and kicking
+        $replica1 incr x
+        wait_for_condition 50 1000 {
+            [$replica2 get x] == 2 &&
+            [$replica3 get x] == 2 &&
+            [$replica4 get x] == 2 &&
+            [$master get x] == 2
+        } else {
+            fail "replicas didn't get incr"
+        }
+
+        # make sure there are full syncs other than the initial ones
+        assert_equal [status $master sync_full] 4
+        assert_equal [status $replica1 sync_full] 0
+        assert_equal [status $replica2 sync_full] 0
+        assert_equal [status $replica3 sync_full] 0
+        assert_equal [status $replica4 sync_full] 0
+
+        # force psync
+        $master client kill type master
+        $replica2 client kill type master
+        $replica3 client kill type master
+        $replica4 client kill type master
+
+        # make sure replication is still alive and kicking
+        $replica1 incr x
+        wait_for_condition 50 1000 {
+            [$replica2 get x] == 3 &&
+            [$replica3 get x] == 3 &&
+            [$replica4 get x] == 3 &&
+            [$master get x] == 3
+        } else {
+            fail "replicas didn't get incr"
+        }
+
+        # make sure there are full syncs other than the initial ones
+        assert_equal [status $master sync_full] 4
+        assert_equal [status $replica1 sync_full] 0
+        assert_equal [status $replica2 sync_full] 0
+        assert_equal [status $replica3 sync_full] 0
+        assert_equal [status $replica4 sync_full] 0
+}
+}}}}}
+
+start_server {tags {"psync2"}} {
+start_server {} {
+start_server {} {
+
+    for {set j 0} {$j < 3} {incr j} {
+        set R($j) [srv [expr 0-$j] client]
+        set R_host($j) [srv [expr 0-$j] host]
+        set R_port($j) [srv [expr 0-$j] port]
+        $R($j) CONFIG SET repl-ping-replica-period 1
+    }
+
+    test "Chained replicas disconnect when replica re-connect with the same master" {
+        # Add a second replica as a chained replica of the current replica
+        $R(1) replicaof $R_host(0) $R_port(0)
+        $R(2) replicaof $R_host(1) $R_port(1)
+        wait_for_condition 50 1000 {
+            [status $R(2) master_link_status] == "up"
+        } else {
+            fail "Chained replica not replicating from its master"
+        }
+
+        # Do a write on the master, and wait for 3 seconds for the master to
+        # send some PINGs to its replica
+        $R(0) INCR counter2
+        after 2000
+        set sync_partial_master [status $R(0) sync_partial_ok]
+        set sync_partial_replica [status $R(1) sync_partial_ok]
+        $R(0) CONFIG SET repl-ping-replica-period 100
+
+        # Disconnect the master's direct replica
+        $R(0) client kill type replica
+        wait_for_condition 50 1000 {
+            [status $R(1) master_link_status] == "up" && 
+            [status $R(2) master_link_status] == "up" &&
+            [status $R(0) sync_partial_ok] == $sync_partial_master + 1 &&
+            [status $R(1) sync_partial_ok] == $sync_partial_replica + 1
+        } else {
+            fail "Disconnected replica failed to PSYNC with master"
+        }
+
+        # Verify that the replica and its replica's meaningful and real
+        # offsets match with the master
+        assert_equal [status $R(0) master_repl_offset] [status $R(1) master_repl_offset]
+        assert_equal [status $R(0) master_repl_offset] [status $R(2) master_repl_offset]
+        assert_equal [status $R(0) master_repl_meaningful_offset] [status $R(1) master_repl_meaningful_offset]
+        assert_equal [status $R(0) master_repl_meaningful_offset] [status $R(2) master_repl_meaningful_offset]
+    }
+}}}
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 05e45b999..de0a64728 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -47,6 +47,7 @@ set ::all_tests {
     integration/logging
     integration/psync2
     integration/psync2-reg
+    integration/psync2-pingoff
     unit/pubsub
     unit/slowlog
     unit/scripting

From 2a8af8e675755f2bc4beed7bbe9bcfc86a41e958 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 28 May 2020 09:10:51 +0300
Subject: [PATCH 0418/1098] adjust revived meaningful offset tests

these tests create several edge cases that are otherwise uncovered (at
least not consistently) by the test suite, so although they're no longer
testing what they were meant to test, it's still a good idea to keep
them in hope that they'll expose some issue in the future.
---
 tests/integration/psync2-pingoff.tcl | 59 ++++++++++++++++++----------
 1 file changed, 39 insertions(+), 20 deletions(-)

diff --git a/tests/integration/psync2-pingoff.tcl b/tests/integration/psync2-pingoff.tcl
index 5d8459a92..5a9a46d16 100644
--- a/tests/integration/psync2-pingoff.tcl
+++ b/tests/integration/psync2-pingoff.tcl
@@ -1,6 +1,9 @@
-# Test the meaningful offset implementation to make sure masters
-# are able to PSYNC with replicas even if the replication stream
-# has pending PINGs at the end.
+# These tests were added together with the meaningful offset implementation
+# in redis 6.0.0, which was later abandoned in 6.0.4, they used to test that
+# servers are able to PSYNC with replicas even if the replication stream has
+# PINGs at the end which present in one sever and missing on another.
+# We keep these tests just because they reproduce edge cases in the replication
+# logic in hope they'll be able to spot some problem in the future.
 
 start_server {tags {"psync2"}} {
 start_server {} {
@@ -16,7 +19,7 @@ start_server {} {
     }
 
     # Setup replication
-    test "PSYNC2 meaningful offset: setup" {
+    test "PSYNC2 pingoff: setup" {
         $R(1) replicaof $R_host(0) $R_port(0)
         $R(0) set foo bar
         wait_for_condition 50 1000 {
@@ -27,7 +30,7 @@ start_server {} {
         }
     }
 
-    test "PSYNC2 meaningful offset: write and wait replication" {
+    test "PSYNC2 pingoff: write and wait replication" {
         $R(0) INCR counter
         $R(0) INCR counter
         $R(0) INCR counter
@@ -41,7 +44,7 @@ start_server {} {
     # In this test we'll make sure the replica will get stuck, but with
     # an active connection: this way the master will continue to send PINGs
     # every second (we modified the PING period earlier)
-    test "PSYNC2 meaningful offset: pause replica and promote it" {
+    test "PSYNC2 pingoff: pause replica and promote it" {
         $R(1) MULTI
         $R(1) DEBUG SLEEP 5
         $R(1) SLAVEOF NO ONE
@@ -50,14 +53,22 @@ start_server {} {
     }
 
     test "Make the old master a replica of the new one and check conditions" {
-        set sync_partial [status $R(1) sync_partial_ok]
-        assert {$sync_partial == 0}
+        assert_equal [status $R(1) sync_full] 0
         $R(0) REPLICAOF $R_host(1) $R_port(1)
         wait_for_condition 50 1000 {
-            [status $R(1) sync_partial_ok] == 1
+            [status $R(1) sync_full] == 1
         } else {
-            fail "The new master was not able to partial sync"
+            fail "The new master was not able to sync"
         }
+
+        # make sure replication is still alive and kicking
+        $R(1) incr x
+        wait_for_condition 50 1000 {
+            [$R(0) get x] == 1
+        } else {
+            fail "replica didn't get incr"
+        }
+        assert_equal [status $R(0) master_repl_offset] [status $R(1) master_repl_offset]
     }
 }}
 
@@ -67,7 +78,7 @@ start_server {} {
 start_server {} {
 start_server {} {
 start_server {} {
-    test {pings at the end of replication stream are ignored for psync} {
+    test {test various edge cases of repl topology changes with missing pings at the end} {
         set master [srv -4 client]
         set master_host [srv -4 host]
         set master_port [srv -4 port]
@@ -129,9 +140,9 @@ start_server {} {
             fail "replicas didn't get incr"
         }
 
-        # make sure there are full syncs other than the initial ones
-        assert_equal [status $master sync_full] 4
-        assert_equal [status $replica1 sync_full] 0
+        # make sure we have the right amount of full syncs
+        assert_equal [status $master sync_full] 6
+        assert_equal [status $replica1 sync_full] 2
         assert_equal [status $replica2 sync_full] 0
         assert_equal [status $replica3 sync_full] 0
         assert_equal [status $replica4 sync_full] 0
@@ -153,9 +164,9 @@ start_server {} {
             fail "replicas didn't get incr"
         }
 
-        # make sure there are full syncs other than the initial ones
-        assert_equal [status $master sync_full] 4
-        assert_equal [status $replica1 sync_full] 0
+        # make sure we have the right amount of full syncs
+        assert_equal [status $master sync_full] 6
+        assert_equal [status $replica1 sync_full] 2
         assert_equal [status $replica2 sync_full] 0
         assert_equal [status $replica3 sync_full] 0
         assert_equal [status $replica4 sync_full] 0
@@ -197,7 +208,7 @@ start_server {} {
             [status $R(1) master_link_status] == "up" && 
             [status $R(2) master_link_status] == "up" &&
             [status $R(0) sync_partial_ok] == $sync_partial_master + 1 &&
-            [status $R(1) sync_partial_ok] == $sync_partial_replica + 1
+            [status $R(1) sync_partial_ok] == $sync_partial_replica
         } else {
             fail "Disconnected replica failed to PSYNC with master"
         }
@@ -206,7 +217,15 @@ start_server {} {
         # offsets match with the master
         assert_equal [status $R(0) master_repl_offset] [status $R(1) master_repl_offset]
         assert_equal [status $R(0) master_repl_offset] [status $R(2) master_repl_offset]
-        assert_equal [status $R(0) master_repl_meaningful_offset] [status $R(1) master_repl_meaningful_offset]
-        assert_equal [status $R(0) master_repl_meaningful_offset] [status $R(2) master_repl_meaningful_offset]
+
+        # make sure replication is still alive and kicking
+        $R(0) incr counter2
+        wait_for_condition 50 1000 {
+            [$R(1) get counter2] == 2 && [$R(2) get counter2] == 2
+        } else {
+            fail "replicas didn't get incr"
+        }
+        assert_equal [status $R(0) master_repl_offset] [status $R(1) master_repl_offset]
+        assert_equal [status $R(0) master_repl_offset] [status $R(2) master_repl_offset]
     }
 }}}

From 6a16a636bff425a14d641d21a5b058cc7cf72606 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 28 May 2020 10:08:16 +0200
Subject: [PATCH 0419/1098] Replication: showLatestBacklog() refactored out.

---
 src/networking.c  | 26 +-------------------------
 src/replication.c | 34 ++++++++++++++++++++++++++++++++++
 src/server.h      |  1 +
 3 files changed, 36 insertions(+), 25 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 7a5f1d7b9..8d3e057b7 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -396,31 +396,7 @@ void addReplyErrorLength(client *c, const char *s, size_t len) {
         if (ctype == CLIENT_TYPE_MASTER && server.repl_backlog &&
             server.repl_backlog_histlen > 0)
         {
-            long long dumplen = 256;
-            if (server.repl_backlog_histlen < dumplen)
-                dumplen = server.repl_backlog_histlen;
-
-            /* Identify the first byte to dump. */
-            long long idx =
-              (server.repl_backlog_idx + (server.repl_backlog_size - dumplen)) %
-               server.repl_backlog_size;
-
-            /* Scan the circular buffer to collect 'dumplen' bytes. */
-            sds dump = sdsempty();
-            while(dumplen) {
-                long long thislen =
-                    ((server.repl_backlog_size - idx) < dumplen) ?
-                    (server.repl_backlog_size - idx) : dumplen;
-
-                dump = sdscatrepr(dump,server.repl_backlog+idx,thislen);
-                dumplen -= thislen;
-                idx = 0;
-            }
-
-            /* Finally log such bytes: this is vital debugging info to
-             * understand what happened. */
-            serverLog(LL_WARNING,"Latest backlog is: '%s'", dump);
-            sdsfree(dump);
+            showLatestBacklog();
         }
         server.stat_unexpected_error_replies++;
     }
diff --git a/src/replication.c b/src/replication.c
index 110f0e82d..063a8705e 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -307,6 +307,40 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) {
     }
 }
 
+/* This is a debugging function that gets called when we detect something
+ * wrong with the replication protocol: the goal is to peek into the
+ * replication backlog and show a few final bytes to make simpler to
+ * guess what kind of bug it could be. */
+void showLatestBacklog(void) {
+    if (server.repl_backlog == NULL) return;
+
+    long long dumplen = 256;
+    if (server.repl_backlog_histlen < dumplen)
+        dumplen = server.repl_backlog_histlen;
+
+    /* Identify the first byte to dump. */
+    long long idx =
+      (server.repl_backlog_idx + (server.repl_backlog_size - dumplen)) %
+       server.repl_backlog_size;
+
+    /* Scan the circular buffer to collect 'dumplen' bytes. */
+    sds dump = sdsempty();
+    while(dumplen) {
+        long long thislen =
+            ((server.repl_backlog_size - idx) < dumplen) ?
+            (server.repl_backlog_size - idx) : dumplen;
+
+        dump = sdscatrepr(dump,server.repl_backlog+idx,thislen);
+        dumplen -= thislen;
+        idx = 0;
+    }
+
+    /* Finally log such bytes: this is vital debugging info to
+     * understand what happened. */
+    serverLog(LL_WARNING,"Latest backlog is: '%s'", dump);
+    sdsfree(dump);
+}
+
 /* This function is used in order to proxy what we receive from our master
  * to our sub-slaves. */
 #include <ctype.h>
diff --git a/src/server.h b/src/server.h
index 0c0b4d052..a08585292 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1810,6 +1810,7 @@ void clearReplicationId2(void);
 void chopReplicationBacklog(void);
 void replicationCacheMasterUsingMyself(void);
 void feedReplicationBacklog(void *ptr, size_t len);
+void showLatestBacklog(void);
 void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask);
 void rdbPipeWriteHandlerConnRemoved(struct connection *conn);
 

From 9d38a270dbc8ca465084e79521be218c83953396 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 28 May 2020 10:22:22 +0200
Subject: [PATCH 0420/1098] Test: add the tracking unit as default.

---
 tests/test_helper.tcl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 7fc0e6f00..fba54acb5 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -65,6 +65,7 @@ set ::all_tests {
     unit/wait
     unit/pendingquerybuf
     unit/tls
+    unit/tracking
 }
 # Index to the next test to run in the ::all_tests list.
 set ::next_test 0

From 23f2b4d0a8acef7a01f0b2c2ed16b6ee8719b18c Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 28 May 2020 10:47:27 +0200
Subject: [PATCH 0421/1098] Test: take PSYNC2 test master timeout high during
 switch.

This will likely avoid false positives due to trailing pings.
---
 tests/integration/psync2.tcl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl
index 7c50f620e..3f636463a 100644
--- a/tests/integration/psync2.tcl
+++ b/tests/integration/psync2.tcl
@@ -242,7 +242,6 @@ start_server {} {
             show_cluster_status
             fail "Replicas and master offsets were unable to match *exactly*."
         }
-        $R($master_id) config set repl-ping-replica-period 10
 
         # Limit anyway the maximum number of cycles. This is useful when the
         # test is skipped via --only option of the test suite. In that case

From 1f8ea99b4bc25083d909b3228601fb82efe1e67f Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 29 May 2020 11:07:13 +0200
Subject: [PATCH 0422/1098] Fix handling of special chars in ACL LOAD.

Now it is also possible for ACL SETUSER to accept empty strings
as valid operations (doing nothing), so for instance

    ACL SETUSER myuser ""

Will have just the effect of creating a user in the default state.

This should fix #7329.
---
 src/acl.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index c3a724c33..6dd0f70ac 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -732,10 +732,11 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) {
  * EEXIST: You are adding a key pattern after "*" was already added. This is
  *         almost surely an error on the user side.
  * ENODEV: The password you are trying to remove from the user does not exist.
- * EBADMSG: The hash you are trying to add is not a valid hash. 
+ * EBADMSG: The hash you are trying to add is not a valid hash.
  */
 int ACLSetUser(user *u, const char *op, ssize_t oplen) {
     if (oplen == -1) oplen = strlen(op);
+    if (oplen == 0) return C_OK; /* Empty string is a no-operation. */
     if (!strcasecmp(op,"on")) {
         u->flags |= USER_FLAG_ENABLED;
         u->flags &= ~USER_FLAG_DISABLED;
@@ -1297,7 +1298,7 @@ sds ACLLoadFromFile(const char *filename) {
         if (lines[i][0] == '\0') continue;
 
         /* Split into arguments */
-        argv = sdssplitargs(lines[i],&argc);
+        argv = sdssplitlen(lines[i],sdslen(lines[i])," ",1,&argc);
         if (argv == NULL) {
             errors = sdscatprintf(errors,
                      "%s:%d: unbalanced quotes in acl line. ",
@@ -1329,11 +1330,14 @@ sds ACLLoadFromFile(const char *filename) {
             continue;
         }
 
-        /* Try to process the line using the fake user to validate iif
-         * the rules are able to apply cleanly. */
+        /* Try to process the line using the fake user to validate if
+         * the rules are able to apply cleanly. At this stage we also
+         * trim trailing spaces, so that we don't have to handle that
+         * in ACLSetUser(). */
         ACLSetUser(fakeuser,"reset",-1);
         int j;
         for (j = 2; j < argc; j++) {
+            argv[j] = sdstrim(argv[j],"\t\r\n");
             if (ACLSetUser(fakeuser,argv[j],sdslen(argv[j])) != C_OK) {
                 char *errmsg = ACLSetUserStringError();
                 errors = sdscatprintf(errors,

From c480af900764cefbd60ac037d01065aad1b123d1 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 31 May 2020 15:51:52 +0300
Subject: [PATCH 0423/1098] fix pingoff  test race

---
 tests/integration/psync2-pingoff.tcl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/psync2-pingoff.tcl b/tests/integration/psync2-pingoff.tcl
index 5a9a46d16..cdecfc5c6 100644
--- a/tests/integration/psync2-pingoff.tcl
+++ b/tests/integration/psync2-pingoff.tcl
@@ -64,6 +64,7 @@ start_server {} {
         # make sure replication is still alive and kicking
         $R(1) incr x
         wait_for_condition 50 1000 {
+            [status $R(0) loading] == 0 &&
             [$R(0) get x] == 1
         } else {
             fail "replica didn't get incr"

From ad6b71352d3ad7ef44c2b9ed850b0c3a54eea4b6 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Tue, 2 Jun 2020 11:34:28 +0800
Subject: [PATCH 0424/1098] donot free protected client in
 freeClientsInAsyncFreeQueue

related #7234
---
 src/networking.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 8d3e057b7..cc28732d1 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1239,14 +1239,20 @@ void freeClientAsync(client *c) {
 /* Free the clietns marked as CLOSE_ASAP, return the number of clients
  * freed. */
 int freeClientsInAsyncFreeQueue(void) {
-    int freed = listLength(server.clients_to_close);
-    while (listLength(server.clients_to_close)) {
-        listNode *ln = listFirst(server.clients_to_close);
+    int freed = 0;
+    listIter li;
+    listNode *ln;
+
+    listRewind(server.clients_to_close,&li);
+    while ((ln = listNext(&li)) != NULL) {
         client *c = listNodeValue(ln);
 
+        if (c->flags & CLIENT_PROTECTED) continue;
+
         c->flags &= ~CLIENT_CLOSE_ASAP;
         freeClient(c);
         listDelNode(server.clients_to_close,ln);
+        freed++;
     }
     return freed;
 }

From 5d0774d62f4904db0ea0593c773a280ddc718d83 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Wed, 3 Jun 2020 17:55:18 +0800
Subject: [PATCH 0425/1098] AOF: append origin SET if no expire option

---
 src/aof.c             | 21 +++++++++++++--------
 tests/unit/expire.tcl | 10 ++++++++++
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index 02409abe6..6f8e53712 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -611,19 +611,24 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a
     } else if (cmd->proc == setCommand && argc > 3) {
         int i;
         robj *exarg = NULL, *pxarg = NULL;
-        /* Translate SET [EX seconds][PX milliseconds] to SET and PEXPIREAT */
-        buf = catAppendOnlyGenericCommand(buf,3,argv);
         for (i = 3; i < argc; i ++) {
             if (!strcasecmp(argv[i]->ptr, "ex")) exarg = argv[i+1];
             if (!strcasecmp(argv[i]->ptr, "px")) pxarg = argv[i+1];
         }
         serverAssert(!(exarg && pxarg));
-        if (exarg)
-            buf = catAppendOnlyExpireAtCommand(buf,server.expireCommand,argv[1],
-                                               exarg);
-        if (pxarg)
-            buf = catAppendOnlyExpireAtCommand(buf,server.pexpireCommand,argv[1],
-                                               pxarg);
+
+        if (exarg || pxarg) {
+            /* Translate SET [EX seconds][PX milliseconds] to SET and PEXPIREAT */
+            buf = catAppendOnlyGenericCommand(buf,3,argv);
+            if (exarg)
+                buf = catAppendOnlyExpireAtCommand(buf,server.expireCommand,argv[1],
+                                                   exarg);
+            if (pxarg)
+                buf = catAppendOnlyExpireAtCommand(buf,server.pexpireCommand,argv[1],
+                                                   pxarg);
+        } else {
+            buf = catAppendOnlyGenericCommand(buf,argc,argv);
+        }
     } else {
         /* All the other commands don't need translation or need the
          * same translation already operated in the command vector
diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl
index 11fb82ef0..52d174d75 100644
--- a/tests/unit/expire.tcl
+++ b/tests/unit/expire.tcl
@@ -232,4 +232,14 @@ start_server {tags {"expire"}} {
         set ttl [r ttl foo]
         assert {$ttl <= 100 && $ttl > 90}
     }
+
+    test {SET - use KEEPTTL option, TTL should not be removed after loadaof} {
+        r config set appendonly yes
+        r set foo bar EX 100
+        r set foo bar2 KEEPTTL
+        after 2000
+        r debug loadaof
+        set ttl [r ttl foo]
+        assert {$ttl <= 98 && $ttl > 90}
+    }
 }

From d1e23e04aadcb0053b280905862f43dd794ba29b Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Sat, 6 Jun 2020 11:42:41 +0200
Subject: [PATCH 0426/1098] Revert "avoid using sendfile if tls-replication is
 enabled"

This reverts commit b9abecfc4c97db01fa6f09180c68a92ea5974ac1.
---
 src/replication.c | 65 +++++++++++++++++++++--------------------------
 1 file changed, 29 insertions(+), 36 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index 063a8705e..7b15944a0 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1067,46 +1067,39 @@ void sendBulkToSlave(connection *conn) {
         }
     }
 
-    /* If the preamble was already transferred, send the RDB bulk data.
-     * try to use sendfile system call if supported, unless tls is enabled.
-     * fallback to normal read+write otherwise. */
-    nwritten = 0;
+    /* If the preamble was already transferred, send the RDB bulk data. */
 #if HAVE_SENDFILE
-    if (!server.tls_replication) {
-        if ((nwritten = redis_sendfile(conn->fd,slave->repldbfd,
-            slave->repldboff,PROTO_IOBUF_LEN)) == -1)
-        {
-            if (errno != EAGAIN) {
-                serverLog(LL_WARNING,"Sendfile error sending DB to replica: %s",
-                    strerror(errno));
-                freeClient(slave);
-            }
-            return;
+    if ((nwritten = redis_sendfile(conn->fd,slave->repldbfd,
+        slave->repldboff,PROTO_IOBUF_LEN)) == -1)
+    {
+        if (errno != EAGAIN) {
+            serverLog(LL_WARNING,"Sendfile error sending DB to replica: %s",
+                strerror(errno));
+            freeClient(slave);
         }
+        return;
+    }
+#else
+    ssize_t buflen;
+    char buf[PROTO_IOBUF_LEN];
+
+    lseek(slave->repldbfd,slave->repldboff,SEEK_SET);
+    buflen = read(slave->repldbfd,buf,PROTO_IOBUF_LEN);
+    if (buflen <= 0) {
+        serverLog(LL_WARNING,"Read error sending DB to replica: %s",
+            (buflen == 0) ? "premature EOF" : strerror(errno));
+        freeClient(slave);
+        return;
+    }
+    if ((nwritten = connWrite(conn,buf,buflen)) == -1) {
+        if (connGetState(conn) != CONN_STATE_CONNECTED) {
+            serverLog(LL_WARNING,"Write error sending DB to replica: %s",
+                connGetLastError(conn));
+            freeClient(slave);
+        }
+        return;
     }
 #endif
-    if (!nwritten) {
-        ssize_t buflen;
-        char buf[PROTO_IOBUF_LEN];
-
-        lseek(slave->repldbfd,slave->repldboff,SEEK_SET);
-        buflen = read(slave->repldbfd,buf,PROTO_IOBUF_LEN);
-        if (buflen <= 0) {
-            serverLog(LL_WARNING,"Read error sending DB to replica: %s",
-                (buflen == 0) ? "premature EOF" : strerror(errno));
-            freeClient(slave);
-            return;
-        }
-        if ((nwritten = connWrite(conn,buf,buflen)) == -1) {
-            if (connGetState(conn) != CONN_STATE_CONNECTED) {
-                serverLog(LL_WARNING,"Write error sending DB to replica: %s",
-                    connGetLastError(conn));
-                freeClient(slave);
-            }
-            return;
-        }
-    }
-
     slave->repldboff += nwritten;
     server.stat_net_output_bytes += nwritten;
     if (slave->repldboff == slave->repldbsize) {

From 61074b43a63feaa1655ef4e7d13808d315294443 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Sat, 6 Jun 2020 11:42:45 +0200
Subject: [PATCH 0427/1098] Revert "Implements sendfile for redis."

This reverts commit 9cf500a3f67e4e2ce51414c354e3472faf095d5b.
---
 src/config.h      |  6 ------
 src/replication.c | 51 ++---------------------------------------------
 2 files changed, 2 insertions(+), 55 deletions(-)

diff --git a/src/config.h b/src/config.h
index 23de5d3ad..0fcc42972 100644
--- a/src/config.h
+++ b/src/config.h
@@ -133,12 +133,6 @@ void setproctitle(const char *fmt, ...);
 /* Byte ordering detection */
 #include <sys/types.h> /* This will likely define BYTE_ORDER */
 
-/* Define redis_sendfile. */
-#if defined(__linux__) || (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_5))
-#define HAVE_SENDFILE 1
-ssize_t redis_sendfile(int out_fd, int in_fd, off_t offset, size_t count);
-#endif
-
 #ifndef BYTE_ORDER
 #if (BSD >= 199103)
 # include <machine/endian.h>
diff --git a/src/replication.c b/src/replication.c
index 7b15944a0..d9bff79ad 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1008,41 +1008,10 @@ void removeRDBUsedToSyncReplicas(void) {
     }
 }
 
-#if HAVE_SENDFILE
-/* Implements redis_sendfile to transfer data between file descriptors and
- * avoid transferring data to and from user space.
- * 
- * The function prototype is just like sendfile(2) on Linux. in_fd is a file
- * descriptor opened for reading and out_fd is a descriptor opened for writing.
- * offset specifies where to start reading data from in_fd. count is the number
- * of bytes to copy between the file descriptors.
- * 
- * The return value is the number of bytes written to out_fd, if the transfer
- * was successful. On error, -1 is returned, and errno is set appropriately. */
-ssize_t redis_sendfile(int out_fd, int in_fd, off_t offset, size_t count) {
-#if defined(__linux__)
-    #include <sys/sendfile.h>
-    return sendfile(out_fd, in_fd, &offset, count);
-
-#elif defined(__APPLE__)
-    off_t len = count;
-    /* Notice that it may return -1 and errno is set to EAGAIN even if some
-     * bytes have been sent successfully and the len argument is set correctly
-     * when using a socket marked for non-blocking I/O. */
-    if (sendfile(in_fd, out_fd, offset, &len, NULL, 0) == -1 &&
-        errno != EAGAIN) return -1;
-    else
-        return (ssize_t)len;
-
-#endif
-    errno = ENOSYS;
-    return -1;
-}
-#endif
-
 void sendBulkToSlave(connection *conn) {
     client *slave = connGetPrivateData(conn);
-    ssize_t nwritten;
+    char buf[PROTO_IOBUF_LEN];
+    ssize_t nwritten, buflen;
 
     /* Before sending the RDB file, we send the preamble as configured by the
      * replication process. Currently the preamble is just the bulk count of
@@ -1068,21 +1037,6 @@ void sendBulkToSlave(connection *conn) {
     }
 
     /* If the preamble was already transferred, send the RDB bulk data. */
-#if HAVE_SENDFILE
-    if ((nwritten = redis_sendfile(conn->fd,slave->repldbfd,
-        slave->repldboff,PROTO_IOBUF_LEN)) == -1)
-    {
-        if (errno != EAGAIN) {
-            serverLog(LL_WARNING,"Sendfile error sending DB to replica: %s",
-                strerror(errno));
-            freeClient(slave);
-        }
-        return;
-    }
-#else
-    ssize_t buflen;
-    char buf[PROTO_IOBUF_LEN];
-
     lseek(slave->repldbfd,slave->repldboff,SEEK_SET);
     buflen = read(slave->repldbfd,buf,PROTO_IOBUF_LEN);
     if (buflen <= 0) {
@@ -1099,7 +1053,6 @@ void sendBulkToSlave(connection *conn) {
         }
         return;
     }
-#endif
     slave->repldboff += nwritten;
     server.stat_net_output_bytes += nwritten;
     if (slave->repldboff == slave->repldbsize) {

From 7eba5c308a640e8087a4da78a3840652313ac4d1 Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Sun, 7 Jun 2020 13:34:55 +0800
Subject: [PATCH 0428/1098] return the correct proto version HELLO should
 return the current proto version, while the code hardcoded 3

---
 src/networking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index 8d3e057b7..fdfbe6bf1 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2522,7 +2522,7 @@ void helloCommand(client *c) {
     addReplyBulkCString(c,REDIS_VERSION);
 
     addReplyBulkCString(c,"proto");
-    addReplyLongLong(c,3);
+    addReplyLongLong(c,ver);
 
     addReplyBulkCString(c,"id");
     addReplyLongLong(c,c->id);

From 2fa077b0e93f9312590f6c7c8070eaa0db76747d Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 8 Jun 2020 09:16:32 +0300
Subject: [PATCH 0429/1098] Avoid rejecting WATCH / UNWATCH, like
 MULTI/EXEC/DISCARD

Much like MULTI/EXEC/DISCARD, the WATCH and UNWATCH are not actually
operating on the database or server state, but instead operate on the
client state. the client may send them all in one long pipeline and check
all the responses only at the end, so failing them may lead to a
mismatch between the client state on the server and the one on the
client end, and execute the wrong commands (ones that were meant to be
discarded)

the watched keys are not actually stored in the client struct, but they
are in fact part of the client state. for instance, they're not cleared
or moved in SWAPDB or FLUSHDB.
---
 src/server.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/server.c b/src/server.c
index b7a6a928f..e8e711240 100644
--- a/src/server.c
+++ b/src/server.c
@@ -776,11 +776,11 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"watch",watchCommand,-2,
-     "no-script fast @transaction",
+     "no-script fast ok-loading ok-stale @transaction",
      0,NULL,1,-1,1,0,0,0},
 
     {"unwatch",unwatchCommand,1,
-     "no-script fast @transaction",
+     "no-script fast ok-loading ok-stale @transaction",
      0,NULL,0,0,0,0,0,0},
 
     {"cluster",clusterCommand,-2,
@@ -3627,6 +3627,8 @@ int processCommand(client *c) {
           c->cmd->proc != multiCommand &&
           c->cmd->proc != execCommand &&
           c->cmd->proc != discardCommand &&
+          c->cmd->proc != watchCommand &&
+          c->cmd->proc != unwatchCommand &&
         !(c->cmd->proc == shutdownCommand &&
           c->argc == 2 &&
           tolower(((char*)c->argv[1]->ptr)[0]) == 'n') &&

From 2bb297b1029cf8938fe61b505450416dad3ea5ad Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 8 Jun 2020 09:43:10 +0300
Subject: [PATCH 0430/1098] Don't queue commands in an already aborted MULTI
 state

---
 src/multi.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/multi.c b/src/multi.c
index a331a6240..3e606fcec 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -58,6 +58,13 @@ void queueMultiCommand(client *c) {
     multiCmd *mc;
     int j;
 
+    /* No sense to waste memory if the transaction is already aborted.
+     * this is useful in case client sends these in a pipeline, or doesn't
+     * bother to read previous responses and didn't notice the multi was already
+     * aborted. */
+    if (c->flags & CLIENT_DIRTY_EXEC)
+        return;
+
     c->mstate.commands = zrealloc(c->mstate.commands,
             sizeof(multiCmd)*(c->mstate.count+1));
     mc = c->mstate.commands+c->mstate.count;

From 12504105c42b65127aa016562c713af63b4cbc71 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 8 Jun 2020 09:50:06 +0300
Subject: [PATCH 0431/1098] fix disconnectSlaves, to try to free each slave.

the recent change in that loop (iteration rather than waiting for it to
be empty) was intended to avoid an endless loop in case some slave would
refuse to be freed.

but the lookup of the first client remained, which would have caused it
to try the first one again and again instead of moving on.
---
 src/networking.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index 8d3e057b7..3513f9f9c 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1019,7 +1019,6 @@ void disconnectSlaves(void) {
     listNode *ln;
     listRewind(server.slaves,&li);
     while((ln = listNext(&li))) {
-        listNode *ln = listFirst(server.slaves);
         freeClient((client*)ln->value);
     }
 }

From 2a05fa0d481d12d3747465c4f14470bdca100c5d Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Mon, 8 Jun 2020 23:36:01 -0400
Subject: [PATCH 0432/1098] fix server crash in STRALGO command

---
 src/t_string.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/t_string.c b/src/t_string.c
index 5306069bf..d1a3e1b96 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -534,6 +534,13 @@ void stralgoLCS(client *c) {
             }
             obja = lookupKeyRead(c->db,c->argv[j+1]);
             objb = lookupKeyRead(c->db,c->argv[j+2]);
+
+            if ( !(obja->type == OBJ_STRING) || !(objb->type == OBJ_STRING) ) {
+                addReplyError(c,"Object associate with KEYS option should only be string type");
+                return;
+
+            }
+
             obja = obja ? getDecodedObject(obja) : createStringObject("",0);
             objb = objb ? getDecodedObject(objb) : createStringObject("",0);
             a = obja->ptr;

From cdad0e6485d108f2bf330ae2e8ae5fa7ef158ec1 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 9 Jun 2020 11:52:33 +0200
Subject: [PATCH 0433/1098] Temporary fix for #7353 issue about EVAL during
 -BUSY.

---
 src/multi.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/multi.c b/src/multi.c
index 3e606fcec..60a07dfc7 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -135,6 +135,15 @@ void execCommand(client *c) {
         return;
     }
 
+    /* If we are in -BUSY state, flag the transaction and return the
+     * -BUSY error, like Redis <= 5. This is a temporary fix, may be changed
+     *  ASAP, see issue #7353 on Github. */
+    if (server.lua_timedout) {
+        flagTransaction(c);
+        addReply(c, shared.slowscripterr);
+        return;
+    }
+
     /* Check if we need to abort the EXEC because:
      * 1) Some WATCHed key was touched.
      * 2) There was a previous error while queueing commands.

From 2ebcd63d6a011dc46df26d27b860209a28efe727 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 9 Jun 2020 12:19:14 +0200
Subject: [PATCH 0434/1098] Adapt EVAL+busy script test to new behavior.

---
 tests/unit/multi.tcl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl
index 55f18bec8..0c70fbde7 100644
--- a/tests/unit/multi.tcl
+++ b/tests/unit/multi.tcl
@@ -363,6 +363,9 @@ start_server {tags {"multi"}} {
         set xx [r get xx]
         # make sure that either the whole transcation passed or none of it (we actually expect none)
         assert { $xx == 1 || $xx == 3}
+        # Discard the transaction since EXEC likely got -BUSY error
+        # so the client is still in MULTI state.
+        catch { $rd2 discard ;$rd2 read } e
         # check that the connection is no longer in multi state
         $rd2 ping asdf
         set pong [$rd2 read]

From a7936ef96d521ef7e28a255e5105934f820e774a Mon Sep 17 00:00:00 2001
From: Paul Spooren <mail@aparcar.org>
Date: Tue, 9 Jun 2020 16:53:14 -0400
Subject: [PATCH 0435/1098] LRANK: Add command (the command will be renamed
 LPOS).

The `LRANK` command returns the index (position) of a given element
within a list. Using the `direction` argument it is possible to specify
going from head to tail (acending, 1) or from tail to head (decending,
-1). Only the first found index is returend. The complexity is O(N).

When using lists as a queue it can be of interest at what position a
given element is, for instance to monitor a job processing through a
work queue. This came up within the Python `rq` project which is based
on Redis[0].

[0]: https://github.com/rq/rq/issues/1197

Signed-off-by: Paul Spooren <mail@aparcar.org>
---
 src/help.h   |  5 +++++
 src/server.c |  4 ++++
 src/server.h |  1 +
 src/t_list.c | 34 ++++++++++++++++++++++++++++++++++
 4 files changed, 44 insertions(+)

diff --git a/src/help.h b/src/help.h
index 6d3eb33ed..9b4e90f9f 100644
--- a/src/help.h
+++ b/src/help.h
@@ -668,6 +668,11 @@ struct commandHelp {
     "Remove elements from a list",
     2,
     "1.0.0" },
+    { "LRANK",
+    "key direction element",
+    "Return first index of element in list based on direction",
+    2,
+    "9.9.9" },
     { "LSET",
     "key index element",
     "Set the value of an element in a list by its index",
diff --git a/src/server.c b/src/server.c
index e8e711240..767d3374f 100644
--- a/src/server.c
+++ b/src/server.c
@@ -326,6 +326,10 @@ struct redisCommand redisCommandTable[] = {
      "write @list",
      0,NULL,1,1,1,0,0,0},
 
+    {"lrank",lrankCommand,4,
+     "read-only fast @list",
+     0,NULL,1,1,1,0,0,0},
+
     {"lrem",lremCommand,4,
      "write @list",
      0,NULL,1,1,1,0,0,0},
diff --git a/src/server.h b/src/server.h
index a08585292..68932f656 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2269,6 +2269,7 @@ void flushdbCommand(client *c);
 void flushallCommand(client *c);
 void sortCommand(client *c);
 void lremCommand(client *c);
+void lrankCommand(client *c);
 void rpoplpushCommand(client *c);
 void infoCommand(client *c);
 void mgetCommand(client *c);
diff --git a/src/t_list.c b/src/t_list.c
index 4770a2272..899a20f47 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -487,6 +487,40 @@ void ltrimCommand(client *c) {
     addReply(c,shared.ok);
 }
 
+void lrankCommand(client *c) {
+    robj *subject, *obj;
+    obj = c->argv[3];
+    long direction = 0;
+    long index = 0;
+
+    if ((getLongFromObjectOrReply(c, c->argv[2], &direction, NULL) != C_OK))
+        return;
+
+    subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero);
+    if (subject == NULL || checkType(c,subject,OBJ_LIST)) return;
+
+    listTypeIterator *li;
+    if (direction < 0) {
+        direction = -1;
+        li = listTypeInitIterator(subject,-1,LIST_HEAD);
+    } else {
+        direction = 1;
+        li = listTypeInitIterator(subject,0,LIST_TAIL);
+    }
+
+    listTypeEntry entry;
+    while (listTypeNext(li,&entry)) {
+        if (listTypeEqual(&entry,obj)) {
+            break;
+        }
+        index++;
+    }
+
+    listTypeReleaseIterator(li);
+
+    addReplyLongLong(c,index * direction);
+}
+
 void lremCommand(client *c) {
     robj *subject, *obj;
     obj = c->argv[3];

From e63a5ba12223a7ab7d817ab42983be683aec0aca Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 10 Jun 2020 12:40:24 +0200
Subject: [PATCH 0436/1098] LPOS: implement the final design.

---
 src/server.c |   4 +-
 src/server.h |   2 +-
 src/t_list.c | 105 ++++++++++++++++++++++++++++++++++++++++-----------
 3 files changed, 87 insertions(+), 24 deletions(-)

diff --git a/src/server.c b/src/server.c
index 767d3374f..53dccf875 100644
--- a/src/server.c
+++ b/src/server.c
@@ -326,8 +326,8 @@ struct redisCommand redisCommandTable[] = {
      "write @list",
      0,NULL,1,1,1,0,0,0},
 
-    {"lrank",lrankCommand,4,
-     "read-only fast @list",
+    {"lpos",lposCommand,-3,
+     "read-only @list",
      0,NULL,1,1,1,0,0,0},
 
     {"lrem",lremCommand,4,
diff --git a/src/server.h b/src/server.h
index 68932f656..841e1f941 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2269,7 +2269,7 @@ void flushdbCommand(client *c);
 void flushallCommand(client *c);
 void sortCommand(client *c);
 void lremCommand(client *c);
-void lrankCommand(client *c);
+void lposCommand(client *c);
 void rpoplpushCommand(client *c);
 void infoCommand(client *c);
 void mgetCommand(client *c);
diff --git a/src/t_list.c b/src/t_list.c
index 899a20f47..3735280ff 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -487,38 +487,101 @@ void ltrimCommand(client *c) {
     addReply(c,shared.ok);
 }
 
-void lrankCommand(client *c) {
-    robj *subject, *obj;
-    obj = c->argv[3];
-    long direction = 0;
-    long index = 0;
+/* LPOS key element [matchpos] [ALL] [RELATIVE]
+ *
+ * matchnum is the position of the match, so if it is 1, the first match
+ * is returned, if it is 2 the second match is returned and so forth.
+ * It is 1 by default. If negative has the same meaning but the search is
+ * performed starting from the end of the list.
+ *
+ * A matchnum of 0 is accepted only if ALL is given, and means to return
+ * all the elements.
+ *
+ * If ALL is given, instead of returning the single elmenet, a list of
+ * all the matching elements up to "matchnum" are returned.
+ *
+ * The returned elements indexes are always referring to what LINDEX
+ * would return. So first element from head is 0, and so forth.
+ * However if RELATIVE is given and a negative matchpos is given, the
+ * indexes are returned as if the last element of the list is the element 0,
+ * the penultimante is 1, and so forth. */
+void lposCommand(client *c) {
+    robj *o, *ele;
+    ele = c->argv[2];
+    int all = 0, direction = LIST_TAIL;
+    long matchpos = 1;
 
-    if ((getLongFromObjectOrReply(c, c->argv[2], &direction, NULL) != C_OK))
+    /* Parse the optional "matchpos" argument, and the ALL option. */
+    if (c->argc >= 4 &&
+        getLongFromObjectOrReply(c, c->argv[3], &matchpos, NULL) != C_OK)
+    {
         return;
-
-    subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero);
-    if (subject == NULL || checkType(c,subject,OBJ_LIST)) return;
-
-    listTypeIterator *li;
-    if (direction < 0) {
-        direction = -1;
-        li = listTypeInitIterator(subject,-1,LIST_HEAD);
-    } else {
-        direction = 1;
-        li = listTypeInitIterator(subject,0,LIST_TAIL);
     }
 
+    if (c->argc == 5) {
+        if (!strcasecmp(c->argv[4]->ptr,"all")) {
+            all = 1;
+        } else {
+            addReply(c,shared.syntaxerr);
+            return;
+        }
+    }
+
+    /* Raise an error on incompatible options. */
+    if (!all && matchpos == 0) {
+        addReplyError(c,"A match position of zero is valid only "
+                        "when using the ALL option");
+        return;
+    }
+
+    /* A negative matchpos means start from the tail. */
+    if (matchpos < 0) {
+        matchpos = -matchpos;
+        direction = LIST_HEAD;
+    }
+
+    /* We return NULL or an empty array if there is no such key (or
+     * if we find no matches, depending on the presence of the ALL option. */
+    if ((o = lookupKeyWriteOrReply(c,c->argv[1],NULL)) == NULL) {
+        if (all)
+            addReply(c,shared.emptyarray);
+        else
+            addReply(c,shared.null[c->resp]);
+        return;
+    }
+    if (checkType(c,o,OBJ_LIST)) return;
+
+    /* If we got the ALL option, prepare to emit an array. */
+    void *arraylenptr = NULL;
+    if (all) arraylenptr = addReplyDeferredLen(c);
+
+    /* Seek the element. */
+    listTypeIterator *li;
+    li = listTypeInitIterator(o,direction == LIST_HEAD ? -1 : 0,direction);
     listTypeEntry entry;
+    long llen = listTypeLength(o);
+    long index = 0, matches = 0, matchindex = -1;
     while (listTypeNext(li,&entry)) {
-        if (listTypeEqual(&entry,obj)) {
-            break;
+        if (listTypeEqual(&entry,ele)) {
+            matches++;
+            matchindex = (direction == LIST_TAIL) ? index : llen - index - 1;
+            if (all) addReplyLongLong(c,matchindex);
+            if (matches == matchpos) break;
         }
         index++;
     }
-
     listTypeReleaseIterator(li);
 
-    addReplyLongLong(c,index * direction);
+    /* Reply to the client. Note that arraylenptr is not NULL only if
+     * the ALL option was selected. */
+    if (arraylenptr != NULL) {
+        setDeferredArrayLen(c,arraylenptr,matches);
+    } else {
+        if (matchindex != -1)
+            addReplyLongLong(c,matchindex);
+        else
+            addReply(c,shared.null[c->resp]);
+    }
 }
 
 void lremCommand(client *c) {

From 89ca2afd6b46496e3be5dbce2027b9381c4f431e Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 11 Jun 2020 11:18:16 +0200
Subject: [PATCH 0437/1098] LPOS: update to latest proposal.

See https://gist.github.com/antirez/3591c5096bc79cad8b5a992e08304f48
---
 src/t_list.c | 99 +++++++++++++++++++++++++++++++---------------------
 1 file changed, 59 insertions(+), 40 deletions(-)

diff --git a/src/t_list.c b/src/t_list.c
index 3735280ff..653337d78 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -487,63 +487,75 @@ void ltrimCommand(client *c) {
     addReply(c,shared.ok);
 }
 
-/* LPOS key element [matchpos] [ALL] [RELATIVE]
+/* LPOS key element [FIRST rank] [COUNT num-matches] [MAXLEN len]
  *
- * matchnum is the position of the match, so if it is 1, the first match
+ * FIRST "rank" is the position of the match, so if it is 1, the first match
  * is returned, if it is 2 the second match is returned and so forth.
  * It is 1 by default. If negative has the same meaning but the search is
  * performed starting from the end of the list.
  *
- * A matchnum of 0 is accepted only if ALL is given, and means to return
- * all the elements.
+ * If COUNT is given, instead of returning the single element, a list of
+ * all the matching elements up to "num-matches" are returned. COUNT can
+ * be combiled with FIRST in order to returning only the element starting
+ * from the Nth. If COUNT is zero, all the matching elements are returned.
  *
- * If ALL is given, instead of returning the single elmenet, a list of
- * all the matching elements up to "matchnum" are returned.
+ * MAXLEN tells the command to scan a max of len elements. If zero (the
+ * default), all the elements in the list are scanned if needed.
  *
  * The returned elements indexes are always referring to what LINDEX
- * would return. So first element from head is 0, and so forth.
- * However if RELATIVE is given and a negative matchpos is given, the
- * indexes are returned as if the last element of the list is the element 0,
- * the penultimante is 1, and so forth. */
+ * would return. So first element from head is 0, and so forth. */
 void lposCommand(client *c) {
     robj *o, *ele;
     ele = c->argv[2];
-    int all = 0, direction = LIST_TAIL;
-    long matchpos = 1;
+    int direction = LIST_TAIL;
+    long rank = 1, count = -1, maxlen = 0; /* Count -1: option not given. */
 
-    /* Parse the optional "matchpos" argument, and the ALL option. */
-    if (c->argc >= 4 &&
-        getLongFromObjectOrReply(c, c->argv[3], &matchpos, NULL) != C_OK)
-    {
-        return;
-    }
+    /* Parse the optional arguments. */
+    for (int j = 3; j < c->argc; j++) {
+        char *opt = c->argv[j]->ptr;
+        int moreargs = (c->argc-1)-j;
 
-    if (c->argc == 5) {
-        if (!strcasecmp(c->argv[4]->ptr,"all")) {
-            all = 1;
+        if (!strcasecmp(opt,"FIRST") && moreargs) {
+            j++;
+            if (getLongFromObjectOrReply(c, c->argv[j], &rank, NULL) != C_OK)
+                return;
+            if (rank == 0) {
+                addReplyError(c,"FIRST can't be zero: use 1 to start from "
+                                "the first match, 2 from the second, ...");
+                return;
+            }
+        } else if (!strcasecmp(opt,"COUNT") && moreargs) {
+            j++;
+            if (getLongFromObjectOrReply(c, c->argv[j], &count, NULL) != C_OK)
+                return;
+            if (count < 0) {
+                addReplyError(c,"COUNT can't be negative");
+                return;
+            }
+        } else if (!strcasecmp(opt,"MAXLEN") && moreargs) {
+            j++;
+            if (getLongFromObjectOrReply(c, c->argv[j], &maxlen, NULL) != C_OK)
+                return;
+            if (maxlen < 0) {
+                addReplyError(c,"MAXLEN can't be negative");
+                return;
+            }
         } else {
             addReply(c,shared.syntaxerr);
             return;
         }
     }
 
-    /* Raise an error on incompatible options. */
-    if (!all && matchpos == 0) {
-        addReplyError(c,"A match position of zero is valid only "
-                        "when using the ALL option");
-        return;
-    }
-
-    /* A negative matchpos means start from the tail. */
-    if (matchpos < 0) {
-        matchpos = -matchpos;
+    /* A negative rank means start from the tail. */
+    if (rank < 0) {
+        rank = -rank;
         direction = LIST_HEAD;
     }
 
     /* We return NULL or an empty array if there is no such key (or
-     * if we find no matches, depending on the presence of the ALL option. */
+     * if we find no matches, depending on the presence of the COUNT option. */
     if ((o = lookupKeyWriteOrReply(c,c->argv[1],NULL)) == NULL) {
-        if (all)
+        if (count != -1)
             addReply(c,shared.emptyarray);
         else
             addReply(c,shared.null[c->resp]);
@@ -551,9 +563,9 @@ void lposCommand(client *c) {
     }
     if (checkType(c,o,OBJ_LIST)) return;
 
-    /* If we got the ALL option, prepare to emit an array. */
+    /* If we got the COUNT option, prepare to emit an array. */
     void *arraylenptr = NULL;
-    if (all) arraylenptr = addReplyDeferredLen(c);
+    if (count != -1) arraylenptr = addReplyDeferredLen(c);
 
     /* Seek the element. */
     listTypeIterator *li;
@@ -561,21 +573,28 @@ void lposCommand(client *c) {
     listTypeEntry entry;
     long llen = listTypeLength(o);
     long index = 0, matches = 0, matchindex = -1;
-    while (listTypeNext(li,&entry)) {
+    while (listTypeNext(li,&entry) && (maxlen == 0 || index < maxlen)) {
         if (listTypeEqual(&entry,ele)) {
             matches++;
             matchindex = (direction == LIST_TAIL) ? index : llen - index - 1;
-            if (all) addReplyLongLong(c,matchindex);
-            if (matches == matchpos) break;
+            if (matches >= rank) {
+                if (arraylenptr) {
+                    addReplyLongLong(c,matchindex);
+                    if (count && matches-rank+1 >= count) break;
+                } else {
+                    break;
+                }
+            }
         }
         index++;
+        matchindex = -1; /* Remember if we exit the loop without a match. */
     }
     listTypeReleaseIterator(li);
 
     /* Reply to the client. Note that arraylenptr is not NULL only if
-     * the ALL option was selected. */
+     * the COUNT option was selected. */
     if (arraylenptr != NULL) {
-        setDeferredArrayLen(c,arraylenptr,matches);
+        setDeferredArrayLen(c,arraylenptr,matches-rank+1);
     } else {
         if (matchindex != -1)
             addReplyLongLong(c,matchindex);

From 0091125cae64d409f5f29a93076de065f1c8ff89 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 11 Jun 2020 12:38:51 +0200
Subject: [PATCH 0438/1098] LPOS: tests + crash fix.

---
 src/t_list.c             |  2 +-
 tests/unit/type/list.tcl | 44 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/src/t_list.c b/src/t_list.c
index 653337d78..e580139ab 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -554,7 +554,7 @@ void lposCommand(client *c) {
 
     /* We return NULL or an empty array if there is no such key (or
      * if we find no matches, depending on the presence of the COUNT option. */
-    if ((o = lookupKeyWriteOrReply(c,c->argv[1],NULL)) == NULL) {
+    if ((o = lookupKeyRead(c->db,c->argv[1])) == NULL) {
         if (count != -1)
             addReply(c,shared.emptyarray);
         else
diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl
index 676896a75..a0c04dcaa 100644
--- a/tests/unit/type/list.tcl
+++ b/tests/unit/type/list.tcl
@@ -6,6 +6,50 @@ start_server {
 } {
     source "tests/unit/type/list-common.tcl"
 
+    test {LPOS basic usage} {
+        r DEL mylist
+        r RPUSH mylist a b c 1 2 3 c c
+        assert {[r LPOS mylist a] == 0}
+        assert {[r LPOS mylist c] == 2}
+    }
+
+    test {LPOS FIRST (positive and negative rank) option} {
+        assert {[r LPOS mylist c FIRST 1] == 2}
+        assert {[r LPOS mylist c FIRST 2] == 6}
+        assert {[r LPOS mylist c FIRST 4] eq ""}
+        assert {[r LPOS mylist c FIRST -1] == 7}
+        assert {[r LPOS mylist c FIRST -2] == 6}
+    }
+
+    test {LPOS COUNT option} {
+        assert {[r LPOS mylist c COUNT 0] == {2 6 7}}
+        assert {[r LPOS mylist c COUNT 1] == {2}}
+        assert {[r LPOS mylist c COUNT 2] == {2 6}}
+        assert {[r LPOS mylist c COUNT 100] == {2 6 7}}
+    }
+
+    test {LPOS COUNT + FIRST option} {
+        assert {[r LPOS mylist c COUNT 0 FIRST 2] == {6 7}}
+        assert {[r LPOS mylist c COUNT 2 FIRST -1] == {7 6}}
+    }
+
+    test {LPOS non existing key} {
+        assert {[r LPOS mylistxxx c COUNT 0 FIRST 2] eq {}}
+    }
+
+    test {LPOS no match} {
+        assert {[r LPOS mylist x COUNT 2 FIRST -1] eq {}}
+        assert {[r LPOS mylist x FIRST -1] eq {}}
+    }
+
+    test {LPOS MAXLEN} {
+        assert {[r LPOS mylist a COUNT 0 MAXLEN 1] == {0}}
+        assert {[r LPOS mylist c COUNT 0 MAXLEN 1] == {}}
+        assert {[r LPOS mylist c COUNT 0 MAXLEN 3] == {2}}
+        assert {[r LPOS mylist c COUNT 0 MAXLEN 3 FIRST -1] == {7 6}}
+        assert {[r LPOS mylist c COUNT 0 MAXLEN 7 FIRST 2] == {6}}
+    }
+
     test {LPUSH, RPUSH, LLENGTH, LINDEX, LPOP - ziplist} {
         # first lpush then rpush
         assert_equal 1 [r lpush myziplist1 aa]

From 7008a0ba66fe13af0d584071eaa5fe3f34c56512 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Mon, 8 Jun 2020 23:48:51 -0400
Subject: [PATCH 0439/1098] fix memory leak

---
 src/t_string.c | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index d1a3e1b96..8e367ec80 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -516,13 +516,13 @@ void stralgoLCS(client *c) {
             withmatchlen = 1;
         } else if (!strcasecmp(opt,"MINMATCHLEN") && moreargs) {
             if (getLongLongFromObjectOrReply(c,c->argv[j+1],&minmatchlen,NULL)
-                != C_OK) return;
+                != C_OK) goto clean_up_obj;
             if (minmatchlen < 0) minmatchlen = 0;
             j++;
         } else if (!strcasecmp(opt,"STRINGS") && moreargs > 1) {
             if (a != NULL) {
                 addReplyError(c,"Either use STRINGS or KEYS");
-                return;
+                goto clean_up_obj;
             }
             a = c->argv[j+1]->ptr;
             b = c->argv[j+2]->ptr;
@@ -530,17 +530,14 @@ void stralgoLCS(client *c) {
         } else if (!strcasecmp(opt,"KEYS") && moreargs > 1) {
             if (a != NULL) {
                 addReplyError(c,"Either use STRINGS or KEYS");
-                return;
+                goto clean_up_obj;
             }
             obja = lookupKeyRead(c->db,c->argv[j+1]);
             objb = lookupKeyRead(c->db,c->argv[j+2]);
-
             if ( !(obja->type == OBJ_STRING) || !(objb->type == OBJ_STRING) ) {
                 addReplyError(c,"Object associate with KEYS option should only be string type");
-                return;
-
+                goto clean_up_obj;
             }
-
             obja = obja ? getDecodedObject(obja) : createStringObject("",0);
             objb = objb ? getDecodedObject(objb) : createStringObject("",0);
             a = obja->ptr;
@@ -548,7 +545,7 @@ void stralgoLCS(client *c) {
             j += 2;
         } else {
             addReply(c,shared.syntaxerr);
-            return;
+            goto clean_up_obj;
         }
     }
 
@@ -556,12 +553,12 @@ void stralgoLCS(client *c) {
     if (a == NULL) {
         addReplyError(c,"Please specify two strings: "
                         "STRINGS or KEYS options are mandatory");
-        return;
+        goto clean_up_obj;
     } else if (getlen && getidx) {
         addReplyError(c,
             "If you want both the length and indexes, please "
             "just use IDX.");
-        return;
+        goto clean_up_obj;
     }
 
     /* Compute the LCS using the vanilla dynamic programming technique of
@@ -696,10 +693,12 @@ void stralgoLCS(client *c) {
     }
 
     /* Cleanup. */
-    if (obja) decrRefCount(obja);
-    if (objb) decrRefCount(objb);
     sdsfree(result);
     zfree(lcs);
+
+clean_up_obj:
+    if (obja) decrRefCount(obja);
+    if (objb) decrRefCount(objb);
     return;
 }
 

From ca58198a76e70e148a06b4c695502d51c4ffe5cc Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 12 Jun 2020 12:16:19 +0200
Subject: [PATCH 0440/1098] help.h updated.

---
 src/help.h | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/src/help.h b/src/help.h
index 9b4e90f9f..1b1ac5e08 100644
--- a/src/help.h
+++ b/src/help.h
@@ -43,6 +43,16 @@ struct commandHelp {
     "Generate a pseudorandom secure password to use for ACL users",
     9,
     "6.0.0" },
+    { "ACL GETUSER",
+    "username",
+    "Get the rules for a specific ACL user",
+    9,
+    "6.0.0" },
+    { "ACL HELP",
+    "-",
+    "Show helpful text about the different subcommands",
+    9,
+    "6.0.0" },
     { "ACL LIST",
     "-",
     "List the current ACL rules in ACL config file format",
@@ -64,7 +74,7 @@ struct commandHelp {
     9,
     "6.0.0" },
     { "ACL SETUSER",
-    "rule [rule ...]",
+    "username [rule [rule ...]]",
     "Modify or create the rules for a specific ACL user",
     9,
     "6.0.0" },
@@ -164,7 +174,7 @@ struct commandHelp {
     8,
     "5.0.0" },
     { "CLIENT KILL",
-    "[ip:port] [ID client-id] [TYPE normal|master|slave|pubsub] [ADDR ip:port] [SKIPME yes/no]",
+    "[ip:port] [ID client-id] [TYPE normal|master|slave|pubsub] [USER username] [ADDR ip:port] [SKIPME yes/no]",
     "Kill the connection of a client",
     8,
     "2.4.0" },
@@ -182,14 +192,14 @@ struct commandHelp {
     "ON|OFF|SKIP",
     "Instruct the server whether to reply to commands",
     8,
-    "3.2" },
+    "3.2.0" },
     { "CLIENT SETNAME",
     "connection-name",
     "Set the current connection name",
     8,
     "2.6.9" },
     { "CLIENT TRACKING",
-    "ON|OFF [REDIRECT client-id] [PREFIX prefix] [BCAST] [OPTIN] [OPTOUT] [NOLOOP]",
+    "ON|OFF [REDIRECT client-id] [PREFIX prefix [PREFIX prefix ...]] [BCAST] [OPTIN] [OPTOUT] [NOLOOP]",
     "Enable or disable server assisted client side caching support",
     8,
     "6.0.0" },
@@ -619,7 +629,7 @@ struct commandHelp {
     9,
     "2.8.13" },
     { "LATENCY RESET",
-    "[event]",
+    "[event [event ...]]",
     "Reset latency data for one or more events.",
     9,
     "2.8.13" },
@@ -648,6 +658,11 @@ struct commandHelp {
     "Remove and get the first element in a list",
     2,
     "1.0.0" },
+    { "LPOS",
+    "key element [FIRST rank] [COUNT num-matches] [MAXLEN len]",
+    "Return the index of matching elements on a list",
+    2,
+    "6.0.6" },
     { "LPUSH",
     "key element [element ...]",
     "Prepend one or multiple elements to a list",
@@ -668,11 +683,6 @@ struct commandHelp {
     "Remove elements from a list",
     2,
     "1.0.0" },
-    { "LRANK",
-    "key direction element",
-    "Return first index of element in list based on direction",
-    2,
-    "9.9.9" },
     { "LSET",
     "key index element",
     "Set the value of an element in a list by its index",

From 10553988498acea1d772af69092b67fd5b56d529 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Fri, 12 Jun 2020 12:34:44 +0200
Subject: [PATCH 0441/1098] Fix LCS object type checking. Related to #7379.

---
 src/t_string.c | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index 8e367ec80..259f43142 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -516,13 +516,13 @@ void stralgoLCS(client *c) {
             withmatchlen = 1;
         } else if (!strcasecmp(opt,"MINMATCHLEN") && moreargs) {
             if (getLongLongFromObjectOrReply(c,c->argv[j+1],&minmatchlen,NULL)
-                != C_OK) goto clean_up_obj;
+                != C_OK) goto cleanup;
             if (minmatchlen < 0) minmatchlen = 0;
             j++;
         } else if (!strcasecmp(opt,"STRINGS") && moreargs > 1) {
             if (a != NULL) {
                 addReplyError(c,"Either use STRINGS or KEYS");
-                goto clean_up_obj;
+                goto cleanup;
             }
             a = c->argv[j+1]->ptr;
             b = c->argv[j+2]->ptr;
@@ -530,13 +530,20 @@ void stralgoLCS(client *c) {
         } else if (!strcasecmp(opt,"KEYS") && moreargs > 1) {
             if (a != NULL) {
                 addReplyError(c,"Either use STRINGS or KEYS");
-                goto clean_up_obj;
+                goto cleanup;
             }
             obja = lookupKeyRead(c->db,c->argv[j+1]);
             objb = lookupKeyRead(c->db,c->argv[j+2]);
-            if ( !(obja->type == OBJ_STRING) || !(objb->type == OBJ_STRING) ) {
-                addReplyError(c,"Object associate with KEYS option should only be string type");
-                goto clean_up_obj;
+            if ((obja && obja->type != OBJ_STRING) ||
+                (objb && objb->type != OBJ_STRING))
+            {
+                addReplyError(c,
+                    "The specified keys must contain string values");
+                /* Don't cleanup the objects, we need to do that
+                 * only after callign getDecodedObject(). */
+                obja = NULL;
+                objb = NULL;
+                goto cleanup;
             }
             obja = obja ? getDecodedObject(obja) : createStringObject("",0);
             objb = objb ? getDecodedObject(objb) : createStringObject("",0);
@@ -545,7 +552,7 @@ void stralgoLCS(client *c) {
             j += 2;
         } else {
             addReply(c,shared.syntaxerr);
-            goto clean_up_obj;
+            goto cleanup;
         }
     }
 
@@ -553,12 +560,12 @@ void stralgoLCS(client *c) {
     if (a == NULL) {
         addReplyError(c,"Please specify two strings: "
                         "STRINGS or KEYS options are mandatory");
-        goto clean_up_obj;
+        goto cleanup;
     } else if (getlen && getidx) {
         addReplyError(c,
             "If you want both the length and indexes, please "
             "just use IDX.");
-        goto clean_up_obj;
+        goto cleanup;
     }
 
     /* Compute the LCS using the vanilla dynamic programming technique of
@@ -696,7 +703,7 @@ void stralgoLCS(client *c) {
     sdsfree(result);
     zfree(lcs);
 
-clean_up_obj:
+cleanup:
     if (obja) decrRefCount(obja);
     if (objb) decrRefCount(objb);
     return;

From a89bf734a933e45b9dd3ae85ef4c3b62bd6891d8 Mon Sep 17 00:00:00 2001
From: "meir@redislabs.com" <meir@redislabs.com>
Date: Sun, 14 Jun 2020 10:06:00 +0300
Subject: [PATCH 0442/1098] Fix RM_ScanKey module api not to return int encoded
 strings

The scan key module API provides the scan callback with the current
field name and value (if it exists). Those arguments are RedisModuleString*
which means it supposes to point to robj which is encoded as a string.
Using createStringObjectFromLongLong function might return robj that
points to an integer and so break a module that tries for example to
use RedisModule_StringPtrLen on the given field/value.

The PR introduces a fix that uses the createObject function and sdsfromlonglong function.
Using those function promise that the field and value pass to the to the
scan callback will be Strings.

The PR also changes the Scan test module to use RedisModule_StringPtrLen
to catch the issue. without this, the issue is hidden because
RedisModule_ReplyWithString knows to handle integer encoding of the
given robj (RedisModuleString).

The PR also introduces a new test to verify the issue is solved.
---
 src/module.c                  |  6 +++---
 tests/modules/scan.c          | 20 ++++++++++++++++----
 tests/unit/moduleapi/scan.tcl |  5 +++++
 3 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/src/module.c b/src/module.c
index e3a338dad..226c60fd0 100644
--- a/src/module.c
+++ b/src/module.c
@@ -6708,7 +6708,7 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc
         int pos = 0;
         int64_t ll;
         while(intsetGet(o->ptr,pos++,&ll)) {
-            robj *field = createStringObjectFromLongLong(ll);
+            robj *field = createObject(OBJ_STRING,sdsfromlonglong(ll));
             fn(key, field, NULL, privdata);
             decrRefCount(field);
         }
@@ -6724,12 +6724,12 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc
             ziplistGet(p,&vstr,&vlen,&vll);
             robj *field = (vstr != NULL) ?
                 createStringObject((char*)vstr,vlen) :
-                createStringObjectFromLongLong(vll);
+                createObject(OBJ_STRING,sdsfromlonglong(vll));
             p = ziplistNext(o->ptr,p);
             ziplistGet(p,&vstr,&vlen,&vll);
             robj *value = (vstr != NULL) ?
                 createStringObject((char*)vstr,vlen) :
-                createStringObjectFromLongLong(vll);
+                createObject(OBJ_STRING,sdsfromlonglong(vll));
             fn(key, field, value, privdata);
             p = ziplistNext(o->ptr,p);
             decrRefCount(field);
diff --git a/tests/modules/scan.c b/tests/modules/scan.c
index afede244b..1576bae9e 100644
--- a/tests/modules/scan.c
+++ b/tests/modules/scan.c
@@ -55,11 +55,23 @@ void scan_key_callback(RedisModuleKey *key, RedisModuleString* field, RedisModul
     REDISMODULE_NOT_USED(key);
     scan_key_pd* pd = privdata;
     RedisModule_ReplyWithArray(pd->ctx, 2);
-    RedisModule_ReplyWithString(pd->ctx, field);
-    if (value)
-        RedisModule_ReplyWithString(pd->ctx, value);
-    else
+    size_t fieldCStrLen;
+
+    // The implementation of RedisModuleString is robj with lots of encodings.
+    // We want to make sure the robj that passes to this callback in
+    // String encoded, this is why we use RedisModule_StringPtrLen and
+    // RedisModule_ReplyWithStringBuffer instead of directly use
+    // RedisModule_ReplyWithString.
+    const char* fieldCStr = RedisModule_StringPtrLen(field, &fieldCStrLen);
+    RedisModule_ReplyWithStringBuffer(pd->ctx, fieldCStr, fieldCStrLen);
+    if(value){
+        size_t valueCStrLen;
+        const char* valueCStr = RedisModule_StringPtrLen(value, &valueCStrLen);
+        RedisModule_ReplyWithStringBuffer(pd->ctx, valueCStr, valueCStrLen);
+    } else {
         RedisModule_ReplyWithNull(pd->ctx);
+    }
+
     pd->nreplies++;
 }
 
diff --git a/tests/unit/moduleapi/scan.tcl b/tests/unit/moduleapi/scan.tcl
index de1672e0a..43a0c4d8a 100644
--- a/tests/unit/moduleapi/scan.tcl
+++ b/tests/unit/moduleapi/scan.tcl
@@ -16,6 +16,11 @@ start_server {tags {"modules"}} {
         r hmset hh f1 v1 f2 v2
         lsort [r scan.scan_key hh]
     } {{f1 v1} {f2 v2}}
+    
+    test {Module scan hash dict with int value} {
+        r hmset hh1 f1 1 
+        lsort [r scan.scan_key hh1]
+    } {{f1 1}}
 
     test {Module scan hash dict} {
         r config set hash-max-ziplist-entries 2

From c92464db694172dac8b0f9eeedd366c494d6db8a Mon Sep 17 00:00:00 2001
From: root <root@mongo.config1.com>
Date: Mon, 15 Jun 2020 10:18:14 +0800
Subject: [PATCH 0443/1098] cluster.c remove if of clusterSendFail in
 markNodeAsFailingIfNeeded

---
 src/cluster.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cluster.c b/src/cluster.c
index 24b14d1dc..332d5a4ac 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -1255,7 +1255,7 @@ void markNodeAsFailingIfNeeded(clusterNode *node) {
 
     /* Broadcast the failing node name to everybody, forcing all the other
      * reachable nodes to flag the node as FAIL. */
-    if (nodeIsMaster(myself)) clusterSendFail(node->name);
+    clusterSendFail(node->name);
     clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG);
 }
 

From 784479939d9e560835a9eb7a410304b46047d5f5 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 16 Jun 2020 11:09:45 +0200
Subject: [PATCH 0444/1098] Tracking: fix enableBcastTrackingForPrefix()
 invalid sdslen() call.

Related to #7387.
---
 src/tracking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tracking.c b/src/tracking.c
index eb4113131..8c2dca7ba 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -102,7 +102,7 @@ void disableTracking(client *c) {
 /* Set the client 'c' to track the prefix 'prefix'. If the client 'c' is
  * already registered for the specified prefix, no operation is performed. */
 void enableBcastTrackingForPrefix(client *c, char *prefix, size_t plen) {
-    bcastState *bs = raxFind(PrefixTable,(unsigned char*)prefix,sdslen(prefix));
+    bcastState *bs = raxFind(PrefixTable,(unsigned char*)prefix,plen);
     /* If this is the first client subscribing to such prefix, create
      * the prefix in the table. */
     if (bs == raxNotFound) {

From 4b8d8826afa0f240b26977e9d128144ebf8d5d7a Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Tue, 16 Jun 2020 11:45:03 +0200
Subject: [PATCH 0445/1098] Use cluster connections too, to limit maxclients.

See #7401.
---
 src/cluster.c    |  7 +++++++
 src/cluster.h    |  1 +
 src/networking.c | 23 +++++++++++++++--------
 3 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 24b14d1dc..cacf09bf3 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -691,6 +691,13 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
     }
 }
 
+/* Return the approximated number of sockets we are using in order to
+ * take the cluster bus connections. */
+unsigned long getClusterConnectionsCount(void) {
+    return server.cluster_enabled ?
+           (dictSize(server.cluster->nodes)*2) : 0;
+}
+
 /* -----------------------------------------------------------------------------
  * Key space handling
  * -------------------------------------------------------------------------- */
diff --git a/src/cluster.h b/src/cluster.h
index d3af4a355..596a4629a 100644
--- a/src/cluster.h
+++ b/src/cluster.h
@@ -283,5 +283,6 @@ typedef struct {
 clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, int *ask);
 int clusterRedirectBlockedClientIfNeeded(client *c);
 void clusterRedirectClient(client *c, clusterNode *n, int hashslot, int error_code);
+unsigned long getClusterConnectionsCount(void);
 
 #endif /* __CLUSTER_H */
diff --git a/src/networking.c b/src/networking.c
index 77b9a6fcf..9d36ed3a2 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -892,17 +892,24 @@ static void acceptCommonHandler(connection *conn, int flags, char *ip) {
     client *c;
     UNUSED(ip);
 
-    /* Admission control will happen before a client is created and connAccept()
+    /* Limit the number of connections we take at the same time.
+     *
+     * Admission control will happen before a client is created and connAccept()
      * called, because we don't want to even start transport-level negotiation
-     * if rejected.
-     */
-    if (listLength(server.clients) >= server.maxclients) {
-        char *err = "-ERR max number of clients reached\r\n";
+     * if rejected. */
+    if (listLength(server.clients) + getClusterConnectionsCount()
+        >= server.maxclients)
+    {
+        char *err;
+        if (server.cluster_enabled)
+            err = "-ERR max number of clients reached\r\n";
+        else
+            err = "-ERR max number of clients + cluster "
+                  "connections reached\r\n";
 
         /* That's a best effort error message, don't check write errors.
-         * Note that for TLS connections, no handshake was done yet so nothing is written
-         * and the connection will just drop.
-         */
+         * Note that for TLS connections, no handshake was done yet so nothing
+         * is written and the connection will just drop. */
         if (connWrite(conn,err,strlen(err)) == -1) {
             /* Nothing to do, Just to avoid the warning... */
         }

From 71fafd761ad5f939f485259eb3856ed3766c98be Mon Sep 17 00:00:00 2001
From: chenhui0212 <chenhui19900212@gmail.com>
Date: Tue, 16 Jun 2020 17:50:38 +0800
Subject: [PATCH 0446/1098] fix comments in listpack.c

---
 src/listpack.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/listpack.c b/src/listpack.c
index e1f4d9a02..9e77ab12d 100644
--- a/src/listpack.c
+++ b/src/listpack.c
@@ -773,13 +773,13 @@ unsigned char *lpSeek(unsigned char *lp, long index) {
          * is past the half of the listpack. */
         if (index > numele/2) {
             forward = 0;
-            /* Left to right scanning always expects a negative index. Convert
+            /* Right to left scanning always expects a negative index. Convert
              * our index to negative form. */
             index -= numele;
         }
     } else {
         /* If the listpack length is unspecified, for negative indexes we
-         * want to always scan left-to-right. */
+         * want to always scan right-to-left. */
         if (index < 0) forward = 0;
     }
 

From 4ee011adb5f0d56085ecd3e5d643ab192ef77ce6 Mon Sep 17 00:00:00 2001
From: Tomasz Poradowski <tomasz.poradowski@generiscorp.com>
Date: Wed, 17 Jun 2020 22:22:49 +0200
Subject: [PATCH 0447/1098] ensure SHUTDOWN_NOSAVE in Sentinel mode

- enforcing of SHUTDOWN_NOSAVE flag in one place to make it consitent
  when running in Sentinel mode
---
 src/db.c     | 8 --------
 src/server.c | 9 +++++++++
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/db.c b/src/db.c
index dc4a0b63e..19b2c48e4 100644
--- a/src/db.c
+++ b/src/db.c
@@ -963,14 +963,6 @@ void shutdownCommand(client *c) {
             return;
         }
     }
-    /* When SHUTDOWN is called while the server is loading a dataset in
-     * memory we need to make sure no attempt is performed to save
-     * the dataset on shutdown (otherwise it could overwrite the current DB
-     * with half-read data).
-     *
-     * Also when in Sentinel mode clear the SAVE flag and force NOSAVE. */
-    if (server.loading || server.sentinel_mode)
-        flags = (flags & ~SHUTDOWN_SAVE) | SHUTDOWN_NOSAVE;
     if (prepareForShutdown(flags) == C_OK) exit(0);
     addReplyError(c,"Errors trying to SHUTDOWN. Check logs.");
 }
diff --git a/src/server.c b/src/server.c
index 53dccf875..7c92c9244 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3680,6 +3680,15 @@ void closeListeningSockets(int unlink_unix_socket) {
 }
 
 int prepareForShutdown(int flags) {
+    /* When SHUTDOWN is called while the server is loading a dataset in
+     * memory we need to make sure no attempt is performed to save
+     * the dataset on shutdown (otherwise it could overwrite the current DB
+     * with half-read data).
+     *
+     * Also when in Sentinel mode clear the SAVE flag and force NOSAVE. */
+    if (server.loading || server.sentinel_mode)
+        flags = (flags & ~SHUTDOWN_SAVE) | SHUTDOWN_NOSAVE;
+
     int save = flags & SHUTDOWN_SAVE;
     int nosave = flags & SHUTDOWN_NOSAVE;
 

From f800b52172622b84aa8bdf09aa6cd522aade93be Mon Sep 17 00:00:00 2001
From: chenhui0212 <chenhui19900212@gmail.com>
Date: Thu, 18 Jun 2020 17:28:26 +0800
Subject: [PATCH 0448/1098] Fix comments in function raxLowWalk of listpack.c

---
 src/rax.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/rax.c b/src/rax.c
index 7dcf04582..c8a1fb6b4 100644
--- a/src/rax.c
+++ b/src/rax.c
@@ -487,8 +487,8 @@ static inline size_t raxLowWalk(rax *rax, unsigned char *s, size_t len, raxNode
         if (h->iscompr) j = 0; /* Compressed node only child is at index 0. */
         memcpy(&h,children+j,sizeof(h));
         parentlink = children+j;
-        j = 0; /* If the new node is compressed and we do not
-                  iterate again (since i == l) set the split
+        j = 0; /* If the new node is non compressed and we do not
+                  iterate again (since i == len) set the split
                   position to 0 to signal this node represents
                   the searched key. */
     }

From 1bfa2d27a637119226ee3244d2d219c7e5a7ff33 Mon Sep 17 00:00:00 2001
From: hwware <wen.hui.ware@gmail.com>
Date: Sun, 21 Jun 2020 23:04:28 -0400
Subject: [PATCH 0449/1098] fix memory leak in sentinel connection sharing

---
 src/sentinel.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/sentinel.c b/src/sentinel.c
index fb504ae4d..5be4193dc 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -1076,6 +1076,7 @@ int sentinelTryConnectionSharing(sentinelRedisInstance *ri) {
         releaseInstanceLink(ri->link,NULL);
         ri->link = match->link;
         match->link->refcount++;
+        dictReleaseIterator(di);
         return C_OK;
     }
     dictReleaseIterator(di);

From 59fd178014c7cca1b0c668b30ab0d991dd3030f3 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 22 Jun 2020 11:21:21 +0200
Subject: [PATCH 0450/1098] Clarify maxclients and cluster in conf. Remove
 myself too.

---
 redis.conf    | 5 +++++
 src/cluster.c | 5 ++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/redis.conf b/redis.conf
index 5cea06a47..a51ef007d 100644
--- a/redis.conf
+++ b/redis.conf
@@ -805,6 +805,11 @@ acllog-max-len 128
 # Once the limit is reached Redis will close all the new connections sending
 # an error 'max number of clients reached'.
 #
+# IMPORTANT: When Redis Cluster is used, the max number of connections is also
+# shared with the cluster bus: every node in the cluster will use two
+# connections, one incoming and another outgoing. It is important to size the
+# limit accordingly in case of very large clusters.
+#
 # maxclients 10000
 
 ############################## MEMORY MANAGEMENT ################################
diff --git a/src/cluster.c b/src/cluster.c
index cacf09bf3..87ed09e7e 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -694,8 +694,11 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
 /* Return the approximated number of sockets we are using in order to
  * take the cluster bus connections. */
 unsigned long getClusterConnectionsCount(void) {
+    /* We decrement the number of nodes by one, since there is the
+     * "myself" node too in the list. Each node uses two file descriptors,
+     * one incoming and one outgoing, thus the multiplication by 2. */
     return server.cluster_enabled ?
-           (dictSize(server.cluster->nodes)*2) : 0;
+           ((dictSize(server.cluster->nodes)-1)*2) : 0;
 }
 
 /* -----------------------------------------------------------------------------

From 746297314f3c198a90ceab8462a40abd9fc69f26 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 22 Jun 2020 11:41:19 +0200
Subject: [PATCH 0451/1098] Fix BITFIELD i64 type handling, see #7417.

---
 src/bitops.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/bitops.c b/src/bitops.c
index f506a881b..b37bea2bf 100644
--- a/src/bitops.c
+++ b/src/bitops.c
@@ -257,7 +257,7 @@ int64_t getSignedBitfield(unsigned char *p, uint64_t offset, uint64_t bits) {
     /* If the top significant bit is 1, propagate it to all the
      * higher bits for two's complement representation of signed
      * integers. */
-    if (value & ((uint64_t)1 << (bits-1)))
+    if (bits < 64 && (value & ((uint64_t)1 << (bits-1))))
         value |= ((uint64_t)-1) << bits;
     return value;
 }
@@ -356,7 +356,6 @@ int checkSignedBitfieldOverflow(int64_t value, int64_t incr, uint64_t bits, int
 
 handle_wrap:
     {
-        uint64_t mask = ((uint64_t)-1) << bits;
         uint64_t msb = (uint64_t)1 << (bits-1);
         uint64_t a = value, b = incr, c;
         c = a+b; /* Perform addition as unsigned so that's defined. */
@@ -364,10 +363,13 @@ handle_wrap:
         /* If the sign bit is set, propagate to all the higher order
          * bits, to cap the negative value. If it's clear, mask to
          * the positive integer limit. */
-        if (c & msb) {
-            c |= mask;
-        } else {
-            c &= ~mask;
+        if (bits < 64) {
+            uint64_t mask = ((uint64_t)-1) << bits;
+            if (c & msb) {
+                c |= mask;
+            } else {
+                c &= ~mask;
+            }
         }
         *limit = c;
     }

From 21f62c3346ab22ac37f81110e59dae9372c9e4d1 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Mon, 22 Jun 2020 11:44:11 +0200
Subject: [PATCH 0452/1098] Include cluster.h for getClusterConnectionsCount().

---
 src/networking.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/networking.c b/src/networking.c
index 9d36ed3a2..2795deafd 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -29,6 +29,7 @@
 
 #include "server.h"
 #include "atomicvar.h"
+#include "cluster.h"
 #include <sys/socket.h>
 #include <sys/uio.h>
 #include <math.h>

From 65a3307bc95aadbc91d85cdf9dfbe1b3493222ca Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 11 Jun 2020 21:09:35 +0300
Subject: [PATCH 0453/1098] EXEC always fails with EXECABORT and multi-state is
 cleared

In order to support the use of multi-exec in pipeline, it is important that
MULTI and EXEC are never rejected and it is easy for the client to know if the
connection is still in multi state.

It was easy to make sure MULTI and DISCARD never fail (done by previous
commits) since these only change the client state and don't do any actual
change in the server, but EXEC is a different story.

Since in the past, it was possible for clients to handle some EXEC errors and
retry the EXEC, we now can't affort to return any error on EXEC other than
EXECABORT, which now carries with it the real reason for the abort too.

Other fixes in this commit:
- Some checks that where performed at the time of queuing need to be re-
  validated when EXEC runs, for instance if the transaction contains writes
  commands, it needs to be aborted. there was one check that was already done
  in execCommand (-READONLY), but other checks where missing: -OOM, -MISCONF,
  -NOREPLICAS, -MASTERDOWN
- When a command is rejected by processCommand it was rejected with addReply,
  which was not recognized as an error in case the bad command came from the
  master. this will enable to count or MONITOR these errors in the future.
- make it easier for tests to create additional (non deferred) clients.
- add tests for the fixes of this commit.
---
 src/multi.c           |  43 +++++++--------
 src/networking.c      |  20 ++++---
 src/server.c          |  93 ++++++++++++++++++++-------------
 src/server.h          |   5 ++
 tests/test_helper.tcl |  15 ++++++
 tests/unit/multi.tcl  | 119 +++++++++++++++++++++++++++++++++---------
 6 files changed, 204 insertions(+), 91 deletions(-)

diff --git a/src/multi.c b/src/multi.c
index 60a07dfc7..35ddf92af 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -36,6 +36,7 @@ void initClientMultiState(client *c) {
     c->mstate.commands = NULL;
     c->mstate.count = 0;
     c->mstate.cmd_flags = 0;
+    c->mstate.cmd_inv_flags = 0;
 }
 
 /* Release all the resources associated with MULTI/EXEC state */
@@ -76,6 +77,7 @@ void queueMultiCommand(client *c) {
         incrRefCount(mc->argv[j]);
     c->mstate.count++;
     c->mstate.cmd_flags |= c->cmd->flags;
+    c->mstate.cmd_inv_flags |= ~c->cmd->flags;
 }
 
 void discardTransaction(client *c) {
@@ -122,6 +124,23 @@ void execCommandPropagateExec(client *c) {
               PROPAGATE_AOF|PROPAGATE_REPL);
 }
 
+/* Aborts a transaction, with a specific error message.
+ * The transaction is always aboarted with -EXECABORT so that the client knows
+ * the server exited the multi state, but the actual reason for the abort is
+ * included too. */
+void execCommandAbort(client *c, sds error) {
+    discardTransaction(c);
+
+    if (error[0] == '-') error++;
+    addReplyErrorFormat(c, "-EXECABORT Transaction discarded because of: %s", error);
+
+    /* Send EXEC to clients waiting data from MONITOR. We did send a MULTI
+     * already, and didn't send any of the queued commands, now we'll just send
+     * EXEC so it is clear that the transaction is over. */
+    if (listLength(server.monitors) && !server.loading)
+        replicationFeedMonitors(c,server.monitors,c->db->id,c->argv,c->argc);
+}
+
 void execCommand(client *c) {
     int j;
     robj **orig_argv;
@@ -135,15 +154,6 @@ void execCommand(client *c) {
         return;
     }
 
-    /* If we are in -BUSY state, flag the transaction and return the
-     * -BUSY error, like Redis <= 5. This is a temporary fix, may be changed
-     *  ASAP, see issue #7353 on Github. */
-    if (server.lua_timedout) {
-        flagTransaction(c);
-        addReply(c, shared.slowscripterr);
-        return;
-    }
-
     /* Check if we need to abort the EXEC because:
      * 1) Some WATCHed key was touched.
      * 2) There was a previous error while queueing commands.
@@ -157,21 +167,6 @@ void execCommand(client *c) {
         goto handle_monitor;
     }
 
-    /* If there are write commands inside the transaction, and this is a read
-     * only slave, we want to send an error. This happens when the transaction
-     * was initiated when the instance was a master or a writable replica and
-     * then the configuration changed (for example instance was turned into
-     * a replica). */
-    if (!server.loading && server.masterhost && server.repl_slave_ro &&
-        !(c->flags & CLIENT_MASTER) && c->mstate.cmd_flags & CMD_WRITE)
-    {
-        addReplyError(c,
-            "Transaction contains write commands but instance "
-            "is now a read-only replica. EXEC aborted.");
-        discardTransaction(c);
-        goto handle_monitor;
-    }
-
     /* Exec all the queued commands */
     unwatchAllKeys(c); /* Unwatch ASAP otherwise we'll waste CPU cycles */
     orig_argv = c->argv;
diff --git a/src/networking.c b/src/networking.c
index 77b9a6fcf..539b1fb50 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -406,19 +406,23 @@ void addReplyError(client *c, const char *err) {
     addReplyErrorLength(c,err,strlen(err));
 }
 
+/* See addReplyErrorLength.
+ * Makes sure there are no newlines in the string, otherwise invalid protocol
+ * is emitted. */
+void addReplyErrorSafe(client *c, char *s, size_t len) {
+    size_t j;
+    for (j = 0; j < len; j++) {
+        if (s[j] == '\r' || s[j] == '\n') s[j] = ' ';
+    }
+    addReplyErrorLength(c,s,sdslen(s));
+}
+
 void addReplyErrorFormat(client *c, const char *fmt, ...) {
-    size_t l, j;
     va_list ap;
     va_start(ap,fmt);
     sds s = sdscatvprintf(sdsempty(),fmt,ap);
     va_end(ap);
-    /* Make sure there are no newlines in the string, otherwise invalid protocol
-     * is emitted. */
-    l = sdslen(s);
-    for (j = 0; j < l; j++) {
-        if (s[j] == '\r' || s[j] == '\n') s[j] = ' ';
-    }
-    addReplyErrorLength(c,s,sdslen(s));
+    addReplyErrorSafe(c, s, sdslen(s));
     sdsfree(s);
 }
 
diff --git a/src/server.c b/src/server.c
index e8e711240..678dc305a 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3398,6 +3398,34 @@ void call(client *c, int flags) {
     server.stat_numcommands++;
 }
 
+/* Used when a command that is ready for execution needs to be rejected, due to
+ * varios pre-execution checks. it returns the appropriate error to the client.
+ * If there's a transaction is flags it as dirty, and if the command is EXEC,
+ * it aborts the transaction. */
+void rejectCommand(client *c, robj *reply) {
+    flagTransaction(c);
+    if (c->cmd && c->cmd->proc == execCommand) {
+        execCommandAbort(c, reply->ptr);
+    } else {
+        /* using addReplyError* rather than addReply so that the error can be logged. */
+        addReplyErrorSafe(c, reply->ptr, sdslen(reply->ptr));
+    }
+}
+
+void rejectCommandFormat(client *c, const char *fmt, ...) {
+    flagTransaction(c);
+    va_list ap;
+    va_start(ap,fmt);
+    sds s = sdscatvprintf(sdsempty(),fmt,ap);
+    va_end(ap);
+    if (c->cmd && c->cmd->proc == execCommand) {
+        execCommandAbort(c, s);
+    } else {
+        addReplyErrorSafe(c, s, sdslen(s));
+    }
+    sdsfree(s);
+}
+
 /* If this function gets called we already read a whole
  * command, arguments are in the client argv/argc fields.
  * processCommand() execute the command or prepare the
@@ -3423,23 +3451,30 @@ int processCommand(client *c) {
      * such as wrong arity, bad command name and so forth. */
     c->cmd = c->lastcmd = lookupCommand(c->argv[0]->ptr);
     if (!c->cmd) {
-        flagTransaction(c);
         sds args = sdsempty();
         int i;
         for (i=1; i < c->argc && sdslen(args) < 128; i++)
             args = sdscatprintf(args, "`%.*s`, ", 128-(int)sdslen(args), (char*)c->argv[i]->ptr);
-        addReplyErrorFormat(c,"unknown command `%s`, with args beginning with: %s",
+        rejectCommandFormat(c,"unknown command `%s`, with args beginning with: %s",
             (char*)c->argv[0]->ptr, args);
         sdsfree(args);
         return C_OK;
     } else if ((c->cmd->arity > 0 && c->cmd->arity != c->argc) ||
                (c->argc < -c->cmd->arity)) {
-        flagTransaction(c);
-        addReplyErrorFormat(c,"wrong number of arguments for '%s' command",
+        rejectCommandFormat(c,"wrong number of arguments for '%s' command",
             c->cmd->name);
         return C_OK;
     }
 
+    int is_write_command = (c->cmd->flags & CMD_WRITE) ||
+                           (c->cmd->proc == execCommand && (c->mstate.cmd_flags & CMD_WRITE));
+    int is_denyoom_command = (c->cmd->flags & CMD_DENYOOM) ||
+                             (c->cmd->proc == execCommand && (c->mstate.cmd_flags & CMD_DENYOOM));
+    int is_denystale_command = !(c->cmd->flags & CMD_STALE) ||
+                               (c->cmd->proc == execCommand && (c->mstate.cmd_inv_flags & CMD_STALE));
+    int is_denyloading_command = !(c->cmd->flags & CMD_LOADING) ||
+                                 (c->cmd->proc == execCommand && (c->mstate.cmd_inv_flags & CMD_LOADING));
+
     /* Check if the user is authenticated. This check is skipped in case
      * the default user is flagged as "nopass" and is active. */
     int auth_required = (!(DefaultUser->flags & USER_FLAG_NOPASS) ||
@@ -3449,8 +3484,7 @@ int processCommand(client *c) {
         /* AUTH and HELLO and no auth modules are valid even in
          * non-authenticated state. */
         if (!(c->cmd->flags & CMD_NO_AUTH)) {
-            flagTransaction(c);
-            addReply(c,shared.noautherr);
+            rejectCommand(c,shared.noautherr);
             return C_OK;
         }
     }
@@ -3461,13 +3495,12 @@ int processCommand(client *c) {
     int acl_retval = ACLCheckCommandPerm(c,&acl_keypos);
     if (acl_retval != ACL_OK) {
         addACLLogEntry(c,acl_retval,acl_keypos,NULL);
-        flagTransaction(c);
         if (acl_retval == ACL_DENIED_CMD)
-            addReplyErrorFormat(c,
+            rejectCommandFormat(c,
                 "-NOPERM this user has no permissions to run "
                 "the '%s' command or its subcommand", c->cmd->name);
         else
-            addReplyErrorFormat(c,
+            rejectCommandFormat(c,
                 "-NOPERM this user has no permissions to access "
                 "one of the keys used as arguments");
         return C_OK;
@@ -3515,13 +3548,11 @@ int processCommand(client *c) {
          * is trying to execute is denied during OOM conditions or the client
          * is in MULTI/EXEC context? Error. */
         if (out_of_memory &&
-            (c->cmd->flags & CMD_DENYOOM ||
+            (is_denyoom_command ||
              (c->flags & CLIENT_MULTI &&
-              c->cmd->proc != execCommand &&
               c->cmd->proc != discardCommand)))
         {
-            flagTransaction(c);
-            addReply(c, shared.oomerr);
+            rejectCommand(c, shared.oomerr);
             return C_OK;
         }
 
@@ -3542,17 +3573,14 @@ int processCommand(client *c) {
     int deny_write_type = writeCommandsDeniedByDiskError();
     if (deny_write_type != DISK_ERROR_TYPE_NONE &&
         server.masterhost == NULL &&
-        (c->cmd->flags & CMD_WRITE ||
-         c->cmd->proc == pingCommand))
+        (is_write_command ||c->cmd->proc == pingCommand))
     {
-        flagTransaction(c);
         if (deny_write_type == DISK_ERROR_TYPE_RDB)
-            addReply(c, shared.bgsaveerr);
+            rejectCommand(c, shared.bgsaveerr);
         else
-            addReplySds(c,
-                sdscatprintf(sdsempty(),
+            rejectCommandFormat(c,
                 "-MISCONF Errors writing to the AOF file: %s\r\n",
-                strerror(server.aof_last_write_errno)));
+                strerror(server.aof_last_write_errno));
         return C_OK;
     }
 
@@ -3561,11 +3589,10 @@ int processCommand(client *c) {
     if (server.masterhost == NULL &&
         server.repl_min_slaves_to_write &&
         server.repl_min_slaves_max_lag &&
-        c->cmd->flags & CMD_WRITE &&
+        is_write_command &&
         server.repl_good_slaves_count < server.repl_min_slaves_to_write)
     {
-        flagTransaction(c);
-        addReply(c, shared.noreplicaserr);
+        rejectCommand(c, shared.noreplicaserr);
         return C_OK;
     }
 
@@ -3573,10 +3600,9 @@ int processCommand(client *c) {
      * accept write commands if this is our master. */
     if (server.masterhost && server.repl_slave_ro &&
         !(c->flags & CLIENT_MASTER) &&
-        c->cmd->flags & CMD_WRITE)
+        is_write_command)
     {
-        flagTransaction(c);
-        addReply(c, shared.roslaveerr);
+        rejectCommand(c, shared.roslaveerr);
         return C_OK;
     }
 
@@ -3588,7 +3614,7 @@ int processCommand(client *c) {
         c->cmd->proc != unsubscribeCommand &&
         c->cmd->proc != psubscribeCommand &&
         c->cmd->proc != punsubscribeCommand) {
-        addReplyErrorFormat(c,
+        rejectCommandFormat(c,
             "Can't execute '%s': only (P)SUBSCRIBE / "
             "(P)UNSUBSCRIBE / PING / QUIT are allowed in this context",
             c->cmd->name);
@@ -3600,17 +3626,16 @@ int processCommand(client *c) {
      * link with master. */
     if (server.masterhost && server.repl_state != REPL_STATE_CONNECTED &&
         server.repl_serve_stale_data == 0 &&
-        !(c->cmd->flags & CMD_STALE))
+        is_denystale_command)
     {
-        flagTransaction(c);
-        addReply(c, shared.masterdownerr);
+        rejectCommand(c, shared.masterdownerr);
         return C_OK;
     }
 
     /* Loading DB? Return an error if the command has not the
      * CMD_LOADING flag. */
-    if (server.loading && !(c->cmd->flags & CMD_LOADING)) {
-        addReply(c, shared.loadingerr);
+    if (server.loading && is_denyloading_command) {
+        rejectCommand(c, shared.loadingerr);
         return C_OK;
     }
 
@@ -3625,7 +3650,6 @@ int processCommand(client *c) {
           c->cmd->proc != helloCommand &&
           c->cmd->proc != replconfCommand &&
           c->cmd->proc != multiCommand &&
-          c->cmd->proc != execCommand &&
           c->cmd->proc != discardCommand &&
           c->cmd->proc != watchCommand &&
           c->cmd->proc != unwatchCommand &&
@@ -3636,8 +3660,7 @@ int processCommand(client *c) {
           c->argc == 2 &&
           tolower(((char*)c->argv[1]->ptr)[0]) == 'k'))
     {
-        flagTransaction(c);
-        addReply(c, shared.slowscripterr);
+        rejectCommand(c, shared.slowscripterr);
         return C_OK;
     }
 
diff --git a/src/server.h b/src/server.h
index a08585292..b887afee1 100644
--- a/src/server.h
+++ b/src/server.h
@@ -666,6 +666,9 @@ typedef struct multiState {
     int cmd_flags;          /* The accumulated command flags OR-ed together.
                                So if at least a command has a given flag, it
                                will be set in this field. */
+    int cmd_inv_flags;      /* Same as cmd_flags, OR-ing the ~flags. so that it
+                               is possible to know if all the commands have a
+                               certain flag. */
     int minreplicas;        /* MINREPLICAS for synchronous replication */
     time_t minreplicas_timeout; /* MINREPLICAS timeout as unixtime. */
 } multiState;
@@ -1626,6 +1629,7 @@ void addReplyBulkLongLong(client *c, long long ll);
 void addReply(client *c, robj *obj);
 void addReplySds(client *c, sds s);
 void addReplyBulkSds(client *c, sds s);
+void addReplyErrorSafe(client *c, char *s, size_t len);
 void addReplyError(client *c, const char *err);
 void addReplyStatus(client *c, const char *status);
 void addReplyDouble(client *c, double d);
@@ -1724,6 +1728,7 @@ void touchWatchedKey(redisDb *db, robj *key);
 void touchWatchedKeysOnFlush(int dbid);
 void discardTransaction(client *c);
 void flagTransaction(client *c);
+void execCommandAbort(client *c, sds error);
 void execCommandPropagateMulti(client *c);
 void execCommandPropagateExec(client *c);
 
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index fba54acb5..ef9bf7fdf 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -196,6 +196,21 @@ proc redis_deferring_client {args} {
     return $client
 }
 
+proc redis_client {args} {
+    set level 0
+    if {[llength $args] > 0 && [string is integer [lindex $args 0]]} {
+        set level [lindex $args 0]
+        set args [lrange $args 1 end]
+    }
+
+    # create client that defers reading reply
+    set client [redis [srv $level "host"] [srv $level "port"] 0 $::tls]
+
+    # select the right db and read the response (OK)
+    $client select 9
+    return $client
+}
+
 # Provide easy access to INFO properties. Same semantic as "proc r".
 proc s {args} {
     set level 0
diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl
index 0c70fbde7..44a822ba6 100644
--- a/tests/unit/multi.tcl
+++ b/tests/unit/multi.tcl
@@ -325,74 +325,145 @@ start_server {tags {"multi"}} {
         # check that if MULTI arrives during timeout, it is either refused, or
         # allowed to pass, and we don't end up executing half of the transaction
         set rd1 [redis_deferring_client]
-        set rd2 [redis_deferring_client]
+        set r2 [redis_client]
         r config set lua-time-limit 10
         r set xx 1
         $rd1 eval {while true do end} 0
         after 200
-        catch { $rd2 multi; $rd2 read } e
-        catch { $rd2 incr xx; $rd2 read } e
+        catch { $r2 multi; } e
+        catch { $r2 incr xx; } e
         r script kill
         after 200 ; # Give some time to Lua to call the hook again...
-        catch { $rd2 incr xx; $rd2 read } e
-        catch { $rd2 exec; $rd2 read } e
+        catch { $r2 incr xx; } e
+        catch { $r2 exec; } e
+        assert_match {EXECABORT*previous errors*} $e
         set xx [r get xx]
         # make sure that either the whole transcation passed or none of it (we actually expect none)
         assert { $xx == 1 || $xx == 3}
         # check that the connection is no longer in multi state
-        $rd2 ping asdf
-        set pong [$rd2 read]
+        set pong [$r2 ping asdf]
         assert_equal $pong "asdf"
+        $rd1 close; $r2 close
     }
 
     test {EXEC and script timeout} {
         # check that if EXEC arrives during timeout, we don't end up executing
         # half of the transaction, and also that we exit the multi state
         set rd1 [redis_deferring_client]
-        set rd2 [redis_deferring_client]
+        set r2 [redis_client]
         r config set lua-time-limit 10
         r set xx 1
-        catch { $rd2 multi; $rd2 read } e
-        catch { $rd2 incr xx; $rd2 read } e
+        catch { $r2 multi; } e
+        catch { $r2 incr xx; } e
         $rd1 eval {while true do end} 0
         after 200
-        catch { $rd2 incr xx; $rd2 read } e
-        catch { $rd2 exec; $rd2 read } e
+        catch { $r2 incr xx; } e
+        catch { $r2 exec; } e
+        assert_match {EXECABORT*BUSY*} $e
         r script kill
         after 200 ; # Give some time to Lua to call the hook again...
         set xx [r get xx]
         # make sure that either the whole transcation passed or none of it (we actually expect none)
         assert { $xx == 1 || $xx == 3}
-        # Discard the transaction since EXEC likely got -BUSY error
-        # so the client is still in MULTI state.
-        catch { $rd2 discard ;$rd2 read } e
         # check that the connection is no longer in multi state
-        $rd2 ping asdf
-        set pong [$rd2 read]
+        set pong [$r2 ping asdf]
         assert_equal $pong "asdf"
+        $rd1 close; $r2 close
     }
 
     test {MULTI-EXEC body and script timeout} {
         # check that we don't run an imcomplete transaction due to some commands
         # arriving during busy script
         set rd1 [redis_deferring_client]
-        set rd2 [redis_deferring_client]
+        set r2 [redis_client]
         r config set lua-time-limit 10
         r set xx 1
-        catch { $rd2 multi; $rd2 read } e
-        catch { $rd2 incr xx; $rd2 read } e
+        catch { $r2 multi; } e
+        catch { $r2 incr xx; } e
         $rd1 eval {while true do end} 0
         after 200
-        catch { $rd2 incr xx; $rd2 read } e
+        catch { $r2 incr xx; } e
         r script kill
         after 200 ; # Give some time to Lua to call the hook again...
-        catch { $rd2 exec; $rd2 read } e
+        catch { $r2 exec; } e
+        assert_match {EXECABORT*previous errors*} $e
         set xx [r get xx]
         # make sure that either the whole transcation passed or none of it (we actually expect none)
         assert { $xx == 1 || $xx == 3}
         # check that the connection is no longer in multi state
-        $rd2 ping asdf
-        set pong [$rd2 read]
+        set pong [$r2 ping asdf]
         assert_equal $pong "asdf"
+        $rd1 close; $r2 close
+    }
+
+    test {just EXEC and script timeout} {
+        # check that if EXEC arrives during timeout, we don't end up executing
+        # actual commands during busy script, and also that we exit the multi state
+        set rd1 [redis_deferring_client]
+        set r2 [redis_client]
+        r config set lua-time-limit 10
+        r set xx 1
+        catch { $r2 multi; } e
+        catch { $r2 incr xx; } e
+        $rd1 eval {while true do end} 0
+        after 200
+        catch { $r2 exec; } e
+        assert_match {EXECABORT*BUSY*} $e
+        r script kill
+        after 200 ; # Give some time to Lua to call the hook again...
+        set xx [r get xx]
+        # make we didn't execute the transaction
+        assert { $xx == 1}
+        # check that the connection is no longer in multi state
+        set pong [$r2 ping asdf]
+        assert_equal $pong "asdf"
+        $rd1 close; $r2 close
+    }
+
+    test {exec with write commands and state change} {
+        # check that exec that contains write commands fails if server state changed since they were queued
+        set r1 [redis_client]
+        r set xx 1
+        r multi
+        r incr xx
+        $r1 config set min-replicas-to-write 2
+        catch {r exec} e
+        assert_match {*EXECABORT*NOREPLICAS*} $e
+        set xx [r get xx]
+        # make sure that the INCR wasn't executed
+        assert { $xx == 1}
+        $r1 config set min-replicas-to-write 0
+        $r1 close;
+    }
+
+    test {exec with read commands and stale replica state change} {
+        # check that exec that contains read commands fails if server state changed since they were queued
+        r config set replica-serve-stale-data no
+        set r1 [redis_client]
+        r set xx 1
+
+        # check that GET is disallowed on stale replica, even if the replica becomes stale only after queuing.
+        r multi
+        r get xx
+        $r1 replicaof localhsot 0
+        catch {r exec} e
+        assert_match {*EXECABORT*MASTERDOWN*} $e
+
+        # check that PING is allowed
+        r multi
+        r ping
+        $r1 replicaof localhsot 0
+        set pong [r exec]
+        assert {$pong == "PONG"}
+
+        # check that when replica is not stale, GET is allowed
+        # while we're at it, let's check that multi is allowed on stale replica too
+        r multi
+        $r1 replicaof no one
+        r get xx
+        set xx [r exec]
+        # make sure that the INCR was executed
+        assert { $xx == 1 }
+        $r1 close;
     }
 }

From 2d6d9f75524f280b9bc02613ebe9727b92c1429e Mon Sep 17 00:00:00 2001
From: Dave Nielsen <dnielsen@gmail.com>
Date: Tue, 23 Jun 2020 09:51:12 -0700
Subject: [PATCH 0454/1098] updated copyright year

Changed "2015" to "2020"
---
 COPYING | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/COPYING b/COPYING
index ac68e012b..a381681a1 100644
--- a/COPYING
+++ b/COPYING
@@ -1,4 +1,4 @@
-Copyright (c) 2006-2015, Salvatore Sanfilippo
+Copyright (c) 2006-2020, Salvatore Sanfilippo
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

From a5a3a7bbc61203398ecc1d5b52c76214f5672776 Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Wed, 24 Jun 2020 09:07:17 +0200
Subject: [PATCH 0455/1098] LPOS: option FIRST renamed RANK.

---
 src/t_list.c             | 10 +++++-----
 tests/unit/type/list.tcl | 28 ++++++++++++++--------------
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/t_list.c b/src/t_list.c
index e580139ab..2c339888d 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -487,16 +487,16 @@ void ltrimCommand(client *c) {
     addReply(c,shared.ok);
 }
 
-/* LPOS key element [FIRST rank] [COUNT num-matches] [MAXLEN len]
+/* LPOS key element [RANK rank] [COUNT num-matches] [MAXLEN len]
  *
- * FIRST "rank" is the position of the match, so if it is 1, the first match
+ * The "rank" is the position of the match, so if it is 1, the first match
  * is returned, if it is 2 the second match is returned and so forth.
  * It is 1 by default. If negative has the same meaning but the search is
  * performed starting from the end of the list.
  *
  * If COUNT is given, instead of returning the single element, a list of
  * all the matching elements up to "num-matches" are returned. COUNT can
- * be combiled with FIRST in order to returning only the element starting
+ * be combiled with RANK in order to returning only the element starting
  * from the Nth. If COUNT is zero, all the matching elements are returned.
  *
  * MAXLEN tells the command to scan a max of len elements. If zero (the
@@ -515,12 +515,12 @@ void lposCommand(client *c) {
         char *opt = c->argv[j]->ptr;
         int moreargs = (c->argc-1)-j;
 
-        if (!strcasecmp(opt,"FIRST") && moreargs) {
+        if (!strcasecmp(opt,"RANK") && moreargs) {
             j++;
             if (getLongFromObjectOrReply(c, c->argv[j], &rank, NULL) != C_OK)
                 return;
             if (rank == 0) {
-                addReplyError(c,"FIRST can't be zero: use 1 to start from "
+                addReplyError(c,"RANK can't be zero: use 1 to start from "
                                 "the first match, 2 from the second, ...");
                 return;
             }
diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl
index a0c04dcaa..0e39d7d95 100644
--- a/tests/unit/type/list.tcl
+++ b/tests/unit/type/list.tcl
@@ -13,12 +13,12 @@ start_server {
         assert {[r LPOS mylist c] == 2}
     }
 
-    test {LPOS FIRST (positive and negative rank) option} {
-        assert {[r LPOS mylist c FIRST 1] == 2}
-        assert {[r LPOS mylist c FIRST 2] == 6}
-        assert {[r LPOS mylist c FIRST 4] eq ""}
-        assert {[r LPOS mylist c FIRST -1] == 7}
-        assert {[r LPOS mylist c FIRST -2] == 6}
+    test {LPOS RANK (positive and negative rank) option} {
+        assert {[r LPOS mylist c RANK 1] == 2}
+        assert {[r LPOS mylist c RANK 2] == 6}
+        assert {[r LPOS mylist c RANK 4] eq ""}
+        assert {[r LPOS mylist c RANK -1] == 7}
+        assert {[r LPOS mylist c RANK -2] == 6}
     }
 
     test {LPOS COUNT option} {
@@ -28,26 +28,26 @@ start_server {
         assert {[r LPOS mylist c COUNT 100] == {2 6 7}}
     }
 
-    test {LPOS COUNT + FIRST option} {
-        assert {[r LPOS mylist c COUNT 0 FIRST 2] == {6 7}}
-        assert {[r LPOS mylist c COUNT 2 FIRST -1] == {7 6}}
+    test {LPOS COUNT + RANK option} {
+        assert {[r LPOS mylist c COUNT 0 RANK 2] == {6 7}}
+        assert {[r LPOS mylist c COUNT 2 RANK -1] == {7 6}}
     }
 
     test {LPOS non existing key} {
-        assert {[r LPOS mylistxxx c COUNT 0 FIRST 2] eq {}}
+        assert {[r LPOS mylistxxx c COUNT 0 RANK 2] eq {}}
     }
 
     test {LPOS no match} {
-        assert {[r LPOS mylist x COUNT 2 FIRST -1] eq {}}
-        assert {[r LPOS mylist x FIRST -1] eq {}}
+        assert {[r LPOS mylist x COUNT 2 RANK -1] eq {}}
+        assert {[r LPOS mylist x RANK -1] eq {}}
     }
 
     test {LPOS MAXLEN} {
         assert {[r LPOS mylist a COUNT 0 MAXLEN 1] == {0}}
         assert {[r LPOS mylist c COUNT 0 MAXLEN 1] == {}}
         assert {[r LPOS mylist c COUNT 0 MAXLEN 3] == {2}}
-        assert {[r LPOS mylist c COUNT 0 MAXLEN 3 FIRST -1] == {7 6}}
-        assert {[r LPOS mylist c COUNT 0 MAXLEN 7 FIRST 2] == {6}}
+        assert {[r LPOS mylist c COUNT 0 MAXLEN 3 RANK -1] == {7 6}}
+        assert {[r LPOS mylist c COUNT 0 MAXLEN 7 RANK 2] == {6}}
     }
 
     test {LPUSH, RPUSH, LLENGTH, LINDEX, LPOP - ziplist} {

From ad0a9df77a2ccf3fdf309dcdd1b54cf350fcbe3c Mon Sep 17 00:00:00 2001
From: antirez <antirez@gmail.com>
Date: Thu, 25 Jun 2020 12:58:21 +0200
Subject: [PATCH 0456/1098] Update comment to clarify change in #7398.

---
 src/cluster.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/cluster.c b/src/cluster.c
index e15e59fda..e7a32a9a2 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -1264,7 +1264,10 @@ void markNodeAsFailingIfNeeded(clusterNode *node) {
     node->fail_time = mstime();
 
     /* Broadcast the failing node name to everybody, forcing all the other
-     * reachable nodes to flag the node as FAIL. */
+     * reachable nodes to flag the node as FAIL.
+     * We do that even if this node is a replica and not a master: anyway
+     * the failing state is triggered collecting failure reports from masters,
+     * so here the replica is only helping propagating this status. */
     clusterSendFail(node->name);
     clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG);
 }

From 1978f996d8b13db112d5d2fdf4a4ce2baf636729 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <276441700@qq.com>
Date: Fri, 10 Jul 2020 13:20:27 +0800
Subject: [PATCH 0457/1098] BITOP: propagate only when it really SET or DEL
 targetkey (#5783)

For example:
    BITOP not targetkey sourcekey

If targetkey and sourcekey doesn't exist, BITOP has no effect,
we do not propagate it, thus can save aof and replica flow.
---
 src/bitops.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/bitops.c b/src/bitops.c
index b37bea2bf..4b1a09aa4 100644
--- a/src/bitops.c
+++ b/src/bitops.c
@@ -759,11 +759,12 @@ void bitopCommand(client *c) {
         setKey(c,c->db,targetkey,o);
         notifyKeyspaceEvent(NOTIFY_STRING,"set",targetkey,c->db->id);
         decrRefCount(o);
+        server.dirty++;
     } else if (dbDelete(c->db,targetkey)) {
         signalModifiedKey(c,c->db,targetkey);
         notifyKeyspaceEvent(NOTIFY_GENERIC,"del",targetkey,c->db->id);
+        server.dirty++;
     }
-    server.dirty++;
     addReplyLongLong(c,maxlen); /* Return the output string length in bytes. */
 }
 

From 9bbf768d3ceaa882c7dcc0033fc3cb4be0973248 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 10 Jul 2020 08:25:26 +0300
Subject: [PATCH 0458/1098] change references to the github repo location
 (#7479)

---
 BUGS                             | 2 +-
 README.md                        | 6 +++---
 src/debug.c                      | 2 +-
 utils/generate-command-help.rb   | 2 +-
 utils/releasetools/changelog.tcl | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/BUGS b/BUGS
index a8e936892..7af259340 100644
--- a/BUGS
+++ b/BUGS
@@ -1 +1 @@
-Please check https://github.com/antirez/redis/issues
+Please check https://github.com/redis/redis/issues
diff --git a/README.md b/README.md
index c08013416..55537e01f 100644
--- a/README.md
+++ b/README.md
@@ -205,8 +205,8 @@ source distribution.
 Please see the [CONTRIBUTING][2] file in this source distribution for more
 information.
 
-[1]: https://github.com/antirez/redis/blob/unstable/COPYING
-[2]: https://github.com/antirez/redis/blob/unstable/CONTRIBUTING
+[1]: https://github.com/redis/redis/blob/unstable/COPYING
+[2]: https://github.com/redis/redis/blob/unstable/CONTRIBUTING
 
 Redis internals
 ===
@@ -236,7 +236,7 @@ Inside the root are the following important directories:
 
 * `src`: contains the Redis implementation, written in C.
 * `tests`: contains the unit tests, implemented in Tcl.
-* `deps`: contains libraries Redis uses. Everything needed to compile Redis is inside this directory; your system just needs to provide `libc`, a POSIX compatible interface and a C compiler. Notably `deps` contains a copy of `jemalloc`, which is the default allocator of Redis under Linux. Note that under `deps` there are also things which started with the Redis project, but for which the main repository is not `antirez/redis`.
+* `deps`: contains libraries Redis uses. Everything needed to compile Redis is inside this directory; your system just needs to provide `libc`, a POSIX compatible interface and a C compiler. Notably `deps` contains a copy of `jemalloc`, which is the default allocator of Redis under Linux. Note that under `deps` there are also things which started with the Redis project, but for which the main repository is not `redis/redis`.
 
 There are a few more directories but they are not very important for our goals
 here. We'll focus mostly on `src`, where the Redis implementation is contained,
diff --git a/src/debug.c b/src/debug.c
index d79226bf2..ca113bcaa 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1569,7 +1569,7 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
     serverLogRaw(LL_WARNING|LL_RAW,
 "\n=== REDIS BUG REPORT END. Make sure to include from START to END. ===\n\n"
 "       Please report the crash by opening an issue on github:\n\n"
-"           http://github.com/antirez/redis/issues\n\n"
+"           http://github.com/redis/redis/issues\n\n"
 "  Suspect RAM error? Use redis-server --test-memory to verify it.\n\n"
 );
 
diff --git a/utils/generate-command-help.rb b/utils/generate-command-help.rb
index 29acef69d..e57acf4b9 100755
--- a/utils/generate-command-help.rb
+++ b/utils/generate-command-help.rb
@@ -53,7 +53,7 @@ def commands
   require "json"
   require "uri"
 
-  url = URI.parse "https://raw.githubusercontent.com/antirez/redis-doc/master/commands.json"
+  url = URI.parse "https://raw.githubusercontent.com/redis/redis-doc/master/commands.json"
   client = Net::HTTP.new url.host, url.port
   client.use_ssl = true
   response = client.get url.path
diff --git a/utils/releasetools/changelog.tcl b/utils/releasetools/changelog.tcl
index 06e38ba99..2288794bb 100755
--- a/utils/releasetools/changelog.tcl
+++ b/utils/releasetools/changelog.tcl
@@ -30,6 +30,6 @@ append template [exec git log $branch~$count..$branch "--format=format:%an in co
 
 #Older, more verbose version.
 #
-#append template [exec git log $branch~30..$branch "--format=format:+-------------------------------------------------------------------------------%n| %s%n| By %an, %ai%n+--------------------------------------------------------------------------------%nhttps://github.com/antirez/redis/commit/%H%n%n%b" --stat]
+#append template [exec git log $branch~30..$branch "--format=format:+-------------------------------------------------------------------------------%n| %s%n| By %an, %ai%n+--------------------------------------------------------------------------------%nhttps://github.com/redis/redis/commit/%H%n%n%b" --stat]
 
 puts $template

From 69ade87325eedebdb44760af9a8c28e15381888e Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 10 Jul 2020 08:26:52 +0300
Subject: [PATCH 0459/1098] tests/valgrind: don't use debug restart (#7404)

* tests/valgrind: don't use debug restart

DEBUG REATART causes two issues:
1. it uses execve which replaces the original process and valgrind doesn't
   have a chance to check for errors, so leaks go unreported.
2. valgrind report invalid calls to close() which we're unable to resolve.

So now the tests use restart_server mechanism in the tests, that terminates
the old server and starts a new one, new PID, but same stdout, stderr.

since the stderr can contain two or more valgrind report, it is not enough
to just check for the absence of leaks, we also need to check for some known
errors, we do both, and fail if we either find an error, or can't find a
report saying there are no leaks.

other changes:
- when killing a server that was already terminated we check for leaks too.
- adding DEBUG LEAK which was used to test it.
- adding --trace-children to valgrind, although no longer needed.
- since the stdout contains two or more runs, we need slightly different way
  of checking if the new process is up (explicitly looking for the new PID)
- move the code that handles --wait-server to happen earlier (before
  watching the startup message in the log), and serve the restarted server too.

* squashme - CR fixes
---
 src/debug.c                  |   4 +
 tests/integration/psync2.tcl |   6 +-
 tests/integration/rdb.tcl    |  14 +---
 tests/support/server.tcl     | 147 +++++++++++++++++++++++++----------
 4 files changed, 114 insertions(+), 57 deletions(-)

diff --git a/src/debug.c b/src/debug.c
index ca113bcaa..a74c22647 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -378,6 +378,7 @@ void debugCommand(client *c) {
 "DEBUG PROTOCOL [string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false]",
 "ERROR <string> -- Return a Redis protocol error with <string> as message. Useful for clients unit tests to simulate Redis errors.",
 "LOG <message> -- write message to the server log.",
+"LEAK <string> -- Create a memory leak of the input string.",
 "HTSTATS <dbid> -- Return hash table statistics of the specified Redis database.",
 "HTSTATS-KEY <key> -- Like htstats but for the hash table stored as key's value.",
 "LOADAOF -- Flush the AOF buffers on disk and reload the AOF in memory.",
@@ -430,6 +431,9 @@ NULL
     } else if (!strcasecmp(c->argv[1]->ptr,"log") && c->argc == 3) {
         serverLog(LL_WARNING, "DEBUG LOG: %s", (char*)c->argv[2]->ptr);
         addReply(c,shared.ok);
+    } else if (!strcasecmp(c->argv[1]->ptr,"leak") && c->argc == 3) {
+        sdsdup(c->argv[2]->ptr);
+        addReply(c,shared.ok);
     } else if (!strcasecmp(c->argv[1]->ptr,"reload")) {
         int flush = 1, save = 1;
         int flags = RDBFLAGS_NONE;
diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl
index 3f636463a..1b996ffd4 100644
--- a/tests/integration/psync2.tcl
+++ b/tests/integration/psync2.tcl
@@ -280,7 +280,8 @@ start_server {} {
         set sync_partial_err [status $R($master_id) sync_partial_err]
         catch {
             $R($slave_id) config rewrite
-            $R($slave_id) debug restart
+            restart_server [expr {0-$slave_id}] true
+            set R($slave_id) [srv [expr {0-$slave_id}] client]
         }
         # note: just waiting for connected_slaves==4 has a race condition since
         # we might do the check before the master realized that the slave disconnected
@@ -328,7 +329,8 @@ start_server {} {
 
         catch {
             $R($slave_id) config rewrite
-            $R($slave_id) debug restart
+            restart_server [expr {0-$slave_id}] true
+            set R($slave_id) [srv [expr {0-$slave_id}] client]
         }
 
         # Reconfigure the slave correctly again, when it's back online.
diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl
index 123e9c8b6..b176bf199 100644
--- a/tests/integration/rdb.tcl
+++ b/tests/integration/rdb.tcl
@@ -137,18 +137,8 @@ test {client freed during loading} {
         # 100mb of rdb, 100k keys will load in more than 1 second
         r debug populate 100000 key 1000
 
-        catch {
-            r debug restart
-        }
+        restart_server 0 false
 
-        set stdout [srv 0 stdout]
-        while 1 {
-            # check that the new server actually started and is ready for connections
-            if {[exec grep -i "Server initialized" | wc -l < $stdout] > 1} {
-                break
-            }
-            after 10
-        }
         # make sure it's still loading
         assert_equal [s loading] 1
 
@@ -180,4 +170,4 @@ test {client freed during loading} {
         # no need to keep waiting for loading to complete
         exec kill [srv 0 pid]
     }
-}
\ No newline at end of file
+}
diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index 146ebc72c..ea7d0b13c 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -17,7 +17,14 @@ proc check_valgrind_errors stderr {
     set buf [read $fd]
     close $fd
 
+    # look for stack trace and other errors, or the absense of a leak free summary
     if {[regexp -- { at 0x} $buf] ||
+        [regexp -- {Warning} $buf] ||
+        [regexp -- {Invalid} $buf] ||
+        [regexp -- {Mismatched} $buf] ||
+        [regexp -- {uninitialized} $buf] ||
+        [regexp -- {has a fishy} $buf] ||
+        [regexp -- {overlap} $buf] ||
         (![regexp -- {definitely lost: 0 bytes} $buf] &&
          ![regexp -- {no leaks are possible} $buf])} {
         send_data_packet $::test_server_fd err "Valgrind error: $buf\n"
@@ -29,7 +36,13 @@ proc kill_server config {
     if {$::external} return
 
     # nevermind if its already dead
-    if {![is_alive $config]} { return }
+    if {![is_alive $config]} {
+        # Check valgrind errors if needed
+        if {$::valgrind} {
+            check_valgrind_errors [dict get $config stderr]
+        }
+        return
+    }
     set pid [dict get $config pid]
 
     # check for leaks
@@ -153,6 +166,55 @@ proc create_server_config_file {filename config} {
     close $fp
 }
 
+proc spawn_server {config_file stdout stderr} {
+    if {$::valgrind} {
+        set pid [exec valgrind --track-origins=yes --trace-children=yes --suppressions=[pwd]/src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file >> $stdout 2>> $stderr &]
+    } elseif ($::stack_logging) {
+        set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/redis-server $config_file >> $stdout 2>> $stderr &]
+    } else {
+        set pid [exec src/redis-server $config_file >> $stdout 2>> $stderr &]
+    }
+
+    if {$::wait_server} {
+        set msg "server started PID: $pid. press any key to continue..."
+        puts $msg
+        read stdin 1
+    }
+
+    # Tell the test server about this new instance.
+    send_data_packet $::test_server_fd server-spawned $pid
+    return $pid
+}
+
+# Wait for actual startup, return 1 if port is busy, 0 otherwise
+proc wait_server_started {config_file stdout pid} {
+    set checkperiod 100; # Milliseconds
+    set maxiter [expr {120*1000/$checkperiod}] ; # Wait up to 2 minutes.
+    set port_busy 0
+    while 1 {
+        if {[regexp -- " PID: $pid" [exec cat $stdout]]} {
+            break
+        }
+        after $checkperiod
+        incr maxiter -1
+        if {$maxiter == 0} {
+            start_server_error $config_file "No PID detected in log $stdout"
+            puts "--- LOG CONTENT ---"
+            puts [exec cat $stdout]
+            puts "-------------------"
+            break
+        }
+
+        # Check if the port is actually busy and the server failed
+        # for this reason.
+        if {[regexp {Could not create server TCP} [exec cat $stdout]]} {
+            set port_busy 1
+            break
+        }
+    }
+    return $port_busy
+}
+
 proc start_server {options {code undefined}} {
     # If we are running against an external server, we just push the
     # host/port pair in the stack the first time
@@ -248,44 +310,10 @@ proc start_server {options {code undefined}} {
 
         send_data_packet $::test_server_fd "server-spawning" "port $port"
 
-        if {$::valgrind} {
-            set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &]
-        } elseif ($::stack_logging) {
-            set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/redis-server $config_file > $stdout 2> $stderr &]
-        } else {
-            set pid [exec src/redis-server $config_file > $stdout 2> $stderr &]
-        }
-
-        # Tell the test server about this new instance.
-        send_data_packet $::test_server_fd server-spawned $pid
+        set pid [spawn_server $config_file $stdout $stderr]
 
         # check that the server actually started
-        # ugly but tries to be as fast as possible...
-        if {$::valgrind} {set retrynum 1000} else {set retrynum 100}
-
-        # Wait for actual startup
-        set checkperiod 100; # Milliseconds
-        set maxiter [expr {120*1000/100}] ; # Wait up to 2 minutes.
-        set port_busy 0
-        while {![info exists _pid]} {
-            regexp {PID:\s(\d+)} [exec cat $stdout] _ _pid
-            after $checkperiod
-            incr maxiter -1
-            if {$maxiter == 0} {
-                start_server_error $config_file "No PID detected in log $stdout"
-                puts "--- LOG CONTENT ---"
-                puts [exec cat $stdout]
-                puts "-------------------"
-                break
-            }
-
-            # Check if the port is actually busy and the server failed
-            # for this reason.
-            if {[regexp {Could not create server TCP} [exec cat $stdout]]} {
-                set port_busy 1
-                break
-            }
-        }
+        set port_busy [wait_server_started $config_file $stdout $pid]
 
         # Sometimes we have to try a different port, even if we checked
         # for availability. Other test clients may grab the port before we
@@ -302,6 +330,7 @@ proc start_server {options {code undefined}} {
             continue; # Try again
         }
 
+        if {$::valgrind} {set retrynum 1000} else {set retrynum 100}
         if {$code ne "undefined"} {
             set serverisup [server_is_up $::host $port $retrynum]
         } else {
@@ -345,12 +374,6 @@ proc start_server {options {code undefined}} {
             error_and_quit $config_file $line
         }
 
-        if {$::wait_server} {
-            set msg "server started PID: [dict get $srv "pid"]. press any key to continue..."
-            puts $msg
-            read stdin 1
-        }
-
         while 1 {
             # check that the server actually started and is ready for connections
             if {[exec grep -i "Ready to accept" | wc -l < $stdout] > 0} {
@@ -370,6 +393,9 @@ proc start_server {options {code undefined}} {
         if {[catch { uplevel 1 $code } error]} {
             set backtrace $::errorInfo
 
+            # fetch srv back from the server list, in case it was restarted by restart_server (new PID)
+            set srv [lindex $::servers end]
+
             # Kill the server without checking for leaks
             dict set srv "skipleaks" 1
             kill_server $srv
@@ -387,6 +413,9 @@ proc start_server {options {code undefined}} {
             error $error $backtrace
         }
 
+        # fetch srv back from the server list, in case it was restarted by restart_server (new PID)
+        set srv [lindex $::servers end]
+
         # Don't do the leak check when no tests were run
         if {$num_tests == $::num_tests} {
             dict set srv "skipleaks" 1
@@ -402,3 +431,35 @@ proc start_server {options {code undefined}} {
         set _ $srv
     }
 }
+
+proc restart_server {level wait_ready} {
+    set srv [lindex $::servers end+$level]
+    kill_server $srv
+
+    set stdout [dict get $srv "stdout"]
+    set stderr [dict get $srv "stderr"]
+    set config_file [dict get $srv "config_file"]
+
+    set prev_ready_count [exec grep -i "Ready to accept" | wc -l < $stdout]
+
+    set pid [spawn_server $config_file $stdout $stderr]
+
+    # check that the server actually started
+    wait_server_started $config_file $stdout $pid
+
+    # update the pid in the servers list
+    dict set srv "pid" $pid
+    # re-set $srv in the servers list
+    lset ::servers end+$level $srv
+
+    if {$wait_ready} {
+        while 1 {
+            # check that the server actually started and is ready for connections
+            if {[exec grep -i "Ready to accept" | wc -l < $stdout] > $prev_ready_count + 1} {
+                break
+            }
+            after 10
+        }
+    }
+    reconnect $level
+}

From 8e76e13472b7d277af78691775c2cf845f68ab90 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 10 Jul 2020 08:28:22 +0300
Subject: [PATCH 0460/1098] stabilize tests that look for log lines (#7367)

tests were sensitive to additional log lines appearing in the log
causing the search to come empty handed.

instead of just looking for the n last log lines, capture the log lines
before performing the action, and then search from that offset.
---
 tests/integration/replication.tcl | 16 ++++++++++------
 tests/support/util.tcl            | 22 +++++++++++++++++++---
 tests/unit/moduleapi/testrdb.tcl  |  6 ++++--
 3 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl
index 7c03c4bc6..d47ec4fe4 100644
--- a/tests/integration/replication.tcl
+++ b/tests/integration/replication.tcl
@@ -430,6 +430,7 @@ test {diskless loading short read} {
             }
 
             # Start the replication process...
+            set loglines [count_log_lines -1]
             $master config set repl-diskless-sync-delay 0
             $replica replicaof $master_host $master_port
 
@@ -439,7 +440,7 @@ test {diskless loading short read} {
             for {set i 0} {$i < $attempts} {incr i} {
                 # wait for the replica to start reading the rdb
                 # using the log file since the replica only responds to INFO once in 2mb
-                wait_for_log_message -1 "*Loading DB in memory*" 5 2000 1
+                wait_for_log_message -1 "*Loading DB in memory*" $loglines 2000 1
 
                 # add some additional random sleep so that we kill the master on a different place each time
                 after [expr {int(rand()*100)}]
@@ -448,7 +449,7 @@ test {diskless loading short read} {
                 set killed [$master client kill type replica]
 
                 if {[catch {
-                    set res [wait_for_log_message -1 "*Internal error in RDB*" 5 100 10]
+                    set res [wait_for_log_message -1 "*Internal error in RDB*" $loglines 100 10]
                     if {$::verbose} {
                         puts $res
                     }
@@ -461,6 +462,7 @@ test {diskless loading short read} {
                     $master config set repl-backlog-size [expr {16384 + $i}]
                 }
                 # wait for loading to stop (fail)
+                set loglines [count_log_lines -1]
                 wait_for_condition 100 10 {
                     [s -1 loading] eq 0
                 } else {
@@ -535,6 +537,7 @@ start_server {tags {"repl"}} {
                     # start replication
                     # it's enough for just one replica to be slow, and have it's write handler enabled
                     # so that the whole rdb generation process is bound to that
+                    set loglines [count_log_lines -1]
                     [lindex $replicas 0] config set repl-diskless-load swapdb
                     [lindex $replicas 0] config set key-load-delay 100
                     [lindex $replicas 0] replicaof $master_host $master_port
@@ -542,7 +545,7 @@ start_server {tags {"repl"}} {
 
                     # wait for the replicas to start reading the rdb
                     # using the log file since the replica only responds to INFO once in 2mb
-                    wait_for_log_message -1 "*Loading DB in memory*" 8 800 10
+                    wait_for_log_message -1 "*Loading DB in memory*" $loglines 800 10
 
                     if {$measure_time} {
                         set master_statfile "/proc/$master_pid/stat"
@@ -558,6 +561,7 @@ start_server {tags {"repl"}} {
                     $master incr $all_drop
 
                     # disconnect replicas depending on the current test
+                    set loglines [count_log_lines -2]
                     if {$all_drop == "all" || $all_drop == "fast"} {
                         exec kill [srv 0 pid]
                         set replicas_alive [lreplace $replicas_alive 1 1]
@@ -576,13 +580,13 @@ start_server {tags {"repl"}} {
 
                     # make sure we got what we were aiming for, by looking for the message in the log file
                     if {$all_drop == "all"} {
-                        wait_for_log_message -2 "*Diskless rdb transfer, last replica dropped, killing fork child*" 12 1 1
+                        wait_for_log_message -2 "*Diskless rdb transfer, last replica dropped, killing fork child*" $loglines 1 1
                     }
                     if {$all_drop == "no"} {
-                        wait_for_log_message -2 "*Diskless rdb transfer, done reading from pipe, 2 replicas still up*" 12 1 1
+                        wait_for_log_message -2 "*Diskless rdb transfer, done reading from pipe, 2 replicas still up*" $loglines 1 1
                     }
                     if {$all_drop == "slow" || $all_drop == "fast"} {
-                        wait_for_log_message -2 "*Diskless rdb transfer, done reading from pipe, 1 replicas still up*" 12 1 1
+                        wait_for_log_message -2 "*Diskless rdb transfer, done reading from pipe, 1 replicas still up*" $loglines 1 1
                     }
 
                     # make sure we don't have a busy loop going thought epoll_wait
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index 8bec95374..fce3ffd18 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -99,11 +99,27 @@ proc wait_for_ofs_sync {r1 r2} {
     }
 }
 
-proc wait_for_log_message {srv_idx pattern last_lines maxtries delay} {
+# count current log lines in server's stdout
+proc count_log_lines {srv_idx} {
+    set _ [exec wc -l < [srv $srv_idx stdout]]
+}
+
+# verify pattern exists in server's sdtout after a certain line number
+proc verify_log_message {srv_idx pattern from_line} {
+    set lines_after [count_log_lines]
+    set lines [expr $lines_after - $from_line]
+    set result [exec tail -$lines < [srv $srv_idx stdout]]
+    if {![string match $pattern $result]} {
+        error "assertion:expected message not found in log file: $pattern"
+    }
+}
+
+# wait for pattern to be found in server's stdout after certain line number
+proc wait_for_log_message {srv_idx pattern from_line maxtries delay} {
     set retry $maxtries
     set stdout [srv $srv_idx stdout]
     while {$retry} {
-        set result [exec tail -$last_lines < $stdout]
+        set result [exec tail +$from_line < $stdout]
         set result [split $result "\n"]
         foreach line $result {
             if {[string match $pattern $line]} {
@@ -114,7 +130,7 @@ proc wait_for_log_message {srv_idx pattern last_lines maxtries delay} {
         after $delay
     }
     if {$retry == 0} {
-        fail "log message of '$pattern' not found"
+        fail "log message of '$pattern' not found in $stdout after line: $from_line"
     }
 }
 
diff --git a/tests/unit/moduleapi/testrdb.tcl b/tests/unit/moduleapi/testrdb.tcl
index a93b34b69..98641ae0a 100644
--- a/tests/unit/moduleapi/testrdb.tcl
+++ b/tests/unit/moduleapi/testrdb.tcl
@@ -67,6 +67,7 @@ tags "modules" {
                     }
 
                     # Start the replication process...
+                    set loglines [count_log_lines -1]
                     $master config set repl-diskless-sync-delay 0
                     $replica replicaof $master_host $master_port
 
@@ -76,7 +77,7 @@ tags "modules" {
                     for {set i 0} {$i < $attempts} {incr i} {
                         # wait for the replica to start reading the rdb
                         # using the log file since the replica only responds to INFO once in 2mb
-                        wait_for_log_message -1 "*Loading DB in memory*" 5 2000 1
+                        wait_for_log_message -1 "*Loading DB in memory*" $loglines 2000 1
 
                         # add some additional random sleep so that we kill the master on a different place each time
                         after [expr {int(rand()*100)}]
@@ -85,7 +86,7 @@ tags "modules" {
                         set killed [$master client kill type replica]
 
                         if {[catch {
-                            set res [wait_for_log_message -1 "*Internal error in RDB*" 5 100 10]
+                            set res [wait_for_log_message -1 "*Internal error in RDB*" $loglines 100 10]
                             if {$::verbose} {
                                 puts $res
                             }
@@ -98,6 +99,7 @@ tags "modules" {
                             $master config set repl-backlog-size [expr {16384 + $i}]
                         }
                         # wait for loading to stop (fail)
+                        set loglines [count_log_lines -1]
                         wait_for_condition 100 10 {
                             [s -1 loading] eq 0
                         } else {

From 909bc97c526db757a3d022b29911ff6d08eba59c Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 10 Jul 2020 08:29:02 +0300
Subject: [PATCH 0461/1098] skip a test that uses +inf on valgrind (#7440)

On some platforms strtold("+inf") with valgrind returns a non-inf result

[err]: INCRBYFLOAT does not allow NaN or Infinity in tests/unit/type/incr.tcl
Expected 'ERR*would produce*' to equal or match '1189731495357231765085759.....'
---
 tests/unit/type/incr.tcl | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/tests/unit/type/incr.tcl b/tests/unit/type/incr.tcl
index b7a135203..dbf45e455 100644
--- a/tests/unit/type/incr.tcl
+++ b/tests/unit/type/incr.tcl
@@ -130,15 +130,18 @@ start_server {tags {"incr"}} {
         format $err
     } {WRONGTYPE*}
 
-    test {INCRBYFLOAT does not allow NaN or Infinity} {
-        r set foo 0
-        set err {}
-        catch {r incrbyfloat foo +inf} err
-        set err
-        # p.s. no way I can force NaN to test it from the API because
-        # there is no way to increment / decrement by infinity nor to
-        # perform divisions.
-    } {ERR*would produce*}
+    # On some platforms strtold("+inf") with valgrind returns a non-inf result
+    if {!$::valgrind} {
+        test {INCRBYFLOAT does not allow NaN or Infinity} {
+            r set foo 0
+            set err {}
+            catch {r incrbyfloat foo +inf} err
+            set err
+            # p.s. no way I can force NaN to test it from the API because
+            # there is no way to increment / decrement by infinity nor to
+            # perform divisions.
+        } {ERR*would produce*}
+    }
 
     test {INCRBYFLOAT decrement} {
         r set foo 1

From d6180c8c8674ffdae3d6efa5f946d85fe9163464 Mon Sep 17 00:00:00 2001
From: huangzhw <hzweveryday@126.com>
Date: Fri, 10 Jul 2020 13:29:44 +0800
Subject: [PATCH 0462/1098] defrag.c activeDefragSdsListAndDict when defrag
 sdsele, We can't use (#7492)

it to calculate hash, we should use newsds.
---
 src/defrag.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/defrag.c b/src/defrag.c
index 6e5296632..2d8db8ea5 100644
--- a/src/defrag.c
+++ b/src/defrag.c
@@ -348,7 +348,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) {
         sdsele = ln->value;
         if ((newsds = activeDefragSds(sdsele))) {
             /* When defragging an sds value, we need to update the dict key */
-            uint64_t hash = dictGetHash(d, sdsele);
+            uint64_t hash = dictGetHash(d, newsds);
             replaceSateliteDictKeyPtrAndOrDefragDictEntry(d, sdsele, newsds, hash, &defragged);
             ln->value = newsds;
             defragged++;

From 5977a94842a25140520297fe4bfda15e0e4de711 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 10 Jul 2020 10:02:37 +0300
Subject: [PATCH 0463/1098] RESTORE ABSTTL won't store expired keys into the db
 (#7472)

Similarly to EXPIREAT with TTL in the past, which implicitly deletes the
key and return success, RESTORE should not store key that are already
expired into the db.
When used together with REPLACE it should emit a DEL to keyspace
notification and replication stream.
---
 src/cluster.c       | 30 ++++++++++++++++++++++--------
 src/expire.c        | 18 +++++++++++-------
 src/server.h        |  1 +
 tests/unit/dump.tcl | 13 ++++++++++++-
 4 files changed, 46 insertions(+), 16 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index e7a32a9a2..88b810d13 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4988,7 +4988,8 @@ void restoreCommand(client *c) {
     }
 
     /* Make sure this key does not already exist here... */
-    if (!replace && lookupKeyWrite(c->db,c->argv[1]) != NULL) {
+    robj *key = c->argv[1];
+    if (!replace && lookupKeyWrite(c->db,key) != NULL) {
         addReply(c,shared.busykeyerr);
         return;
     }
@@ -5010,24 +5011,37 @@ void restoreCommand(client *c) {
 
     rioInitWithBuffer(&payload,c->argv[3]->ptr);
     if (((type = rdbLoadObjectType(&payload)) == -1) ||
-        ((obj = rdbLoadObject(type,&payload,c->argv[1]->ptr)) == NULL))
+        ((obj = rdbLoadObject(type,&payload,key->ptr)) == NULL))
     {
         addReplyError(c,"Bad data format");
         return;
     }
 
     /* Remove the old key if needed. */
-    if (replace) dbDelete(c->db,c->argv[1]);
+    int deleted = 0;
+    if (replace)
+        deleted = dbDelete(c->db,key);
+
+    if (ttl && !absttl) ttl+=mstime();
+    if (ttl && checkAlreadyExpired(ttl)) {
+        if (deleted) {
+            rewriteClientCommandVector(c,2,shared.del,key);
+            signalModifiedKey(c,c->db,key);
+            notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
+            server.dirty++;
+        }
+        addReply(c, shared.ok);
+        return;
+    }
 
     /* Create the key and set the TTL if any */
-    dbAdd(c->db,c->argv[1],obj);
+    dbAdd(c->db,key,obj);
     if (ttl) {
-        if (!absttl) ttl+=mstime();
-        setExpire(c,c->db,c->argv[1],ttl);
+        setExpire(c,c->db,key,ttl);
     }
     objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock,1000);
-    signalModifiedKey(c,c->db,c->argv[1]);
-    notifyKeyspaceEvent(NOTIFY_GENERIC,"restore",c->argv[1],c->db->id);
+    signalModifiedKey(c,c->db,key);
+    notifyKeyspaceEvent(NOTIFY_GENERIC,"restore",key,c->db->id);
     addReply(c,shared.ok);
     server.dirty++;
 }
diff --git a/src/expire.c b/src/expire.c
index 30a27193d..f2d135e2b 100644
--- a/src/expire.c
+++ b/src/expire.c
@@ -475,6 +475,16 @@ void flushSlaveKeysWithExpireList(void) {
     }
 }
 
+int checkAlreadyExpired(long long when) {
+    /* EXPIRE with negative TTL, or EXPIREAT with a timestamp into the past
+     * should never be executed as a DEL when load the AOF or in the context
+     * of a slave instance.
+     *
+     * Instead we add the already expired key to the database with expire time
+     * (possibly in the past) and wait for an explicit DEL from the master. */
+    return (when <= mstime() && !server.loading && !server.masterhost);
+}
+
 /*-----------------------------------------------------------------------------
  * Expires Commands
  *----------------------------------------------------------------------------*/
@@ -502,13 +512,7 @@ void expireGenericCommand(client *c, long long basetime, int unit) {
         return;
     }
 
-    /* EXPIRE with negative TTL, or EXPIREAT with a timestamp into the past
-     * should never be executed as a DEL when load the AOF or in the context
-     * of a slave instance.
-     *
-     * Instead we take the other branch of the IF statement setting an expire
-     * (possibly in the past) and wait for an explicit DEL from the master. */
-    if (when <= mstime() && !server.loading && !server.masterhost) {
+    if (checkAlreadyExpired(when)) {
         robj *aux;
 
         int deleted = server.lazyfree_lazy_expire ? dbAsyncDelete(c->db,key) :
diff --git a/src/server.h b/src/server.h
index 6c36385e1..8c0facd04 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2070,6 +2070,7 @@ void propagateExpire(redisDb *db, robj *key, int lazy);
 int expireIfNeeded(redisDb *db, robj *key);
 long long getExpire(redisDb *db, robj *key);
 void setExpire(client *c, redisDb *db, robj *key, long long when);
+int checkAlreadyExpired(long long when);
 robj *lookupKey(redisDb *db, robj *key, int flags);
 robj *lookupKeyRead(redisDb *db, robj *key);
 robj *lookupKeyWrite(redisDb *db, robj *key);
diff --git a/tests/unit/dump.tcl b/tests/unit/dump.tcl
index 062d803b5..a9def9206 100644
--- a/tests/unit/dump.tcl
+++ b/tests/unit/dump.tcl
@@ -36,7 +36,18 @@ start_server {tags {"dump"}} {
         assert {$ttl >= 2900 && $ttl <= 3100}
         r get foo
     } {bar}
-    
+
+    test {RESTORE with ABSTTL in the past} {
+        r set foo bar
+        set encoded [r dump foo]
+        set now [clock milliseconds]
+        r debug set-active-expire 0
+        r restore foo [expr $now-3000] $encoded absttl REPLACE
+        catch {r debug object foo} e
+        r debug set-active-expire 1
+        set e
+    } {ERR no such key}
+
     test {RESTORE can set LRU} {
         r set foo bar
         set encoded [r dump foo]

From 6f8a8647de5b16ff937dc25046151619a2eafabb Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 5 Jul 2020 12:36:25 +0300
Subject: [PATCH 0464/1098] redis-cli --bigkeys fixed to handle non-printable
 key names

---
 src/redis-cli.c | 40 ++++++++++++++++++++++++----------------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 75845f346..e1f40373a 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -7246,7 +7246,9 @@ static void getKeyTypes(dict *types_dict, redisReply *keys, typeinfo **types) {
 
     /* Pipeline TYPE commands */
     for(i=0;i<keys->elements;i++) {
-        redisAppendCommand(context, "TYPE %s", keys->element[i]->str);
+        const char* argv[] = {"TYPE", keys->element[i]->str};
+        size_t lens[] = {4, keys->element[i]->len};
+        redisAppendCommandArgv(context, 2, argv, lens);
     }
 
     /* Retrieve types */
@@ -7292,15 +7294,21 @@ static void getKeySizes(redisReply *keys, typeinfo **types,
         if(!types[i] || (!types[i]->sizecmd && !memkeys))
             continue;
 
-        if (!memkeys)
-            redisAppendCommand(context, "%s %s",
-                types[i]->sizecmd, keys->element[i]->str);
-        else if (memkeys_samples==0)
-            redisAppendCommand(context, "%s %s %s",
-                "MEMORY", "USAGE", keys->element[i]->str);
-        else
-            redisAppendCommand(context, "%s %s %s SAMPLES %u",
-                "MEMORY", "USAGE", keys->element[i]->str, memkeys_samples);
+        if (!memkeys) {
+            const char* argv[] = {types[i]->sizecmd, keys->element[i]->str};
+            size_t lens[] = {strlen(types[i]->sizecmd), keys->element[i]->len};
+            redisAppendCommandArgv(context, 2, argv, lens);
+        } else if (memkeys_samples==0) {
+            const char* argv[] = {"MEMORY", "USAGE", keys->element[i]->str};
+            size_t lens[] = {6, 5, keys->element[i]->len};
+            redisAppendCommandArgv(context, 3, argv, lens);
+        } else {
+            sds samplesstr = sdsfromlonglong(memkeys_samples);
+            const char* argv[] = {"MEMORY", "USAGE", keys->element[i]->str, "SAMPLES", samplesstr};
+            size_t lens[] = {6, 5, keys->element[i]->len, 7, sdslen(samplesstr)};
+            redisAppendCommandArgv(context, 5, argv, lens);
+            sdsfree(samplesstr);
+        }
     }
 
     /* Retrieve sizes */
@@ -7396,20 +7404,20 @@ static void findBigKeys(int memkeys, unsigned memkeys_samples) {
             sampled++;
 
             if(type->biggest<sizes[i]) {
-                printf(
-                   "[%05.2f%%] Biggest %-6s found so far '%s' with %llu %s\n",
-                   pct, type->name, keys->element[i]->str, sizes[i],
-                   !memkeys? type->sizeunit: "bytes");
-
                 /* Keep track of biggest key name for this type */
                 if (type->biggest_key)
                     sdsfree(type->biggest_key);
-                type->biggest_key = sdsnew(keys->element[i]->str);
+                type->biggest_key = sdscatrepr(sdsempty(), keys->element[i]->str, keys->element[i]->len);
                 if(!type->biggest_key) {
                     fprintf(stderr, "Failed to allocate memory for key!\n");
                     exit(1);
                 }
 
+                printf(
+                   "[%05.2f%%] Biggest %-6s found so far '%s' with %llu %s\n",
+                   pct, type->name, type->biggest_key, sizes[i],
+                   !memkeys? type->sizeunit: "bytes");
+
                 /* Keep track of the biggest size for this type */
                 type->biggest = sizes[i];
             }

From b23e2510364bd13d9988921235c400c5657a8438 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 7 Jul 2020 16:15:44 +0300
Subject: [PATCH 0465/1098] redis-cli --hotkeys fixed to handle non-printable
 key names

---
 src/redis-cli.c | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index e1f40373a..c5ba48447 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -7481,21 +7481,27 @@ static void getKeyFreqs(redisReply *keys, unsigned long long *freqs) {
 
     /* Pipeline OBJECT freq commands */
     for(i=0;i<keys->elements;i++) {
-        redisAppendCommand(context, "OBJECT freq %s", keys->element[i]->str);
+        const char* argv[] = {"OBJECT", "FREQ", keys->element[i]->str};
+        size_t lens[] = {6, 4, keys->element[i]->len};
+        redisAppendCommandArgv(context, 3, argv, lens);
     }
 
     /* Retrieve freqs */
     for(i=0;i<keys->elements;i++) {
         if(redisGetReply(context, (void**)&reply)!=REDIS_OK) {
+            sds keyname = sdscatrepr(sdsempty(), keys->element[i]->str, keys->element[i]->len);
             fprintf(stderr, "Error getting freq for key '%s' (%d: %s)\n",
-                keys->element[i]->str, context->err, context->errstr);
+                keyname, context->err, context->errstr);
+            sdsfree(keyname);
             exit(1);
         } else if(reply->type != REDIS_REPLY_INTEGER) {
             if(reply->type == REDIS_REPLY_ERROR) {
                 fprintf(stderr, "Error: %s\n", reply->str);
                 exit(1);
             } else {
-                fprintf(stderr, "Warning: OBJECT freq on '%s' failed (may have been deleted)\n", keys->element[i]->str);
+                sds keyname = sdscatrepr(sdsempty(), keys->element[i]->str, keys->element[i]->len);
+                fprintf(stderr, "Warning: OBJECT freq on '%s' failed (may have been deleted)\n", keyname);
+                sdsfree(keyname);
                 freqs[i] = 0;
             }
         } else {
@@ -7566,10 +7572,10 @@ static void findHotKeys(void) {
                 memmove(hotkeys,hotkeys+1,sizeof(hotkeys[0])*k);
             }
             counters[k] = freqs[i];
-            hotkeys[k] = sdsnew(keys->element[i]->str);
+            hotkeys[k] = sdscatrepr(sdsempty(), keys->element[i]->str, keys->element[i]->len);
             printf(
                "[%05.2f%%] Hot key '%s' found so far with counter %llu\n",
-               pct, keys->element[i]->str, freqs[i]);
+               pct, hotkeys[k], freqs[i]);
         }
 
         /* Sleep if we've been directed to do so */

From d9f970d8d3f0b694f1e8915cab6d4eab9cfb2ef1 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@users.noreply.github.com>
Date: Fri, 10 Jul 2020 10:25:55 +0300
Subject: [PATCH 0466/1098] TLS: Add missing redis-cli options. (#7456)

* Tests: fix and reintroduce redis-cli tests.

These tests have been broken and disabled for 10 years now!

* TLS: add remaining redis-cli support.

This adds support for the redis-cli --pipe, --rdb and --replica options
previously unsupported in --tls mode.

* Fix writeConn().
---
 src/redis-cli.c                 | 102 +++++++++++++++++++---------
 tests/integration/redis-cli.tcl | 115 ++++++++++++++++++++++++++------
 tests/test_helper.tcl           |   1 +
 3 files changed, 166 insertions(+), 52 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index c5ba48447..0148964bf 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -1989,6 +1989,7 @@ static void repl(void) {
 
             if (argv == NULL) {
                 printf("Invalid argument(s)\n");
+                fflush(stdout);
                 linenoiseFree(line);
                 continue;
             } else if (argc > 0) {
@@ -6784,10 +6785,53 @@ void sendCapa() {
     sendReplconf("capa", "eof");
 }
 
+/* Wrapper around hiredis to allow arbitrary reads and writes.
+ *
+ * We piggybacks on top of hiredis to achieve transparent TLS support,
+ * and use its internal buffers so it can co-exist with commands
+ * previously/later issued on the connection.
+ *
+ * Interface is close to enough to read()/write() so things should mostly
+ * work transparently.
+ */
+
+/* Write a raw buffer through a redisContext. If we already have something
+ * in the buffer (leftovers from hiredis operations) it will be written
+ * as well.
+ */
+static ssize_t writeConn(redisContext *c, const char *buf, size_t buf_len)
+{
+    int done = 0;
+
+    c->obuf = sdscatlen(c->obuf, buf, buf_len);
+    if (redisBufferWrite(c, &done) == REDIS_ERR) {
+        sdsrange(c->obuf, 0, -(buf_len+1));
+        if (!(c->flags & REDIS_BLOCK))
+            errno = EAGAIN;
+        return -1;
+    }
+
+    size_t left = sdslen(c->obuf);
+    sdsclear(c->obuf);
+    if (!done) {
+        return buf_len - left;
+    }
+
+    return buf_len;
+}
+
+/* Read raw bytes through a redisContext. The read operation is not greedy
+ * and may not fill the buffer entirely.
+ */
+static ssize_t readConn(redisContext *c, char *buf, size_t len)
+{
+    return c->funcs->read(c, buf, len);
+}
+
 /* Sends SYNC and reads the number of bytes in the payload. Used both by
  * slaveMode() and getRDB().
  * returns 0 in case an EOF marker is used. */
-unsigned long long sendSync(int fd, char *out_eof) {
+unsigned long long sendSync(redisContext *c, char *out_eof) {
     /* To start we need to send the SYNC command and return the payload.
      * The hiredis client lib does not understand this part of the protocol
      * and we don't want to mess with its buffers, so everything is performed
@@ -6796,7 +6840,7 @@ unsigned long long sendSync(int fd, char *out_eof) {
     ssize_t nread;
 
     /* Send the SYNC command. */
-    if (write(fd,"SYNC\r\n",6) != 6) {
+    if (writeConn(c, "SYNC\r\n", 6) != 6) {
         fprintf(stderr,"Error writing to master\n");
         exit(1);
     }
@@ -6804,7 +6848,7 @@ unsigned long long sendSync(int fd, char *out_eof) {
     /* Read $<payload>\r\n, making sure to read just up to "\n" */
     p = buf;
     while(1) {
-        nread = read(fd,p,1);
+        nread = readConn(c,p,1);
         if (nread <= 0) {
             fprintf(stderr,"Error reading bulk length while SYNCing\n");
             exit(1);
@@ -6825,11 +6869,10 @@ unsigned long long sendSync(int fd, char *out_eof) {
 }
 
 static void slaveMode(void) {
-    int fd = context->fd;
     static char eofmark[RDB_EOF_MARK_SIZE];
     static char lastbytes[RDB_EOF_MARK_SIZE];
     static int usemark = 0;
-    unsigned long long payload = sendSync(fd, eofmark);
+    unsigned long long payload = sendSync(context,eofmark);
     char buf[1024];
     int original_output = config.output;
 
@@ -6849,7 +6892,7 @@ static void slaveMode(void) {
     while(payload) {
         ssize_t nread;
 
-        nread = read(fd,buf,(payload > sizeof(buf)) ? sizeof(buf) : payload);
+        nread = readConn(context,buf,(payload > sizeof(buf)) ? sizeof(buf) : payload);
         if (nread <= 0) {
             fprintf(stderr,"Error reading RDB payload while SYNCing\n");
             exit(1);
@@ -6892,14 +6935,15 @@ static void slaveMode(void) {
 /* This function implements --rdb, so it uses the replication protocol in order
  * to fetch the RDB file from a remote server. */
 static void getRDB(clusterManagerNode *node) {
-    int s, fd;
+    int fd;
+    redisContext *s;
     char *filename;
     if (node != NULL) {
         assert(node->context);
-        s = node->context->fd;
+        s = node->context;
         filename = clusterManagerGetNodeRDBFilename(node);
     } else {
-        s = context->fd;
+        s = context;
         filename = config.rdb_filename;
     }
     static char eofmark[RDB_EOF_MARK_SIZE];
@@ -6934,7 +6978,7 @@ static void getRDB(clusterManagerNode *node) {
     while(payload) {
         ssize_t nread, nwritten;
 
-        nread = read(s,buf,(payload > sizeof(buf)) ? sizeof(buf) : payload);
+        nread = readConn(s,buf,(payload > sizeof(buf)) ? sizeof(buf) : payload);
         if (nread <= 0) {
             fprintf(stderr,"I/O Error reading RDB payload from socket\n");
             exit(1);
@@ -6968,7 +7012,7 @@ static void getRDB(clusterManagerNode *node) {
     } else {
         fprintf(stderr,"Transfer finished with success.\n");
     }
-    close(s); /* Close the file descriptor ASAP as fsync() may take time. */
+    redisFree(s); /* Close the file descriptor ASAP as fsync() may take time. */
     fsync(fd);
     close(fd);
     fprintf(stderr,"Transfer finished with success.\n");
@@ -6985,11 +7029,9 @@ static void getRDB(clusterManagerNode *node) {
 
 #define PIPEMODE_WRITE_LOOP_MAX_BYTES (128*1024)
 static void pipeMode(void) {
-    int fd = context->fd;
     long long errors = 0, replies = 0, obuf_len = 0, obuf_pos = 0;
-    char ibuf[1024*16], obuf[1024*16]; /* Input and output buffers */
+    char obuf[1024*16]; /* Output buffer */
     char aneterr[ANET_ERR_LEN];
-    redisReader *reader = redisReaderCreate();
     redisReply *reply;
     int eof = 0; /* True once we consumed all the standard input. */
     int done = 0;
@@ -6999,47 +7041,38 @@ static void pipeMode(void) {
     srand(time(NULL));
 
     /* Use non blocking I/O. */
-    if (anetNonBlock(aneterr,fd) == ANET_ERR) {
+    if (anetNonBlock(aneterr,context->fd) == ANET_ERR) {
         fprintf(stderr, "Can't set the socket in non blocking mode: %s\n",
             aneterr);
         exit(1);
     }
 
+    context->flags &= ~REDIS_BLOCK;
+
     /* Transfer raw protocol and read replies from the server at the same
      * time. */
     while(!done) {
         int mask = AE_READABLE;
 
         if (!eof || obuf_len != 0) mask |= AE_WRITABLE;
-        mask = aeWait(fd,mask,1000);
+        mask = aeWait(context->fd,mask,1000);
 
         /* Handle the readable state: we can read replies from the server. */
         if (mask & AE_READABLE) {
-            ssize_t nread;
             int read_error = 0;
 
-            /* Read from socket and feed the hiredis reader. */
             do {
-                nread = read(fd,ibuf,sizeof(ibuf));
-                if (nread == -1 && errno != EAGAIN && errno != EINTR) {
-                    fprintf(stderr, "Error reading from the server: %s\n",
-                        strerror(errno));
+                if (!read_error && redisBufferRead(context) == REDIS_ERR) {
                     read_error = 1;
-                    break;
                 }
-                if (nread > 0) {
-                    redisReaderFeed(reader,ibuf,nread);
-                    last_read_time = time(NULL);
-                }
-            } while(nread > 0);
 
-            /* Consume replies. */
-            do {
-                if (redisReaderGetReply(reader,(void**)&reply) == REDIS_ERR) {
+                reply = NULL;
+                if (redisGetReply(context, (void **) &reply) == REDIS_ERR) {
                     fprintf(stderr, "Error reading replies from server\n");
                     exit(1);
                 }
                 if (reply) {
+                    last_read_time = time(NULL);
                     if (reply->type == REDIS_REPLY_ERROR) {
                         fprintf(stderr,"%s\n", reply->str);
                         errors++;
@@ -7072,7 +7105,7 @@ static void pipeMode(void) {
             while(1) {
                 /* Transfer current buffer to server. */
                 if (obuf_len != 0) {
-                    ssize_t nwritten = write(fd,obuf+obuf_pos,obuf_len);
+                    ssize_t nwritten = writeConn(context,obuf+obuf_pos,obuf_len);
 
                     if (nwritten == -1) {
                         if (errno != EAGAIN && errno != EINTR) {
@@ -7088,6 +7121,10 @@ static void pipeMode(void) {
                     loop_nwritten += nwritten;
                     if (obuf_len != 0) break; /* Can't accept more data. */
                 }
+                if (context->err) {
+                    fprintf(stderr, "Server I/O Error: %s\n", context->errstr);
+                    exit(1);
+                }
                 /* If buffer is empty, load from stdin. */
                 if (obuf_len == 0 && !eof) {
                     ssize_t nread = read(STDIN_FILENO,obuf,sizeof(obuf));
@@ -7138,7 +7175,6 @@ static void pipeMode(void) {
             break;
         }
     }
-    redisReaderFree(reader);
     printf("errors: %lld, replies: %lld\n", errors, replies);
     if (errors)
         exit(1);
diff --git a/tests/integration/redis-cli.tcl b/tests/integration/redis-cli.tcl
index 5d1635950..016e4915c 100644
--- a/tests/integration/redis-cli.tcl
+++ b/tests/integration/redis-cli.tcl
@@ -1,14 +1,13 @@
 source tests/support/cli.tcl
 
 start_server {tags {"cli"}} {
-    proc open_cli {} {
+    proc open_cli {{opts "-n 9"}} {
         set ::env(TERM) dumb
-        set cmdline [rediscli [srv port] "-n 9"]
+        set cmdline [rediscli [srv port] $opts]
         set fd [open "|$cmdline" "r+"]
         fconfigure $fd -buffering none
         fconfigure $fd -blocking false
         fconfigure $fd -translation binary
-        assert_equal "redis> " [read_cli $fd]
         set _ $fd
     }
 
@@ -32,11 +31,14 @@ start_server {tags {"cli"}} {
     }
 
     # Helpers to run tests in interactive mode
+
+    proc format_output {output} {
+        set _ [string trimright [regsub -all "\r" $output ""] "\n"]
+    }
+
     proc run_command {fd cmd} {
         write_cli $fd $cmd
-        set lines [split [read_cli $fd] "\n"]
-        assert_equal "redis> " [lindex $lines end]
-        join [lrange $lines 0 end-1] "\n"
+        set _ [format_output [read_cli $fd]]
     }
 
     proc test_interactive_cli {name code} {
@@ -58,7 +60,7 @@ start_server {tags {"cli"}} {
 
     proc _run_cli {opts args} {
         set cmd [rediscli [srv port] [list -n 9 {*}$args]]
-        foreach {key value} $args {
+        foreach {key value} $opts {
             if {$key eq "pipe"} {
                 set cmd "sh -c \"$value | $cmd\""
             }
@@ -72,7 +74,7 @@ start_server {tags {"cli"}} {
         fconfigure $fd -translation binary
         set resp [read $fd 1048576]
         close $fd
-        set _ $resp
+        set _ [format_output $resp]
     }
 
     proc run_cli {args} {
@@ -80,11 +82,11 @@ start_server {tags {"cli"}} {
     }
 
     proc run_cli_with_input_pipe {cmd args} {
-        _run_cli [list pipe $cmd] {*}$args
+        _run_cli [list pipe $cmd] -x {*}$args
     }
 
     proc run_cli_with_input_file {path args} {
-        _run_cli [list path $path] {*}$args
+        _run_cli [list path $path] -x {*}$args
     }
 
     proc test_nontty_cli {name code} {
@@ -101,7 +103,7 @@ start_server {tags {"cli"}} {
     test_interactive_cli "INFO response should be printed raw" {
         set lines [split [run_command $fd info] "\n"]
         foreach line $lines {
-            assert [regexp {^[a-z0-9_]+:[a-z0-9_]+} $line]
+            assert [regexp {^$|^#|^[a-z0-9_]+:.+} $line]
         }
     }
 
@@ -121,7 +123,7 @@ start_server {tags {"cli"}} {
     test_interactive_cli "Multi-bulk reply" {
         r rpush list foo
         r rpush list bar
-        assert_equal "1. \"foo\"\n2. \"bar\"" [run_command $fd "lrange list 0 -1"]
+        assert_equal "1) \"foo\"\n2) \"bar\"" [run_command $fd "lrange list 0 -1"]
     }
 
     test_interactive_cli "Parsing quotes" {
@@ -144,35 +146,35 @@ start_server {tags {"cli"}} {
     }
 
     test_tty_cli "Status reply" {
-        assert_equal "OK\n" [run_cli set key bar]
+        assert_equal "OK" [run_cli set key bar]
         assert_equal "bar" [r get key]
     }
 
     test_tty_cli "Integer reply" {
         r del counter
-        assert_equal "(integer) 1\n" [run_cli incr counter]
+        assert_equal "(integer) 1" [run_cli incr counter]
     }
 
     test_tty_cli "Bulk reply" {
         r set key "tab\tnewline\n"
-        assert_equal "\"tab\\tnewline\\n\"\n" [run_cli get key]
+        assert_equal "\"tab\\tnewline\\n\"" [run_cli get key]
     }
 
     test_tty_cli "Multi-bulk reply" {
         r del list
         r rpush list foo
         r rpush list bar
-        assert_equal "1. \"foo\"\n2. \"bar\"\n" [run_cli lrange list 0 -1]
+        assert_equal "1) \"foo\"\n2) \"bar\"" [run_cli lrange list 0 -1]
     }
 
     test_tty_cli "Read last argument from pipe" {
-        assert_equal "OK\n" [run_cli_with_input_pipe "echo foo" set key]
+        assert_equal "OK" [run_cli_with_input_pipe "echo foo" set key]
         assert_equal "foo\n" [r get key]
     }
 
     test_tty_cli "Read last argument from file" {
         set tmpfile [write_tmpfile "from file"]
-        assert_equal "OK\n" [run_cli_with_input_file $tmpfile set key]
+        assert_equal "OK" [run_cli_with_input_file $tmpfile set key]
         assert_equal "from file" [r get key]
     }
 
@@ -188,7 +190,7 @@ start_server {tags {"cli"}} {
 
     test_nontty_cli "Bulk reply" {
         r set key "tab\tnewline\n"
-        assert_equal "tab\tnewline\n" [run_cli get key]
+        assert_equal "tab\tnewline" [run_cli get key]
     }
 
     test_nontty_cli "Multi-bulk reply" {
@@ -208,4 +210,79 @@ start_server {tags {"cli"}} {
         assert_equal "OK" [run_cli_with_input_file $tmpfile set key]
         assert_equal "from file" [r get key]
     }
+
+    proc test_redis_cli_rdb_dump {} {
+        r flushdb
+
+        set dir [lindex [r config get dir] 1]
+
+        assert_equal "OK" [r debug populate 100000 key 1000]
+        catch {run_cli --rdb "$dir/cli.rdb"} output
+        assert_match {*Transfer finished with success*} $output
+
+        file delete "$dir/dump.rdb"
+        file rename "$dir/cli.rdb" "$dir/dump.rdb"
+
+        assert_equal "OK" [r set should-not-exist 1]
+        assert_equal "OK" [r debug reload nosave]
+        assert_equal {} [r get should-not-exist]
+    }
+
+    test_nontty_cli "Dumping an RDB" {
+        # Disk-based master
+        assert_match "OK" [r config set repl-diskless-sync no]
+        test_redis_cli_rdb_dump
+
+        # Disk-less master
+        assert_match "OK" [r config set repl-diskless-sync yes]
+        assert_match "OK" [r config set repl-diskless-sync-delay 0]
+        test_redis_cli_rdb_dump
+    }
+
+    test_nontty_cli "Connecting as a replica" {
+        set fd [open_cli "--replica"]
+        wait_for_condition 50 500 {
+            [string match {*slave0:*state=online*} [r info]]
+        } else {
+            fail "redis-cli --replica did not connect"
+        }
+
+        for {set i 0} {$i < 100} {incr i} {
+           r set test-key test-value-$i
+        }
+        r client kill type slave
+        catch {
+            assert_match {*SET*key-a*} [read_cli $fd]
+        }
+
+        close_cli $fd
+    }
+
+    test_nontty_cli "Piping raw protocol" {
+        set fd [open_cli "--pipe"]
+        fconfigure $fd -blocking true
+
+        # Create a new deferring client and overwrite its fd
+        set client [redis [srv 0 "host"] [srv 0 "port"] 1 0]
+        set ::redis::fd($::redis::id) $fd
+        $client select 9
+
+        r del test-counter
+        for {set i 0} {$i < 10000} {incr i} {
+            $client incr test-counter
+            $client set large-key [string repeat "x" 20000]
+        }
+
+        for {set i 0} {$i < 1000} {incr i} {
+            $client set very-large-key [string repeat "x" 512000]
+        }
+
+        close $fd write
+        set output [read_cli $fd]
+
+        assert_equal {10000} [r get test-counter]
+        assert_match {*All data transferred*errors: 0*replies: 21001*} $output
+
+        close_cli $fd
+    }
 }
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index ef9bf7fdf..1527aa1b5 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -48,6 +48,7 @@ set ::all_tests {
     integration/psync2
     integration/psync2-reg
     integration/psync2-pingoff
+    integration/redis-cli
     unit/pubsub
     unit/slowlog
     unit/scripting

From 6a014af79a9cbafc0be946047f216544f116b598 Mon Sep 17 00:00:00 2001
From: James Hilliard <james.hilliard1@gmail.com>
Date: Fri, 10 Jul 2020 01:30:09 -0600
Subject: [PATCH 0467/1098] Use pkg-config to properly detect libssl and
 libcrypto libraries (#7452)

---
 src/Makefile | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/Makefile b/src/Makefile
index b8c05c32b..80c627c24 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -192,9 +192,21 @@ ifeq ($(MALLOC),jemalloc)
 endif
 
 ifeq ($(BUILD_TLS),yes)
-    FINAL_CFLAGS+=-DUSE_OPENSSL $(OPENSSL_CFLAGS)
-    FINAL_LDFLAGS+=$(OPENSSL_LDFLAGS)
-    FINAL_LIBS += ../deps/hiredis/libhiredis_ssl.a -lssl -lcrypto
+	FINAL_CFLAGS+=-DUSE_OPENSSL $(OPENSSL_CFLAGS)
+	FINAL_LDFLAGS+=$(OPENSSL_LDFLAGS)
+	LIBSSL_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libssl && echo $$?)
+ifeq ($(LIBSSL_PKGCONFIG),0)
+	LIBSSL_LIBS=$(shell $(PKG_CONFIG) --libs libssl)
+else
+	LIBSSL_LIBS=-lssl
+endif
+	LIBCRYPTO_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libcrypto && echo $$?)
+ifeq ($(LIBCRYPTO_PKGCONFIG),0)
+	LIBCRYPTO_LIBS=$(shell $(PKG_CONFIG) --libs libcrypto)
+else
+	LIBCRYPTO_LIBS=-lcrypto
+endif
+	FINAL_LIBS += ../deps/hiredis/libhiredis_ssl.a $(LIBSSL_LIBS) $(LIBCRYPTO_LIBS)
 endif
 
 REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS)

From 5266293a0fdee57fe6bb8a408a2e2ff0c66f0259 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@users.noreply.github.com>
Date: Fri, 10 Jul 2020 10:32:21 +0300
Subject: [PATCH 0468/1098] TLS: Ignore client cert when tls-auth-clients off.
 (#7457)

---
 src/tls.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/tls.c b/src/tls.c
index a62f2284e..4b9948195 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -337,9 +337,7 @@ connection *connCreateAcceptedTLS(int fd, int require_auth) {
     conn->c.state = CONN_STATE_ACCEPTING;
 
     if (!require_auth) {
-        /* We still verify certificates if provided, but don't require them.
-         */
-        SSL_set_verify(conn->ssl, SSL_VERIFY_PEER, NULL);
+        SSL_set_verify(conn->ssl, SSL_VERIFY_NONE, NULL);
     }
 
     SSL_set_fd(conn->ssl, conn->c.fd);

From 3e6f2b1a45176ac3d81b95cb6025f30d7aaa1393 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@users.noreply.github.com>
Date: Fri, 10 Jul 2020 11:33:47 +0300
Subject: [PATCH 0469/1098] TLS: Session caching configuration support. (#7420)

* TLS: Session caching configuration support.
* TLS: Remove redundant config initialization.
---
 TLS.md                       |  2 --
 redis.conf                   | 16 ++++++++++++++++
 src/config.c                 | 11 ++++++++++-
 src/server.h                 |  3 +++
 src/tls.c                    | 12 +++++++++---
 tests/unit/introspection.tcl | 28 ++++++++++++++++++----------
 6 files changed, 56 insertions(+), 16 deletions(-)

diff --git a/TLS.md b/TLS.md
index e480c1e9d..2d020d0ce 100644
--- a/TLS.md
+++ b/TLS.md
@@ -68,8 +68,6 @@ but there are probably other good reasons to improve that part anyway.
 To-Do List
 ----------
 
-- [ ] Add session caching support. Check if/how it's handled by clients to
-  assess how useful/important it is.
 - [ ] redis-benchmark support. The current implementation is a mix of using
   hiredis for parsing and basic networking (establishing connections), but
   directly manipulating sockets for most actions. This will need to be cleaned
diff --git a/redis.conf b/redis.conf
index a51ef007d..8c53f015a 100644
--- a/redis.conf
+++ b/redis.conf
@@ -199,6 +199,22 @@ tcp-keepalive 300
 #
 # tls-prefer-server-ciphers yes
 
+# By default, TLS session caching is enabled to allow faster and less expensive
+# reconnections by clients that support it. Use the following directive to disable
+# caching.
+#
+# tls-session-caching no
+
+# Change the default number of TLS sessions cached. A zero value sets the cache
+# to unlimited size. The default size is 20480.
+#
+# tls-session-cache-size 5000
+
+# Change the default timeout of cached TLS sessions. The default timeout is 300
+# seconds.
+#
+# tls-session-cache-timeout 60
+
 ################################# GENERAL #####################################
 
 # By default Redis does not run as a daemon. Use 'yes' if you need it.
diff --git a/src/config.c b/src/config.c
index 64854592c..acf1b069f 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2071,7 +2071,7 @@ static int updateTlsCfg(char *val, char *prev, char **err) {
     UNUSED(prev);
     UNUSED(err);
     if (tlsConfigure(&server.tls_ctx_config) == C_ERR) {
-        *err = "Unable to configure tls-cert-file. Check server logs.";
+        *err = "Unable to update TLS configuration. Check server logs.";
         return 0;
     }
     return 1;
@@ -2081,6 +2081,12 @@ static int updateTlsCfgBool(int val, int prev, char **err) {
     UNUSED(prev);
     return updateTlsCfg(NULL, NULL, err);
 }
+
+static int updateTlsCfgInt(long long val, long long prev, char **err) {
+    UNUSED(val);
+    UNUSED(prev);
+    return updateTlsCfg(NULL, NULL, err);
+}
 #endif  /* USE_OPENSSL */
 
 standardConfig configs[] = {
@@ -2216,10 +2222,13 @@ standardConfig configs[] = {
 
 #ifdef USE_OPENSSL
     createIntConfig("tls-port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG, NULL, NULL), /* TCP port. */
+    createIntConfig("tls-session-cache-size", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tls_ctx_config.session_cache_size, 20*1024, INTEGER_CONFIG, NULL, updateTlsCfgInt),
+    createIntConfig("tls-session-cache-timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tls_ctx_config.session_cache_timeout, 300, INTEGER_CONFIG, NULL, updateTlsCfgInt),
     createBoolConfig("tls-cluster", NULL, MODIFIABLE_CONFIG, server.tls_cluster, 0, NULL, NULL),
     createBoolConfig("tls-replication", NULL, MODIFIABLE_CONFIG, server.tls_replication, 0, NULL, NULL),
     createBoolConfig("tls-auth-clients", NULL, MODIFIABLE_CONFIG, server.tls_auth_clients, 1, NULL, NULL),
     createBoolConfig("tls-prefer-server-ciphers", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.prefer_server_ciphers, 0, NULL, updateTlsCfgBool),
+    createBoolConfig("tls-session-caching", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.session_caching, 1, NULL, updateTlsCfgBool),
     createStringConfig("tls-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.cert_file, NULL, NULL, updateTlsCfg),
     createStringConfig("tls-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.key_file, NULL, NULL, updateTlsCfg),
     createStringConfig("tls-dh-params-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.dh_params_file, NULL, NULL, updateTlsCfg),
diff --git a/src/server.h b/src/server.h
index 8c0facd04..3f471efcb 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1011,6 +1011,9 @@ typedef struct redisTLSContextConfig {
     char *ciphers;
     char *ciphersuites;
     int prefer_server_ciphers;
+    int session_caching;
+    int session_cache_size;
+    int session_cache_timeout;
 } redisTLSContextConfig;
 
 /*-----------------------------------------------------------------------------
diff --git a/src/tls.c b/src/tls.c
index 4b9948195..8b2bb58e1 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -148,9 +148,6 @@ void tlsInit(void) {
     }
 
     pending_list = listCreate();
-
-    /* Server configuration */
-    server.tls_auth_clients = 1;    /* Secure by default */
 }
 
 /* Attempt to configure/reconfigure TLS. This operation is atomic and will
@@ -184,6 +181,15 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
     SSL_CTX_set_options(ctx, SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS);
 #endif
 
+    if (ctx_config->session_caching) {
+        SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_SERVER);
+        SSL_CTX_sess_set_cache_size(ctx, ctx_config->session_cache_size);
+        SSL_CTX_set_timeout(ctx, ctx_config->session_cache_timeout);
+        SSL_CTX_set_session_id_context(ctx, (void *) "redis", 5);
+    } else {
+        SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_OFF);
+    }
+
     int protocols = parseProtocolsConfig(ctx_config->protocols);
     if (protocols == -1) goto error;
 
diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl
index b60ca0d48..d681e06d5 100644
--- a/tests/unit/introspection.tcl
+++ b/tests/unit/introspection.tcl
@@ -78,17 +78,8 @@ start_server {tags {"introspection"}} {
             syslog-facility
             databases
             port
-            io-threads
             tls-port
-            tls-prefer-server-ciphers
-            tls-cert-file
-            tls-key-file
-            tls-dh-params-file
-            tls-ca-cert-file
-            tls-ca-cert-dir
-            tls-protocols
-            tls-ciphers
-            tls-ciphersuites
+            io-threads
             logfile
             unixsocketperm
             slaveof
@@ -100,6 +91,23 @@ start_server {tags {"introspection"}} {
             bgsave_cpulist
         }
 
+        if {!$::tls} {
+            append skip_configs {
+                tls-prefer-server-ciphers
+                tls-session-cache-timeout
+                tls-session-cache-size
+                tls-session-caching
+                tls-cert-file
+                tls-key-file
+                tls-dh-params-file
+                tls-ca-cert-file
+                tls-ca-cert-dir
+                tls-protocols
+                tls-ciphers
+                tls-ciphersuites
+            }
+        }
+
         set configs {}
         foreach {k v} [r config get *] {
             if {[lsearch $skip_configs $k] != -1} {

From d792db7948c9de9c4ac3b7669fac2dbc9eb7b173 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Fri, 10 Jul 2020 16:41:48 +0800
Subject: [PATCH 0470/1098] Add missing latency-monitor tcl test to
 test_helper.tcl. (#6782)

---
 tests/test_helper.tcl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 1527aa1b5..51c364601 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -35,6 +35,7 @@ set ::all_tests {
     unit/quit
     unit/aofrw
     unit/acl
+    unit/latency-monitor
     integration/block-repl
     integration/replication
     integration/replication-2

From af39d750cd6af99774b606899794bef405c32429 Mon Sep 17 00:00:00 2001
From: Jiayuan Chen <mrpre@163.com>
Date: Fri, 10 Jul 2020 21:02:18 +0800
Subject: [PATCH 0471/1098] Fix typo in deps README (#7500)

---
 deps/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deps/README.md b/deps/README.md
index 685dbb40d..f923c06ad 100644
--- a/deps/README.md
+++ b/deps/README.md
@@ -47,7 +47,7 @@ Hiredis
 Hiredis uses the SDS string library, that must be the same version used inside Redis itself. Hiredis is also very critical for Sentinel. Historically Redis often used forked versions of hiredis in a way or the other. In order to upgrade it is advised to take a lot of care:
 
 1. Check with diff if hiredis API changed and what impact it could have in Redis.
-2. Make sure thet the SDS library inside Hiredis and inside Redis are compatible.
+2. Make sure that the SDS library inside Hiredis and inside Redis are compatible.
 3. After the upgrade, run the Redis Sentinel test.
 4. Check manually that redis-cli and redis-benchmark behave as expecteed, since we have no tests for CLI utilities currently.
 

From d5648d617e1ed5b9cfa575ad412bc9d450b16afd Mon Sep 17 00:00:00 2001
From: Abhishek Soni <abhisheksoni2720@gmail.com>
Date: Fri, 10 Jul 2020 18:35:29 +0530
Subject: [PATCH 0472/1098] fix: typo in CI job name (#7466)

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 439e3f3df..730eaf0dd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -31,7 +31,7 @@ jobs:
     - name: make
       run: make
 
-  biuld-32bit:
+  build-32bit:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1

From 279b4a14643f247365637da13439fcb407072a9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=A9=AC=E6=B0=B8=E6=B3=BD?= <1014057907@qq.com>
Date: Fri, 10 Jul 2020 21:37:11 +0800
Subject: [PATCH 0473/1098] fix benchmark in cluster mode fails to authenticate
 (#7488)

Co-authored-by: Oran Agra <oran@redislabs.com> (styling)
---
 src/redis-benchmark.c | 98 +++++++++++++++++++++++++------------------
 1 file changed, 57 insertions(+), 41 deletions(-)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 38d4ca51b..f47cbe333 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -1,4 +1,4 @@
-/* Redis benchmark utility. 
+/* Redis benchmark utility.
  *
  * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
  * All rights reserved.
@@ -183,6 +183,8 @@ static void *execBenchmarkThread(void *ptr);
 static clusterNode *createClusterNode(char *ip, int port);
 static redisConfig *getRedisConfig(const char *ip, int port,
                                    const char *hostsocket);
+static redisContext *getRedisContext(const char *ip, int port,
+                                     const char *hostsocket);
 static void freeRedisConfig(redisConfig *cfg);
 static int fetchClusterSlotsConfiguration(client c);
 static void updateClusterSlotsConfiguration();
@@ -238,6 +240,52 @@ void _serverAssert(const char *estr, const char *file, int line) {
     *((char*)-1) = 'x';
 }
 
+static redisContext *getRedisContext(const char *ip, int port,
+                                     const char *hostsocket)
+{
+    redisContext *ctx = NULL;
+    redisReply *reply =  NULL;
+    if (hostsocket == NULL)
+        ctx = redisConnect(ip, port);
+    else
+        ctx = redisConnectUnix(hostsocket);
+    if (ctx == NULL || ctx->err) {
+        fprintf(stderr,"Could not connect to Redis at ");
+        char *err = (ctx != NULL ? ctx->errstr : "");
+        if (hostsocket == NULL)
+            fprintf(stderr,"%s:%d: %s\n",ip,port,err);
+        else
+            fprintf(stderr,"%s: %s\n",hostsocket,err);
+        goto cleanup;
+    }
+    if (config.auth == NULL)
+        return ctx;
+    if (config.user == NULL)
+        reply = redisCommand(ctx,"AUTH %s", config.auth);
+    else
+        reply = redisCommand(ctx,"AUTH %s %s", config.user, config.auth);
+    if (reply != NULL) {
+        if (reply->type == REDIS_REPLY_ERROR) {
+            if (hostsocket == NULL)
+                fprintf(stderr, "Node %s:%d replied with error:\n%s\n", ip, port, reply->str);
+            else
+                fprintf(stderr, "Node %s replied with error:\n%s\n", hostsocket, reply->str);
+            goto cleanup;
+        }
+        freeReplyObject(reply);
+        return ctx;
+    }
+    fprintf(stderr, "ERROR: failed to fetch reply from ");
+    if (hostsocket == NULL)
+        fprintf(stderr, "%s:%d\n", ip, port);
+    else
+        fprintf(stderr, "%s\n", hostsocket);
+cleanup:
+    freeReplyObject(reply);
+    redisFree(ctx);
+    return NULL;
+}
+
 static redisConfig *getRedisConfig(const char *ip, int port,
                                    const char *hostsocket)
 {
@@ -245,33 +293,11 @@ static redisConfig *getRedisConfig(const char *ip, int port,
     if (!cfg) return NULL;
     redisContext *c = NULL;
     redisReply *reply = NULL, *sub_reply = NULL;
-    if (hostsocket == NULL)
-        c = redisConnect(ip, port);
-    else
-        c = redisConnectUnix(hostsocket);
-    if (c == NULL || c->err) {
-        fprintf(stderr,"Could not connect to Redis at ");
-        char *err = (c != NULL ? c->errstr : "");
-        if (hostsocket == NULL) fprintf(stderr,"%s:%d: %s\n",ip,port,err);
-        else fprintf(stderr,"%s: %s\n",hostsocket,err);
-        goto fail;
+    c = getRedisContext(ip, port, hostsocket);
+    if (c == NULL) {
+        freeRedisConfig(cfg);
+        return NULL;
     }
-
-    if(config.auth) {
-        void *authReply = NULL;
-        if (config.user == NULL)
-            redisAppendCommand(c, "AUTH %s", config.auth);
-        else
-            redisAppendCommand(c, "AUTH %s %s", config.user, config.auth);
-        if (REDIS_OK != redisGetReply(c, &authReply)) goto fail;
-        if (reply) freeReplyObject(reply);
-        reply = ((redisReply *) authReply);
-        if (reply->type == REDIS_REPLY_ERROR) {
-            fprintf(stderr, "ERROR: %s\n", reply->str);
-            goto fail;
-        }
-    }
-
     redisAppendCommand(c, "CONFIG GET %s", "save");
     redisAppendCommand(c, "CONFIG GET %s", "appendonly");
     int i = 0;
@@ -994,16 +1020,8 @@ static int fetchClusterConfiguration() {
     int success = 1;
     redisContext *ctx = NULL;
     redisReply *reply =  NULL;
-    if (config.hostsocket == NULL)
-        ctx = redisConnect(config.hostip,config.hostport);
-    else
-        ctx = redisConnectUnix(config.hostsocket);
-    if (ctx->err) {
-        fprintf(stderr,"Could not connect to Redis at ");
-        if (config.hostsocket == NULL) {
-            fprintf(stderr,"%s:%d: %s\n",config.hostip,config.hostport,
-                    ctx->errstr);
-        } else fprintf(stderr,"%s: %s\n",config.hostsocket,ctx->errstr);
+    ctx = getRedisContext(config.hostip, config.hostport, config.hostsocket);
+    if (ctx == NULL) {
         exit(1);
     }
     clusterNode *firstNode = createClusterNode((char *) config.hostip,
@@ -1199,11 +1217,9 @@ static int fetchClusterSlotsConfiguration(client c) {
         assert(node->port);
         /* Use first node as entry point to connect to. */
         if (ctx == NULL) {
-            ctx = redisConnect(node->ip, node->port);
-            if (!ctx || ctx->err) {
+            ctx = getRedisContext(node->ip, node->port, NULL);
+            if (!ctx) {
                 success = 0;
-                if (ctx && ctx->err)
-                    fprintf(stderr, "REDIS CONNECTION ERROR: %s\n", ctx->errstr);
                 goto cleanup;
             }
         }

From e9aba28932c6a0450c14ecd984603d41f31e0f0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9D=A8=E5=8D=9A=E4=B8=9C?= <bodong.ybd@alibaba-inc.com>
Date: Sat, 11 Jul 2020 20:52:41 +0800
Subject: [PATCH 0474/1098] STORE variants: SINTER,SUNION,SDIFF,ZUNION use
 setKey instead of dbDelete+dbAdd (#7489)

one of the differences (other than consistent code with SORT, GEORADIUS), is that the LFU of the old key is retained.
---
 src/t_set.c  | 32 ++++++++++++++++----------------
 src/t_zset.c | 10 +++-------
 2 files changed, 19 insertions(+), 23 deletions(-)

diff --git a/src/t_set.c b/src/t_set.c
index c2e73a6e6..75678f47c 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -895,21 +895,21 @@ void sinterGenericCommand(client *c, robj **setkeys,
     if (dstkey) {
         /* Store the resulting set into the target, if the intersection
          * is not an empty set. */
-        int deleted = dbDelete(c->db,dstkey);
         if (setTypeSize(dstset) > 0) {
-            dbAdd(c->db,dstkey,dstset);
+            setKey(c,c->db,dstkey,dstset);
             addReplyLongLong(c,setTypeSize(dstset));
             notifyKeyspaceEvent(NOTIFY_SET,"sinterstore",
                 dstkey,c->db->id);
+            server.dirty++;
         } else {
-            decrRefCount(dstset);
             addReply(c,shared.czero);
-            if (deleted)
-                notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
-                    dstkey,c->db->id);
+            if (dbDelete(c->db,dstkey)) {
+                server.dirty++;
+                signalModifiedKey(c,c->db,dstkey);
+                notifyKeyspaceEvent(NOTIFY_GENERIC,"del",dstkey,c->db->id);
+            }
         }
-        signalModifiedKey(c,c->db,dstkey);
-        server.dirty++;
+        decrRefCount(dstset);
     } else {
         setDeferredSetLen(c,replylen,cardinality);
     }
@@ -1069,22 +1069,22 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
     } else {
         /* If we have a target key where to store the resulting set
          * create this key with the result set inside */
-        int deleted = dbDelete(c->db,dstkey);
         if (setTypeSize(dstset) > 0) {
-            dbAdd(c->db,dstkey,dstset);
+            setKey(c,c->db,dstkey,dstset);
             addReplyLongLong(c,setTypeSize(dstset));
             notifyKeyspaceEvent(NOTIFY_SET,
                 op == SET_OP_UNION ? "sunionstore" : "sdiffstore",
                 dstkey,c->db->id);
+            server.dirty++;
         } else {
-            decrRefCount(dstset);
             addReply(c,shared.czero);
-            if (deleted)
-                notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
-                    dstkey,c->db->id);
+            if (dbDelete(c->db,dstkey)) {
+                server.dirty++;
+                signalModifiedKey(c,c->db,dstkey);
+                notifyKeyspaceEvent(NOTIFY_GENERIC,"del",dstkey,c->db->id);
+            }
         }
-        signalModifiedKey(c,c->db,dstkey);
-        server.dirty++;
+        decrRefCount(dstset);
     }
     zfree(sets);
 }
diff --git a/src/t_zset.c b/src/t_zset.c
index 9c409cd96..16131ee85 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -2184,7 +2184,6 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
     robj *dstobj;
     zset *dstzset;
     zskiplistNode *znode;
-    int touched = 0;
 
     /* expect setnum input keys to be given */
     if ((getLongFromObjectOrReply(c, c->argv[2], &setnum, NULL) != C_OK))
@@ -2377,26 +2376,23 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
         serverPanic("Unknown operator");
     }
 
-    if (dbDelete(c->db,dstkey))
-        touched = 1;
     if (dstzset->zsl->length) {
         zsetConvertToZiplistIfNeeded(dstobj,maxelelen);
-        dbAdd(c->db,dstkey,dstobj);
+        setKey(c,c->db,dstkey,dstobj);
         addReplyLongLong(c,zsetLength(dstobj));
-        signalModifiedKey(c,c->db,dstkey);
         notifyKeyspaceEvent(NOTIFY_ZSET,
             (op == SET_OP_UNION) ? "zunionstore" : "zinterstore",
             dstkey,c->db->id);
         server.dirty++;
     } else {
-        decrRefCount(dstobj);
         addReply(c,shared.czero);
-        if (touched) {
+        if (dbDelete(c->db,dstkey)) {
             signalModifiedKey(c,c->db,dstkey);
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",dstkey,c->db->id);
             server.dirty++;
         }
     }
+    decrRefCount(dstobj);
     zfree(src);
 }
 

From 67660881ed5b19a979425c37b3e8beea3349043c Mon Sep 17 00:00:00 2001
From: jimgreen2013 <jimgreen2013@qq.com>
Date: Sun, 12 Jul 2020 03:51:44 +0800
Subject: [PATCH 0475/1098] fix description about ziplist, the code is ok
 (#6318)

* fix description about ZIP_BIG_PREVLEN(the code is ok), it's similar to
antirez#4705

* fix description about ziplist entry encoding field (the code is ok),
the max length should be 2^32 - 1 when encoding is 5 bytes
---
 src/ziplist.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/ziplist.c b/src/ziplist.c
index ddae0d96f..13881c117 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -86,7 +86,7 @@
  * |10000000|qqqqqqqq|rrrrrrrr|ssssssss|tttttttt| - 5 bytes
  *      String value with length greater than or equal to 16384 bytes.
  *      Only the 4 bytes following the first byte represents the length
- *      up to 32^2-1. The 6 lower bits of the first byte are not used and
+ *      up to 2^32-1. The 6 lower bits of the first byte are not used and
  *      are set to zero.
  *      IMPORTANT: The 32 bit number is stored in big endian.
  * |11000000| - 3 bytes
@@ -194,7 +194,7 @@
 #define ZIP_BIG_PREVLEN 254 /* Max number of bytes of the previous entry, for
                                the "prevlen" field prefixing each entry, to be
                                represented with just a single byte. Otherwise
-                               it is represented as FF AA BB CC DD, where
+                               it is represented as FE AA BB CC DD, where
                                AA BB CC DD are a 4 bytes unsigned integer
                                representing the previous entry len. */
 

From 7f19a04f0f049720ff5f84f3ab1aa81014f2f4ed Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 12 Jul 2020 13:55:26 +0300
Subject: [PATCH 0476/1098] update release scripts for new hosts, and CI to run
 more tests (#7480)

* update daily CI to include cluster and sentinel tests
* update daily CI to run when creating a new release
* update release scripts to work on the new redis.io hosts
---
 .github/workflows/daily.yml             | 26 ++++++++++++++++---
 utils/releasetools/01_create_tarball.sh |  3 +--
 utils/releasetools/02_upload_tarball.sh | 22 ++++++++++++++---
 utils/releasetools/03_test_release.sh   | 33 +++++++++++++------------
 utils/releasetools/04_release_hash.sh   | 10 ++++++--
 5 files changed, 68 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index acc4dd33a..4d54fbc42 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -1,14 +1,16 @@
 name: Daily
 
 on:
+  release:
+    types: [created]
   schedule:
-    - cron: '0 7 * * *'
+    - cron: '0 0 * * *'
 
 jobs:
 
   test-jemalloc:
     runs-on: ubuntu-latest
-    timeout-minutes: 1200
+    timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v1
     - name: make
@@ -19,10 +21,14 @@ jobs:
         ./runtest --accurate --verbose
     - name: module api test
       run: ./runtest-moduleapi --verbose
+    - name: sentinel tests
+      run: ./runtest-sentinel
+    - name: cluster tests
+      run: ./runtest-cluster
 
   test-libc-malloc:
     runs-on: ubuntu-latest
-    timeout-minutes: 1200
+    timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v1
     - name: make
@@ -33,9 +39,14 @@ jobs:
         ./runtest --accurate --verbose
     - name: module api test
       run: ./runtest-moduleapi --verbose
+    - name: sentinel tests
+      run: ./runtest-sentinel
+    - name: cluster tests
+      run: ./runtest-cluster
 
   test-32bit:
     runs-on: ubuntu-latest
+    timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v1
     - name: make
@@ -50,9 +61,14 @@ jobs:
       run: |
         make -C tests/modules 32bit # the script below doesn't have an argument, we must build manually ahead of time
         ./runtest-moduleapi --verbose
+    - name: sentinel tests
+      run: ./runtest-sentinel
+    - name: cluster tests
+      run: ./runtest-cluster
 
   test-tls:
     runs-on: ubuntu-latest
+    timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v1
     - name: make
@@ -65,6 +81,10 @@ jobs:
         ./runtest --accurate --verbose --tls
     - name: module api test
       run: ./runtest-moduleapi --verbose --tls
+    - name: sentinel tests
+      run: ./runtest-sentinel
+    - name: cluster tests
+      run: ./runtest-cluster
 
   test-valgrind:
     runs-on: ubuntu-latest
diff --git a/utils/releasetools/01_create_tarball.sh b/utils/releasetools/01_create_tarball.sh
index 54bca8c04..366a61e2c 100755
--- a/utils/releasetools/01_create_tarball.sh
+++ b/utils/releasetools/01_create_tarball.sh
@@ -1,14 +1,13 @@
 #!/bin/sh
 if [ $# != "1" ]
 then
-    echo "Usage: ./mkrelease.sh <git-ref>"
+    echo "Usage: ./utils/releasetools/01_create_tarball.sh <version_tag>"
     exit 1
 fi
 
 TAG=$1
 TARNAME="redis-${TAG}.tar"
 echo "Generating /tmp/${TARNAME}"
-cd ~/hack/redis
 git archive $TAG --prefix redis-${TAG}/ > /tmp/$TARNAME || exit 1
 echo "Gizipping the archive"
 rm -f /tmp/$TARNAME.gz
diff --git a/utils/releasetools/02_upload_tarball.sh b/utils/releasetools/02_upload_tarball.sh
index ed7065388..6400efad7 100755
--- a/utils/releasetools/02_upload_tarball.sh
+++ b/utils/releasetools/02_upload_tarball.sh
@@ -1,6 +1,22 @@
 #!/bin/bash
+if [ $# != "1" ]
+then
+    echo "Usage: ./utils/releasetools/02_upload_tarball.sh <version_tag>"
+    exit 1
+fi
+
 echo "Uploading..."
-scp /tmp/redis-${1}.tar.gz antirez@antirez.com:/var/virtual/download.redis.io/httpdocs/releases/
-echo "Updating web site... (press any key if it is a stable release, or Ctrl+C)"
+scp /tmp/redis-${1}.tar.gz ubuntu@host.redis.io:/var/www/download/releases/
+echo "Updating web site... "
+echo "Please check the github action tests for the release."
+echo "Press any key if it is a stable release, or Ctrl+C to abort"
 read x
-ssh antirez@antirez.com "cd /var/virtual/download.redis.io/httpdocs; ./update.sh ${1}"
+ssh ubuntu@host.redis.io "cd /var/www/download;
+                          rm -rf redis-${1}.tar.gz;
+                          wget http://download.redis.io/releases/redis-${1}.tar.gz;
+                          tar xvzf redis-${1}.tar.gz;
+                          rm -rf redis-stable;
+                          mv redis-${1} redis-stable;
+                          tar cvzf redis-stable.tar.gz redis-stable;
+                          rm -rf redis-${1}.tar.gz;
+                          "
diff --git a/utils/releasetools/03_test_release.sh b/utils/releasetools/03_test_release.sh
index 3dfdcd6a3..169e965d5 100755
--- a/utils/releasetools/03_test_release.sh
+++ b/utils/releasetools/03_test_release.sh
@@ -1,7 +1,8 @@
 #!/bin/sh
+set -e
 if [ $# != "1" ]
 then
-    echo "Usage: ${0} <git-ref>"
+    echo "Usage: ./utils/releasetools/03_test_release.sh <version_tag>"
     exit 1
 fi
 
@@ -9,18 +10,18 @@ TAG=$1
 TARNAME="redis-${TAG}.tar.gz"
 DOWNLOADURL="http://download.redis.io/releases/${TARNAME}"
 
-ssh antirez@metal "export TERM=xterm;
-                   cd /tmp;
-                   rm -rf test_release_tmp_dir;
-                   cd test_release_tmp_dir;
-                   rm -f $TARNAME;
-                   rm -rf redis-${TAG};
-                   wget $DOWNLOADURL;
-                   tar xvzf $TARNAME;
-                   cd redis-${TAG};
-                   make;
-                   ./runtest;
-                   ./runtest-sentinel;
-                   if [ -x runtest-cluster ]; then
-                       ./runtest-cluster;
-                   fi"
+echo "Doing sanity test on the actual tarball"
+
+cd /tmp
+rm -rf test_release_tmp_dir
+cd test_release_tmp_dir
+rm -f $TARNAME
+rm -rf redis-${TAG}
+wget $DOWNLOADURL
+tar xvzf $TARNAME
+cd redis-${TAG}
+make
+./runtest
+./runtest-sentinel
+./runtest-cluster
+./runtest-moduleapi
diff --git a/utils/releasetools/04_release_hash.sh b/utils/releasetools/04_release_hash.sh
index 9d5c6ad4b..bc1ebb66c 100755
--- a/utils/releasetools/04_release_hash.sh
+++ b/utils/releasetools/04_release_hash.sh
@@ -1,8 +1,14 @@
 #!/bin/bash
+if [ $# != "1" ]
+then
+    echo "Usage: ./utils/releasetools/04_release_hash.sh <version_tag>"
+    exit 1
+fi
+
 SHA=$(curl -s http://download.redis.io/releases/redis-${1}.tar.gz | shasum -a 256 | cut -f 1 -d' ')
 ENTRY="hash redis-${1}.tar.gz sha256 $SHA http://download.redis.io/releases/redis-${1}.tar.gz"
 echo $ENTRY >> ~/hack/redis-hashes/README
-vi ~/hack/redis-hashes/README
+vi ../redis-hashes/README
 echo "Press any key to commit, Ctrl-C to abort)."
 read yes
-(cd ~/hack/redis-hashes; git commit -a -m "${1} hash."; git push)
+(cd ../redis-hashes; git commit -a -m "${1} hash."; git push)

From 02ef355f98691adba4126bbdab0d4d2bfe475701 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 13 Jul 2020 16:09:08 +0300
Subject: [PATCH 0477/1098] runtest --stop pause stops before terminating the
 redis server (#7513)

in the majority of the cases (on this rarely used feature) we want to
stop and be able to connect to the shard with redis-cli.
since these are two different processes interracting with the tty we
need to stop both, and we'll have to hit enter twice, but it's not that
bad considering it is rarely used.
---
 tests/support/test.tcl | 6 ++++++
 tests/test_helper.tcl  | 4 ++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/support/test.tcl b/tests/support/test.tcl
index 5e8916236..a5573f583 100644
--- a/tests/support/test.tcl
+++ b/tests/support/test.tcl
@@ -153,6 +153,12 @@ proc test {name code {okpattern undefined}} {
 
             incr ::num_failed
             send_data_packet $::test_server_fd err [join $details "\n"]
+
+            if {$::stop_on_failure} {
+                puts "Test error (last server port:[srv port], log:[srv stdout]), press enter to teardown the test."
+                flush stdout
+                gets stdin
+            }
         } else {
             # Re-raise, let handler up the stack take care of this.
             error $error $::errorInfo
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 51c364601..7ce0d545e 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -359,8 +359,8 @@ proc read_from_test_client fd {
         puts $err
         lappend ::failed_tests $err
         set ::active_clients_task($fd) "(ERR) $data"
-            if {$::stop_on_failure} {
-            puts -nonewline "(Test stopped, press enter to continue)"
+        if {$::stop_on_failure} {
+            puts -nonewline "(Test stopped, press enter to resume the tests)"
             flush stdout
             gets stdin
         }

From e5227aab899628653285478a9d1083e8e8f51b57 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 13 Jul 2020 16:40:03 +0300
Subject: [PATCH 0478/1098] fix recently added time sensitive tests failing
 with valgrind (#7512)

interestingly the latency monitor test fails because valgrind is slow
enough so that the time inside PEXPIREAT command from the moment of
the first mstime() call to get the basetime until checkAlreadyExpired
calls mstime() again is more than 1ms, and that test was too sensitive.

using this opportunity to speed up the test (unrelated to the failure)
the fix is just the longer time passed to PEXPIRE.
---
 tests/integration/redis-cli.tcl |  2 +-
 tests/unit/latency-monitor.tcl  | 16 +++++++++++-----
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/tests/integration/redis-cli.tcl b/tests/integration/redis-cli.tcl
index 016e4915c..aa8b92199 100644
--- a/tests/integration/redis-cli.tcl
+++ b/tests/integration/redis-cli.tcl
@@ -241,7 +241,7 @@ start_server {tags {"cli"}} {
 
     test_nontty_cli "Connecting as a replica" {
         set fd [open_cli "--replica"]
-        wait_for_condition 50 500 {
+        wait_for_condition 200 500 {
             [string match {*slave0:*state=online*} [r info]]
         } else {
             fail "redis-cli --replica did not connect"
diff --git a/tests/unit/latency-monitor.tcl b/tests/unit/latency-monitor.tcl
index 69da13f06..d76867cc6 100644
--- a/tests/unit/latency-monitor.tcl
+++ b/tests/unit/latency-monitor.tcl
@@ -50,15 +50,21 @@ start_server {tags {"latency-monitor"}} {
 
     test {LATENCY of expire events are correctly collected} {
         r config set latency-monitor-threshold 20
+        r flushdb
+        if {$::valgrind} {set count 100000} else {set count 1000000}
         r eval {
             local i = 0
-            while (i < 1000000) do
-                redis.call('sadd','mybigkey',i)
+            while (i < tonumber(ARGV[1])) do
+                redis.call('sadd',KEYS[1],i)
                 i = i+1
              end
-        } 0
-        r pexpire mybigkey 1
-        after 500
+        } 1 mybigkey $count
+        r pexpire mybigkey 50
+        wait_for_condition 5 100 {
+            [r dbsize] == 0
+        } else {
+            fail "key wasn't expired"
+        }
         assert_match {*expire-cycle*} [r latency latest]
     }
 }

From 6a814501448c5673d05eb2cfe4fe650883da8f0e Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 13 Jul 2020 16:40:19 +0300
Subject: [PATCH 0479/1098] RESTORE ABSTTL skip expired keys - leak (#7511)

---
 src/cluster.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/cluster.c b/src/cluster.c
index 88b810d13..45fb5552e 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -5030,6 +5030,7 @@ void restoreCommand(client *c) {
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
             server.dirty++;
         }
+        decrRefCount(obj);
         addReply(c, shared.ok);
         return;
     }

From 938c35302fb618ec15e62443ef3bb3d00d10f5b9 Mon Sep 17 00:00:00 2001
From: Qu Chen <QuChen88@users.noreply.github.com>
Date: Mon, 13 Jul 2020 07:16:06 -0700
Subject: [PATCH 0480/1098] Replica always reports master's config epoch in
 CLUSTER NODES output. (#7235)

---
 src/cluster.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/cluster.c b/src/cluster.c
index 45fb5552e..5dcd69ff8 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4104,11 +4104,15 @@ sds clusterGenNodeDescription(clusterNode *node) {
     else
         ci = sdscatlen(ci," - ",3);
 
+    unsigned long long nodeEpoch = node->configEpoch;
+    if (nodeIsSlave(node) && node->slaveof) {
+        nodeEpoch = node->slaveof->configEpoch;
+    }
     /* Latency from the POV of this node, config epoch, link status */
     ci = sdscatprintf(ci,"%lld %lld %llu %s",
         (long long) node->ping_sent,
         (long long) node->pong_received,
-        (unsigned long long) node->configEpoch,
+        nodeEpoch,
         (node->link || node->flags & CLUSTER_NODE_MYSELF) ?
                     "connected" : "disconnected");
 

From 8128d39737adaf7092c9a367f71fbe9e0a2b33a2 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Tue, 14 Jul 2020 16:35:04 +0800
Subject: [PATCH 0481/1098] Fix out of update help info in tcl tests. (#7516)

Before this commit, the output of "./runtest-cluster --help" is incorrect.
After this commit, the format of the following 3 output is consistent:
./runtest --help
./runtest-cluster --help
./runtest-sentinel --help
---
 tests/instances.tcl | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/instances.tcl b/tests/instances.tcl
index 3a4fadca0..677af6427 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -170,8 +170,6 @@ proc parse_options {} {
                 -keyfile "$::tlsdir/redis.key"
             set ::tls 1
         } elseif {$opt eq "--help"} {
-            puts "Hello, I'm sentinel.tcl and I run Sentinel unit tests."
-            puts "\nOptions:"
             puts "--single <pattern>      Only runs tests specified by pattern."
             puts "--pause-on-error        Pause for manual inspection on error."
             puts "--fail                  Simulate a test failure."

From 254c96255420e950bcad1a46bc4f8617b4373797 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 14 Jul 2020 18:04:08 +0300
Subject: [PATCH 0482/1098] redis-cli tests, fix valgrind timing issue (#7519)

this test when run with valgrind on github actions takes 160 seconds
---
 tests/integration/redis-cli.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/redis-cli.tcl b/tests/integration/redis-cli.tcl
index aa8b92199..c70d14eeb 100644
--- a/tests/integration/redis-cli.tcl
+++ b/tests/integration/redis-cli.tcl
@@ -241,7 +241,7 @@ start_server {tags {"cli"}} {
 
     test_nontty_cli "Connecting as a replica" {
         set fd [open_cli "--replica"]
-        wait_for_condition 200 500 {
+        wait_for_condition 500 500 {
             [string match {*slave0:*state=online*} [r info]]
         } else {
             fail "redis-cli --replica did not connect"

From a176cb56a3c0235adddde33fcbaee2369a5af73e Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 14 Jul 2020 20:21:59 +0300
Subject: [PATCH 0483/1098] diskless master disconnect replicas when rdb child
 failed (#7518)

in case the rdb child failed, crashed or terminated unexpectedly redis
would have marked the replica clients with repl_put_online_on_ack and
then kill them only after a minute when no ack was received.

it would not stream anything to these connections, so the only effect of
this bug is a delay of 1 minute in the replicas attempt to re-connect.
---
 src/replication.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index d9bff79ad..8457150a0 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1240,6 +1240,12 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type) {
         } else if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END) {
             struct redis_stat buf;
 
+            if (bgsaveerr != C_OK) {
+                freeClient(slave);
+                serverLog(LL_WARNING,"SYNC failed. BGSAVE child returned an error");
+                continue;
+            }
+
             /* If this was an RDB on disk save, we have to prepare to send
              * the RDB from disk to the slave socket. Otherwise if this was
              * already an RDB -> Slaves socket transfer, used in the case of
@@ -1278,11 +1284,6 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type) {
                 slave->repl_put_online_on_ack = 1;
                 slave->repl_ack_time = server.unixtime; /* Timeout otherwise. */
             } else {
-                if (bgsaveerr != C_OK) {
-                    freeClient(slave);
-                    serverLog(LL_WARNING,"SYNC failed. BGSAVE child returned an error");
-                    continue;
-                }
                 if ((slave->repldbfd = open(server.rdb_filename,O_RDONLY)) == -1 ||
                     redis_fstat(slave->repldbfd,&buf) == -1) {
                     freeClient(slave);

From dc690161d5652d86d51bd209821bfb0e9c5f7ec2 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Wed, 15 Jul 2020 17:37:44 +0800
Subject: [PATCH 0484/1098] Refactor RM_KeyType() by using macro. (#7486)

---
 src/module.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/module.c b/src/module.c
index 226c60fd0..acfaf2ac7 100644
--- a/src/module.c
+++ b/src/module.c
@@ -2044,7 +2044,7 @@ int RM_KeyType(RedisModuleKey *key) {
     case OBJ_HASH: return REDISMODULE_KEYTYPE_HASH;
     case OBJ_MODULE: return REDISMODULE_KEYTYPE_MODULE;
     case OBJ_STREAM: return REDISMODULE_KEYTYPE_STREAM;
-    default: return 0;
+    default: return REDISMODULE_KEYTYPE_EMPTY;
     }
 }
 

From 93bdbf5aa4857ede0816cf790f951da8e2fa2ae9 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Wed, 15 Jul 2020 17:38:22 +0800
Subject: [PATCH 0485/1098] Fix command help for unexpected options (#7476)

---
 src/acl.c                      | 2 +-
 src/latency.c                  | 2 +-
 src/t_stream.c                 | 2 +-
 tests/unit/acl.tcl             | 5 +++++
 tests/unit/latency-monitor.tcl | 5 +++++
 tests/unit/type/stream.tcl     | 7 +++++++
 6 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 6dd0f70ac..3ce45f03b 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1911,7 +1911,7 @@ void aclCommand(client *c) {
             addReplyBulkCString(c,"client-info");
             addReplyBulkCBuffer(c,le->cinfo,sdslen(le->cinfo));
         }
-    } else if (!strcasecmp(sub,"help")) {
+    } else if (c->argc == 2 && !strcasecmp(sub,"help")) {
         const char *help[] = {
 "LOAD                             -- Reload users from the ACL file.",
 "SAVE                             -- Save the current config to the ACL file.",
diff --git a/src/latency.c b/src/latency.c
index 9a291ac9b..dfdc6668c 100644
--- a/src/latency.c
+++ b/src/latency.c
@@ -621,7 +621,7 @@ NULL
                 resets += latencyResetEvent(c->argv[j]->ptr);
             addReplyLongLong(c,resets);
         }
-    } else if (!strcasecmp(c->argv[1]->ptr,"help") && c->argc >= 2) {
+    } else if (!strcasecmp(c->argv[1]->ptr,"help") && c->argc == 2) {
         addReplyHelp(c, help);
     } else {
         addReplySubcommandSyntaxError(c);
diff --git a/src/t_stream.c b/src/t_stream.c
index 676ddd9bb..f564b1ff9 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1885,7 +1885,7 @@ NULL
         server.dirty++;
         notifyKeyspaceEvent(NOTIFY_STREAM,"xgroup-delconsumer",
                             c->argv[2],c->db->id);
-    } else if (!strcasecmp(opt,"HELP")) {
+    } else if (c->argc == 2 && !strcasecmp(opt,"HELP")) {
         addReplyHelp(c, help);
     } else {
         addReplySubcommandSyntaxError(c);
diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl
index 85c9b81a9..e81280995 100644
--- a/tests/unit/acl.tcl
+++ b/tests/unit/acl.tcl
@@ -255,4 +255,9 @@ start_server {tags {"acl"}} {
         r ACL setuser default on
         set e
     } {*NOAUTH*}
+
+    test {ACL HELP should not have unexpected options} {
+        catch {r ACL help xxx} e
+        assert_match "*Unknown subcommand or wrong number of arguments*" $e
+    }
 }
diff --git a/tests/unit/latency-monitor.tcl b/tests/unit/latency-monitor.tcl
index d76867cc6..18b9ecebb 100644
--- a/tests/unit/latency-monitor.tcl
+++ b/tests/unit/latency-monitor.tcl
@@ -67,4 +67,9 @@ start_server {tags {"latency-monitor"}} {
         }
         assert_match {*expire-cycle*} [r latency latest]
     }
+
+    test {LATENCY HELP should not have unexpected options} {
+        catch {r LATENCY help xxx} e
+        assert_match "*Unknown subcommand or wrong number of arguments*" $e
+    }
 }
diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl
index c2b524d7f..0ff570cab 100644
--- a/tests/unit/type/stream.tcl
+++ b/tests/unit/type/stream.tcl
@@ -461,3 +461,10 @@ start_server {tags {"stream"} overrides {appendonly yes aof-use-rdb-preamble no}
         assert {[dict get [r xinfo stream mystream] last-generated-id] == "2-2"}
     }
 }
+
+start_server {tags {"stream"}} {
+    test {XGROUP HELP should not have unexpected options} {
+        catch {r XGROUP help xxx} e
+        assert_match "*Unknown subcommand or wrong number of arguments*" $e
+    }
+}

From d85af4d6f5fbe9cb9787b81583627cd74b47f838 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Wed, 15 Jul 2020 05:38:47 -0400
Subject: [PATCH 0486/1098] correct error msg for num connections reaching
 maxclients in cluster mode (#7444)

---
 src/networking.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index d35347991..589a459d4 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -907,10 +907,10 @@ static void acceptCommonHandler(connection *conn, int flags, char *ip) {
     {
         char *err;
         if (server.cluster_enabled)
-            err = "-ERR max number of clients reached\r\n";
-        else
             err = "-ERR max number of clients + cluster "
                   "connections reached\r\n";
+        else
+            err = "-ERR max number of clients reached\r\n";
 
         /* That's a best effort error message, don't check write errors.
          * Note that for TLS connections, no handshake was done yet so nothing

From c2b5f1c15bbc5363f92f8e697538759c5d929934 Mon Sep 17 00:00:00 2001
From: Developer-Ecosystem-Engineering
 <65677710+Developer-Ecosystem-Engineering@users.noreply.github.com>
Date: Wed, 15 Jul 2020 02:44:03 -0700
Subject: [PATCH 0487/1098] Add registers dump support for Apple silicon
 (#7453)

Export following environment variables before building on macOS on Apple silicon

export ARCH_FLAGS="-arch arm64"
export SDK_NAME=macosx
export SDK_PATH=$(xcrun --show-sdk-path --sdk $SDK_NAME)
export CFLAGS="$ARCH_FLAGS -isysroot $SDK_PATH -I$SDK_PATH/usr/include"
export CXXFLAGS=$CFLAGS
export LDFLAGS="$ARCH_FLAGS"
export CC="$(xcrun -sdk $SDK_PATH --find clang) $CFLAGS"
export CXX="$(xcrun -sdk $SDK_PATH --find clang++) $CXXFLAGS"
export LD="$(xcrun -sdk $SDK_PATH --find ld) $LDFLAGS"

make
make test
..
All tests passed without errors!

Backtrack logging assumes x86 and required updating
---
 src/debug.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 54 insertions(+), 2 deletions(-)

diff --git a/src/debug.c b/src/debug.c
index a74c22647..60cc2a1fa 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -928,8 +928,11 @@ static void *getMcontextEip(ucontext_t *uc) {
     /* OSX >= 10.6 */
     #if defined(_STRUCT_X86_THREAD_STATE64) && !defined(__i386__)
     return (void*) uc->uc_mcontext->__ss.__rip;
-    #else
+    #elif defined(__i386__)
     return (void*) uc->uc_mcontext->__ss.__eip;
+    #else
+    /* OSX ARM64 */
+    return (void*) arm_thread_state64_get_pc(uc->uc_mcontext->__ss);
     #endif
 #elif defined(__linux__)
     /* Linux */
@@ -1015,7 +1018,7 @@ void logRegisters(ucontext_t *uc) {
         (unsigned long) uc->uc_mcontext->__ss.__gs
     );
     logStackContent((void**)uc->uc_mcontext->__ss.__rsp);
-    #else
+    #elif defined(__i386__)
     /* OSX x86 */
     serverLog(LL_WARNING,
     "\n"
@@ -1041,6 +1044,55 @@ void logRegisters(ucontext_t *uc) {
         (unsigned long) uc->uc_mcontext->__ss.__gs
     );
     logStackContent((void**)uc->uc_mcontext->__ss.__esp);
+    #else
+    /* OSX ARM64 */
+    serverLog(LL_WARNING,
+    "\n"
+    "x0:%016lx x1:%016lx x2:%016lx x3:%016lx\n"
+    "x4:%016lx x5:%016lx x6:%016lx x7:%016lx\n"
+    "x8:%016lx x9:%016lx x10:%016lx x11:%016lx\n"
+    "x12:%016lx x13:%016lx x14:%016lx x15:%016lx\n"
+    "x16:%016lx x17:%016lx x18:%016lx x19:%016lx\n"
+    "x20:%016lx x21:%016lx x22:%016lx x23:%016lx\n"
+    "x24:%016lx x25:%016lx x26:%016lx x27:%016lx\n"
+    "x28:%016lx fp:%016lx lr:%016lx\n"
+    "sp:%016lx pc:%016lx cpsr:%08lx\n",
+        (unsigned long) uc->uc_mcontext->__ss.__x[0],
+        (unsigned long) uc->uc_mcontext->__ss.__x[1],
+        (unsigned long) uc->uc_mcontext->__ss.__x[2],
+        (unsigned long) uc->uc_mcontext->__ss.__x[3],
+        (unsigned long) uc->uc_mcontext->__ss.__x[4],
+        (unsigned long) uc->uc_mcontext->__ss.__x[5],
+        (unsigned long) uc->uc_mcontext->__ss.__x[6],
+        (unsigned long) uc->uc_mcontext->__ss.__x[7],
+        (unsigned long) uc->uc_mcontext->__ss.__x[8],
+        (unsigned long) uc->uc_mcontext->__ss.__x[9],
+        (unsigned long) uc->uc_mcontext->__ss.__x[10],
+        (unsigned long) uc->uc_mcontext->__ss.__x[11],
+        (unsigned long) uc->uc_mcontext->__ss.__x[12],
+        (unsigned long) uc->uc_mcontext->__ss.__x[13],
+        (unsigned long) uc->uc_mcontext->__ss.__x[14],
+        (unsigned long) uc->uc_mcontext->__ss.__x[15],
+        (unsigned long) uc->uc_mcontext->__ss.__x[16],
+        (unsigned long) uc->uc_mcontext->__ss.__x[17],
+        (unsigned long) uc->uc_mcontext->__ss.__x[18],
+        (unsigned long) uc->uc_mcontext->__ss.__x[19],
+        (unsigned long) uc->uc_mcontext->__ss.__x[20],
+        (unsigned long) uc->uc_mcontext->__ss.__x[21],
+        (unsigned long) uc->uc_mcontext->__ss.__x[22],
+        (unsigned long) uc->uc_mcontext->__ss.__x[23],
+        (unsigned long) uc->uc_mcontext->__ss.__x[24],
+        (unsigned long) uc->uc_mcontext->__ss.__x[25],
+        (unsigned long) uc->uc_mcontext->__ss.__x[26],
+        (unsigned long) uc->uc_mcontext->__ss.__x[27],
+        (unsigned long) uc->uc_mcontext->__ss.__x[28],
+        (unsigned long) arm_thread_state64_get_fp(uc->uc_mcontext->__ss),
+        (unsigned long) arm_thread_state64_get_lr(uc->uc_mcontext->__ss),
+        (unsigned long) arm_thread_state64_get_sp(uc->uc_mcontext->__ss),
+        (unsigned long) arm_thread_state64_get_pc(uc->uc_mcontext->__ss),
+        (unsigned long) uc->uc_mcontext->__ss.__cpsr
+    );
+    logStackContent((void**) arm_thread_state64_get_sp(uc->uc_mcontext->__ss));
     #endif
 /* Linux */
 #elif defined(__linux__)

From 9242ccf238cbed018eb3a7fa3a437618345dd52b Mon Sep 17 00:00:00 2001
From: dmurnane <danieltm@gmail.com>
Date: Wed, 15 Jul 2020 06:29:26 -0400
Subject: [PATCH 0488/1098] Notify systemd on sentinel startup (#7168)

Co-authored-by: Daniel Murnane <dmurnane@eitccorp.com>
---
 src/server.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/server.c b/src/server.c
index 1f794e4ed..9c2126bc0 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5198,6 +5198,10 @@ int main(int argc, char **argv) {
     } else {
         InitServerLast();
         sentinelIsRunning();
+        if (server.supervised_mode == SUPERVISED_SYSTEMD) {
+            redisCommunicateSystemd("STATUS=Ready to accept connections\n");
+            redisCommunicateSystemd("READY=1\n");
+        }
     }
 
     /* Warning the user about suspicious maxmemory setting. */

From 5f716ea467d29059a89f90b6ccbdee5a60443200 Mon Sep 17 00:00:00 2001
From: Luke Palmer <luke@lukepalmer.net>
Date: Wed, 15 Jul 2020 13:53:41 -0400
Subject: [PATCH 0489/1098] Send null for invalidate on flush (#7469)

---
 src/tracking.c | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/tracking.c b/src/tracking.c
index 8c2dca7ba..2721de32a 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -198,9 +198,11 @@ void trackingRememberKeys(client *c) {
  *
  * In case the 'proto' argument is non zero, the function will assume that
  * 'keyname' points to a buffer of 'keylen' bytes already expressed in the
- * form of Redis RESP protocol, representing an array of keys to send
- * to the client as value of the invalidation. This is used in BCAST mode
- * in order to optimized the implementation to use less CPU time. */
+ * form of Redis RESP protocol. This is used for:
+ * - In BCAST mode, to send an array of invalidated keys to all
+ *   applicable clients
+ * - Following a flush command, to send a single RESP NULL to indicate
+ *   that all keys are now invalid. */
 void sendTrackingMessage(client *c, char *keyname, size_t keylen, int proto) {
     int using_redirection = 0;
     if (c->client_tracking_redirection) {
@@ -342,17 +344,19 @@ void trackingInvalidateKey(client *c, robj *keyobj) {
     trackingInvalidateKeyRaw(c,keyobj->ptr,sdslen(keyobj->ptr),1);
 }
 
-/* This function is called when one or all the Redis databases are flushed
- * (dbid == -1 in case of FLUSHALL). Caching keys are not specific for
- * each DB but are global: currently what we do is send a special
- * notification to clients with tracking enabled, invalidating the caching
- * key "", which means, "all the keys", in order to avoid flooding clients
- * with many invalidation messages for all the keys they may hold.
+/* This function is called when one or all the Redis databases are
+ * flushed (dbid == -1 in case of FLUSHALL). Caching keys are not
+ * specific for each DB but are global: currently what we do is send a
+ * special notification to clients with tracking enabled, sending a
+ * RESP NULL, which means, "all the keys", in order to avoid flooding
+ * clients with many invalidation messages for all the keys they may
+ * hold.
  */
 void freeTrackingRadixTree(void *rt) {
     raxFree(rt);
 }
 
+/* A RESP NULL is sent to indicate that all keys are invalid */
 void trackingInvalidateKeysOnFlush(int dbid) {
     if (server.tracking_clients) {
         listNode *ln;
@@ -361,7 +365,7 @@ void trackingInvalidateKeysOnFlush(int dbid) {
         while ((ln = listNext(&li)) != NULL) {
             client *c = listNodeValue(ln);
             if (c->flags & CLIENT_TRACKING) {
-                sendTrackingMessage(c,"",1,0);
+                sendTrackingMessage(c,shared.null[c->resp]->ptr,sdslen(shared.null[c->resp]->ptr),1);
             }
         }
     }

From 8596d483bcce1cb19466eb2303ffb2bb4981a45f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9D=A8=E5=8D=9A=E4=B8=9C?= <bodong.ybd@alibaba-inc.com>
Date: Thu, 16 Jul 2020 13:57:27 +0800
Subject: [PATCH 0490/1098] Stream avoid duplicate parse id (#7450)

---
 src/t_stream.c | 58 ++++++++++++++++++++++++++++++++------------------
 1 file changed, 37 insertions(+), 21 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index f564b1ff9..d2c5b45a3 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -40,6 +40,11 @@
 #define STREAM_ITEM_FLAG_DELETED (1<<0)     /* Entry is deleted. Skip it. */
 #define STREAM_ITEM_FLAG_SAMEFIELDS (1<<1)  /* Same fields as master entry. */
 
+/* For stream commands that require multiple IDs
+ * when the number of IDs is less than 'STREAMID_STATIC_VECTOR_LEN',
+ * avoid malloc allocation.*/
+#define STREAMID_STATIC_VECTOR_LEN 8
+
 void streamFreeCG(streamCG *cg);
 void streamFreeNACK(streamNACK *na);
 size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start, streamID *end, size_t count, streamConsumer *consumer);
@@ -1375,7 +1380,6 @@ void xreadCommand(client *c) {
     int streams_count = 0;
     int streams_arg = 0;
     int noack = 0;          /* True if NOACK option was specified. */
-    #define STREAMID_STATIC_VECTOR_LEN 8
     streamID static_ids[STREAMID_STATIC_VECTOR_LEN];
     streamID *ids = static_ids;
     streamCG **groups = NULL;
@@ -1949,18 +1953,19 @@ void xackCommand(client *c) {
      * error: the return value of this command cannot be an error in case
      * the client successfully acknowledged some messages, so it should be
      * executed in a "all or nothing" fashion. */
+    streamID static_ids[STREAMID_STATIC_VECTOR_LEN];
+    streamID *ids = static_ids;
+    int id_count = c->argc-3;
+    if (id_count > STREAMID_STATIC_VECTOR_LEN)
+        ids = zmalloc(sizeof(streamID)*id_count);
     for (int j = 3; j < c->argc; j++) {
-        streamID id;
-        if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return;
+        if (streamParseStrictIDOrReply(c,c->argv[j],&ids[j-3],0) != C_OK) goto cleanup;
     }
 
     int acknowledged = 0;
     for (int j = 3; j < c->argc; j++) {
-        streamID id;
         unsigned char buf[sizeof(streamID)];
-        if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK)
-            serverPanic("StreamID invalid after check. Should not be possible.");
-        streamEncodeID(buf,&id);
+        streamEncodeID(buf,&ids[j-3]);
 
         /* Lookup the ID in the group PEL: it will have a reference to the
          * NACK structure that will have a reference to the consumer, so that
@@ -1975,6 +1980,8 @@ void xackCommand(client *c) {
         }
     }
     addReplyLongLong(c,acknowledged);
+cleanup:
+    if (ids != static_ids) zfree(ids);
 }
 
 /* XPENDING <key> <group> [<start> <stop> <count> [<consumer>]]
@@ -2227,9 +2234,13 @@ void xclaimCommand(client *c) {
      * the client successfully claimed some message, so it should be
      * executed in a "all or nothing" fashion. */
     int j;
+    streamID static_ids[STREAMID_STATIC_VECTOR_LEN];
+    streamID *ids = static_ids;
+    int id_count = c->argc-5;
+    if (id_count > STREAMID_STATIC_VECTOR_LEN)
+        ids = zmalloc(sizeof(streamID)*id_count);
     for (j = 5; j < c->argc; j++) {
-        streamID id;
-        if (streamParseStrictIDOrReply(NULL,c->argv[j],&id,0) != C_OK) break;
+        if (streamParseStrictIDOrReply(NULL,c->argv[j],&ids[j-5],0) != C_OK) break;
     }
     int last_id_arg = j-1; /* Next time we iterate the IDs we now the range. */
 
@@ -2249,24 +2260,24 @@ void xclaimCommand(client *c) {
             j++;
             if (getLongLongFromObjectOrReply(c,c->argv[j],&deliverytime,
                 "Invalid IDLE option argument for XCLAIM")
-                != C_OK) return;
+                != C_OK) goto cleanup;
             deliverytime = now - deliverytime;
         } else if (!strcasecmp(opt,"TIME") && moreargs) {
             j++;
             if (getLongLongFromObjectOrReply(c,c->argv[j],&deliverytime,
                 "Invalid TIME option argument for XCLAIM")
-                != C_OK) return;
+                != C_OK) goto cleanup;
         } else if (!strcasecmp(opt,"RETRYCOUNT") && moreargs) {
             j++;
             if (getLongLongFromObjectOrReply(c,c->argv[j],&retrycount,
                 "Invalid RETRYCOUNT option argument for XCLAIM")
-                != C_OK) return;
+                != C_OK) goto cleanup;
         } else if (!strcasecmp(opt,"LASTID") && moreargs) {
             j++;
-            if (streamParseStrictIDOrReply(c,c->argv[j],&last_id,0) != C_OK) return;
+            if (streamParseStrictIDOrReply(c,c->argv[j],&last_id,0) != C_OK) goto cleanup;
         } else {
             addReplyErrorFormat(c,"Unrecognized XCLAIM option '%s'",opt);
-            return;
+            goto cleanup;
         }
     }
 
@@ -2296,10 +2307,8 @@ void xclaimCommand(client *c) {
     void *arraylenptr = addReplyDeferredLen(c);
     size_t arraylen = 0;
     for (int j = 5; j <= last_id_arg; j++) {
-        streamID id;
+        streamID id = ids[j-5];
         unsigned char buf[sizeof(streamID)];
-        if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK)
-            serverPanic("StreamID invalid after check. Should not be possible.");
         streamEncodeID(buf,&id);
 
         /* Lookup the ID in the group PEL. */
@@ -2379,6 +2388,8 @@ void xclaimCommand(client *c) {
     }
     setDeferredArrayLen(c,arraylenptr,arraylen);
     preventCommandPropagation(c);
+cleanup:
+    if (ids != static_ids) zfree(ids);
 }
 
 
@@ -2397,16 +2408,19 @@ void xdelCommand(client *c) {
     /* We need to sanity check the IDs passed to start. Even if not
      * a big issue, it is not great that the command is only partially
      * executed because at some point an invalid ID is parsed. */
-    streamID id;
+    streamID static_ids[STREAMID_STATIC_VECTOR_LEN];
+    streamID *ids = static_ids;
+    int id_count = c->argc-2;
+    if (id_count > STREAMID_STATIC_VECTOR_LEN)
+        ids = zmalloc(sizeof(streamID)*id_count);
     for (int j = 2; j < c->argc; j++) {
-        if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return;
+        if (streamParseStrictIDOrReply(c,c->argv[j],&ids[j-2],0) != C_OK) goto cleanup;
     }
 
     /* Actually apply the command. */
     int deleted = 0;
     for (int j = 2; j < c->argc; j++) {
-        streamParseStrictIDOrReply(c,c->argv[j],&id,0); /* Retval already checked. */
-        deleted += streamDeleteItem(s,&id);
+        deleted += streamDeleteItem(s,&ids[j-2]);
     }
 
     /* Propagate the write if needed. */
@@ -2416,6 +2430,8 @@ void xdelCommand(client *c) {
         server.dirty += deleted;
     }
     addReplyLongLong(c,deleted);
+cleanup:
+    if (ids != static_ids) zfree(ids);
 }
 
 /* General form: XTRIM <key> [... options ...]

From d484b8a04ed67e79030fcb060e88641acb6e4f98 Mon Sep 17 00:00:00 2001
From: yoav-steinberg <yoav@monfort.co.il>
Date: Thu, 16 Jul 2020 20:59:38 +0300
Subject: [PATCH 0491/1098] Support passing stack allocated module strings to
 moduleCreateArgvFromUserFormat (#7528)

Specifically, the key passed to the module aof_rewrite callback is a stack allocated robj. When passing it to RedisModule_EmitAOF (with appropriate "s" fmt string) redis used to panic when trying to inc the ref count of the stack allocated robj. Now support such robjs by coying them to a new heap robj. This doesn't affect performance because using the alternative "c" or "b" format strings also copies the input to a new heap robj.
---
 src/module.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/module.c b/src/module.c
index acfaf2ac7..9316e004d 100644
--- a/src/module.c
+++ b/src/module.c
@@ -3189,8 +3189,11 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int
             argv[argc++] = createStringObject(cstr,strlen(cstr));
         } else if (*p == 's') {
             robj *obj = va_arg(ap,void*);
+            if (obj->refcount == OBJ_STATIC_REFCOUNT)
+                obj = createStringObject(obj->ptr,sdslen(obj->ptr));
+            else
+                incrRefCount(obj);
             argv[argc++] = obj;
-            incrRefCount(obj);
         } else if (*p == 'b') {
             char *buf = va_arg(ap,char*);
             size_t len = va_arg(ap,size_t);

From a25df9dee0455ce3e072e6dd729736504bdad232 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Thu, 16 Jul 2020 21:01:27 +0300
Subject: [PATCH 0492/1098] Adds GitHub issue templates (#7468)

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: yoav-steinberg <yoav@monfort.co.il>
---
 .github/ISSUE_TEMPLATE/bug_report.md      | 24 +++++++++++++++++++++++
 .github/ISSUE_TEMPLATE/crash_report.md    | 20 +++++++++++++++++++
 .github/ISSUE_TEMPLATE/feature_request.md | 24 +++++++++++++++++++++++
 .github/ISSUE_TEMPLATE/other_stuff.md     |  8 ++++++++
 .github/ISSUE_TEMPLATE/question.md        | 21 ++++++++++++++++++++
 5 files changed, 97 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md
 create mode 100644 .github/ISSUE_TEMPLATE/crash_report.md
 create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md
 create mode 100644 .github/ISSUE_TEMPLATE/other_stuff.md
 create mode 100644 .github/ISSUE_TEMPLATE/question.md

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 000000000..00f21a81b
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,24 @@
+---
+name: Bug report
+about: Help us improve Redis by reporting a bug
+title: '[BUG]'
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+
+A short description of the bug.
+
+**To reproduce**
+
+Steps to reproduce the behavior and/or a minimal code sample.
+
+**Expected behavior**
+
+A description of what you expected to happen.
+
+**Additional information**
+
+Any additional information that is relevant to the problem.
diff --git a/.github/ISSUE_TEMPLATE/crash_report.md b/.github/ISSUE_TEMPLATE/crash_report.md
new file mode 100644
index 000000000..c608ccdc1
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/crash_report.md
@@ -0,0 +1,20 @@
+---
+name: Crash report
+about: Submit a crash report
+title: '[CRASH]'
+labels: ''
+assignees: ''
+
+---
+
+**Crash report**
+
+Paste the complete crash log between the quotes below. Please include a few lines from the log preceding the crash report to provide some context.
+
+```
+```
+
+**Aditional information**
+
+1. OS distribution and version
+2. Steps to reproduce (if any)
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 000000000..732dc647f
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,24 @@
+---
+name: Feature request
+about: Suggest a feature for Redis
+title: '[NEW]'
+labels: ''
+assignees: ''
+
+---
+
+**The problem/use-case that the feature addresses**
+
+A description of the problem that the feature will solve, or the use-case with which the feature will be used.
+
+**Description of the feature**
+
+A description of what you want to happen.
+
+**Alternatives you've considered**
+
+Any alternative solutions or features you've considered, including references to existing open and closed feature requests in this repository.
+
+**Additional information**
+
+Any additional information that is relevant to the feature request.
diff --git a/.github/ISSUE_TEMPLATE/other_stuff.md b/.github/ISSUE_TEMPLATE/other_stuff.md
new file mode 100644
index 000000000..e82cf0024
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/other_stuff.md
@@ -0,0 +1,8 @@
+---
+name: Other
+about: Can't find the right issue type? Use this one!
+title: ''
+labels: ''
+assignees: ''
+
+---
diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md
new file mode 100644
index 000000000..6acde0d06
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/question.md
@@ -0,0 +1,21 @@
+---
+name: Question
+about: Ask the Redis developers
+title: '[QUESTION]'
+labels: ''
+assignees: ''
+
+---
+
+Please keep in mind that this issue tracker should be used for reporting bugs or proposing improvements to the Redis server.
+
+Generally, questions about using Redis should be directed to the [community](https://redis.io/community):
+
+* [the mailing list](https://groups.google.com/forum/#!forum/redis-db)
+* [the `redis` tag at StackOverflow](http://stackoverflow.com/questions/tagged/redis)
+* [/r/redis subreddit](http://www.reddit.com/r/redis)
+* [the irc channel #redis](http://webchat.freenode.net/?channels=redis) on freenode
+
+It is also possible that your question was already asked here, so please do a quick issues search before submitting. Lastly, if your question is about one of Redis' [clients](https://redis.io/clients), you may to contact your client's developers for help.
+
+That said, please feel free to replace all this with your question :)
\ No newline at end of file

From 5df0a64d30e7815c0a4a75a80f165fdee0bd1db6 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Thu, 16 Jul 2020 21:31:36 +0300
Subject: [PATCH 0493/1098] Adds SHA256SUM to redis-stable tarball upload

---
 utils/releasetools/02_upload_tarball.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/releasetools/02_upload_tarball.sh b/utils/releasetools/02_upload_tarball.sh
index 6400efad7..ef1e777cc 100755
--- a/utils/releasetools/02_upload_tarball.sh
+++ b/utils/releasetools/02_upload_tarball.sh
@@ -19,4 +19,5 @@ ssh ubuntu@host.redis.io "cd /var/www/download;
                           mv redis-${1} redis-stable;
                           tar cvzf redis-stable.tar.gz redis-stable;
                           rm -rf redis-${1}.tar.gz;
+                          shasum -a 256 redis-stable.tar.gz > redis-stable.tar.gz.SHA256SUM;
                           "

From 2f4e9c3f9f38e196fe4a03232c97782cbf8b3702 Mon Sep 17 00:00:00 2001
From: Scott Brenner <scott@scottbrenner.me>
Date: Sun, 19 Jul 2020 23:22:24 -0700
Subject: [PATCH 0494/1098] GitHub Actions workflows - use latest version of
 actions/checkout (#7534)

---
 .github/workflows/ci.yml    | 10 +++++-----
 .github/workflows/daily.yml | 10 +++++-----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 730eaf0dd..4d6c1c14c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -7,7 +7,7 @@ jobs:
   test-ubuntu-latest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: make
       run: make
     - name: test
@@ -20,21 +20,21 @@ jobs:
   build-ubuntu-old:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: make
       run: make
 
   build-macos-latest:
     runs-on: macos-latest
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: make
       run: make
 
   build-32bit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: make
       run: |
         sudo apt-get update && sudo apt-get install libc6-dev-i386
@@ -43,7 +43,7 @@ jobs:
   build-libc-malloc:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: make
       run: make MALLOC=libc
 
diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 4d54fbc42..0332c698f 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 14400
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: make
       run: make
     - name: test
@@ -30,7 +30,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 14400
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: make
       run: make MALLOC=libc
     - name: test
@@ -48,7 +48,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 14400
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: make
       run: |
         sudo apt-get update && sudo apt-get install libc6-dev-i386
@@ -70,7 +70,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 14400
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: make
       run: |
         make BUILD_TLS=yes
@@ -90,7 +90,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 14400
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: make
       run: make valgrind
     - name: test

From 86fed3fe093cfa5342b1965b89adb4ac987890ba Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Mon, 20 Jul 2020 18:14:27 +0800
Subject: [PATCH 0495/1098] Refactor streamAppendItem() by deleting redundancy
 condition. (#7487)

It will never happen that "lp != NULL && lp_bytes >= server.stream_node_max_bytes".
Assume that "lp != NULL && lp_bytes >= server.stream_node_max_bytes",
we got the following conditions:
a. lp != NULL
b. lp_bytes >= server.stream_node_max_bytes

If server.stream_node_max_bytes is 0, given condition a, condition b is always satisfied
If server.stream_node_max_bytes is not 0, given condition a and condition b, the codes just a
	few lines above set lp to NULL, a controdiction with condition a

So that condition b is recundant. We could delete it safely.
---
 src/t_stream.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index d2c5b45a3..19b20ee56 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -281,7 +281,7 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_
     }
 
     int flags = STREAM_ITEM_FLAG_NONE;
-    if (lp == NULL || lp_bytes >= server.stream_node_max_bytes) {
+    if (lp == NULL) {
         master_id = id;
         streamEncodeID(rax_key,&id);
         /* Create the listpack having the master entry ID and fields. */

From 19f192c3a5a01eead4d7c93cb372680dfb136005 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 20 Jul 2020 13:57:54 +0300
Subject: [PATCH 0496/1098] Run daily CI on PRs to release a branch (#7535)

---
 .github/workflows/daily.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 0332c698f..5b5f3f7d4 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -1,8 +1,10 @@
 name: Daily
 
 on:
-  release:
-    types: [created]
+  pull_request:
+    branches:
+      # any PR to a release branch.
+      - '[0-9].[0-9]'
   schedule:
     - cron: '0 0 * * *'
 

From e4d7de608c9e4602c883a2ab9bc82d9eb1a634ee Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Mon, 20 Jul 2020 20:33:06 +0800
Subject: [PATCH 0497/1098] Fix cluster redirect for module command with no
 firstkey. (#7539)

Before this commit, processCommand() did not notice that cmd could be a module command
which declared `getkeys-api` and handled it for the purpose of cluster redirect it
as if it doesn't use any keys.

This commit fixed it by reusing the codes in addReplyCommand().
---
 src/server.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/server.c b/src/server.c
index 9c2126bc0..eee3391d9 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3430,6 +3430,12 @@ void rejectCommandFormat(client *c, const char *fmt, ...) {
     sdsfree(s);
 }
 
+/* A command that uses keys but has no pre-determined key position arguments. */
+static int cmdHasMovableKeys(struct redisCommand *cmd) {
+    return (cmd->getkeys_proc && !(cmd->flags & CMD_MODULE)) ||
+            cmd->flags & CMD_MODULE_GETKEYS;
+}
+
 /* If this function gets called we already read a whole
  * command, arguments are in the client argv/argc fields.
  * processCommand() execute the command or prepare the
@@ -3518,7 +3524,7 @@ int processCommand(client *c) {
         !(c->flags & CLIENT_MASTER) &&
         !(c->flags & CLIENT_LUA &&
           server.lua_caller->flags & CLIENT_MASTER) &&
-        !(c->cmd->getkeys_proc == NULL && c->cmd->firstkey == 0 &&
+        !(!cmdHasMovableKeys(c->cmd) && c->cmd->firstkey == 0 &&
           c->cmd->proc != execCommand))
     {
         int hashslot;
@@ -3901,9 +3907,7 @@ void addReplyCommand(client *c, struct redisCommand *cmd) {
         flagcount += addReplyCommandFlag(c,cmd,CMD_ASKING, "asking");
         flagcount += addReplyCommandFlag(c,cmd,CMD_FAST, "fast");
         flagcount += addReplyCommandFlag(c,cmd,CMD_NO_AUTH, "no_auth");
-        if ((cmd->getkeys_proc && !(cmd->flags & CMD_MODULE)) ||
-            cmd->flags & CMD_MODULE_GETKEYS)
-        {
+        if (cmdHasMovableKeys(cmd)) {
             addReplyStatus(c, "movablekeys");
             flagcount += 1;
         }

From 13e50935a84c319763a0d3b2be6ce64962092541 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <276441700@qq.com>
Date: Mon, 20 Jul 2020 22:21:55 +0800
Subject: [PATCH 0498/1098] replication: need handle -NOPERM error after send
 ping (#7538)

---
 src/replication.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/replication.c b/src/replication.c
index 8457150a0..197acde79 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2158,6 +2158,7 @@ void syncWithMaster(connection *conn) {
          * both. */
         if (err[0] != '+' &&
             strncmp(err,"-NOAUTH",7) != 0 &&
+            strncmp(err,"-NOPERM",7) != 0 &&
             strncmp(err,"-ERR operation not permitted",28) != 0)
         {
             serverLog(LL_WARNING,"Error reply to PING from master: '%s'",err);

From 2fbd0271f6fc0a93b9323b6478ec1a7a4ca77614 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Mon, 20 Jul 2020 21:53:03 -0400
Subject: [PATCH 0499/1098] add missing caching command in client help (#7399)

---
 src/networking.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/networking.c b/src/networking.c
index 589a459d4..e3b62f151 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2147,6 +2147,7 @@ void clientCommand(client *c) {
 "SETNAME <name>         -- Assign the name <name> to the current connection.",
 "UNBLOCK <clientid> [TIMEOUT|ERROR] -- Unblock the specified blocked client.",
 "TRACKING (on|off) [REDIRECT <id>] [BCAST] [PREFIX first] [PREFIX second] [OPTIN] [OPTOUT]... -- Enable client keys tracking for client side caching.",
+"CACHING  (yes|no)      -- Enable/Disable tracking of the keys for next command in OPTIN/OPTOUT mode.",
 "GETREDIR               -- Return the client ID we are redirecting to when tracking is enabled.",
 NULL
         };

From 4e8f2d6881a38397bfbf0d7d161959163a5f6e88 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Tue, 21 Jul 2020 01:13:05 -0400
Subject: [PATCH 0500/1098] Add missing calls to raxStop (#7532)

Since the dynamic allocations in raxIterator are only used for deep walks, memory
leak due to missing call to raxStop can only happen for rax with key names longer
than 32 bytes.

Out of all the missing calls, the only ones that may lead to a leak are the rax
for consumer groups and consumers, and these were only in AOFRW and rdbSave, which
normally only happen in fork or at shutdown.
---
 src/aof.c     | 19 +++++++++++-----
 src/defrag.c  |  1 +
 src/rdb.c     | 61 ++++++++++++++++++++++++++++++++++++++++-----------
 src/timeout.c |  1 +
 4 files changed, 63 insertions(+), 19 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index 6f8e53712..cbc0989d0 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1244,12 +1244,16 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) {
         while(raxNext(&ri)) {
             streamCG *group = ri.data;
             /* Emit the XGROUP CREATE in order to create the group. */
-            if (rioWriteBulkCount(r,'*',5) == 0) return 0;
-            if (rioWriteBulkString(r,"XGROUP",6) == 0) return 0;
-            if (rioWriteBulkString(r,"CREATE",6) == 0) return 0;
-            if (rioWriteBulkObject(r,key) == 0) return 0;
-            if (rioWriteBulkString(r,(char*)ri.key,ri.key_len) == 0) return 0;
-            if (rioWriteBulkStreamID(r,&group->last_id) == 0) return 0;
+            if (!rioWriteBulkCount(r,'*',5) ||
+                !rioWriteBulkString(r,"XGROUP",6) ||
+                !rioWriteBulkString(r,"CREATE",6) ||
+                !rioWriteBulkObject(r,key) ||
+                !rioWriteBulkString(r,(char*)ri.key,ri.key_len) ||
+                !rioWriteBulkStreamID(r,&group->last_id)) 
+            {
+                raxStop(&ri);
+                return 0;
+            }
 
             /* Generate XCLAIMs for each consumer that happens to
              * have pending entries. Empty consumers have no semantical
@@ -1270,6 +1274,9 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) {
                                                    ri.key_len,consumer,
                                                    ri_pel.key,nack) == 0)
                     {
+                        raxStop(&ri_pel);
+                        raxStop(&ri_cons);
+                        raxStop(&ri);
                         return 0;
                     }
                 }
diff --git a/src/defrag.c b/src/defrag.c
index 2d8db8ea5..07a16ca6c 100644
--- a/src/defrag.c
+++ b/src/defrag.c
@@ -662,6 +662,7 @@ int scanLaterStraemListpacks(robj *ob, unsigned long *cursor, long long endtime,
         /* if cursor is non-zero, we seek to the static 'last' */
         if (!raxSeek(&ri,">", last, sizeof(last))) {
             *cursor = 0;
+            raxStop(&ri);
             return 0;
         }
         /* assign the iterator node callback after the seek, so that the
diff --git a/src/rdb.c b/src/rdb.c
index 5cec208c5..ac1985d24 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -697,15 +697,23 @@ ssize_t rdbSaveStreamPEL(rio *rdb, rax *pel, int nacks) {
     while(raxNext(&ri)) {
         /* We store IDs in raw form as 128 big big endian numbers, like
          * they are inside the radix tree key. */
-        if ((n = rdbWriteRaw(rdb,ri.key,sizeof(streamID))) == -1) return -1;
+        if ((n = rdbWriteRaw(rdb,ri.key,sizeof(streamID))) == -1) {
+            raxStop(&ri);
+            return -1;
+        }
         nwritten += n;
 
         if (nacks) {
             streamNACK *nack = ri.data;
-            if ((n = rdbSaveMillisecondTime(rdb,nack->delivery_time)) == -1)
+            if ((n = rdbSaveMillisecondTime(rdb,nack->delivery_time)) == -1) {
+                raxStop(&ri);
                 return -1;
+            }
             nwritten += n;
-            if ((n = rdbSaveLen(rdb,nack->delivery_count)) == -1) return -1;
+            if ((n = rdbSaveLen(rdb,nack->delivery_count)) == -1) {
+                raxStop(&ri);
+                return -1;
+            }
             nwritten += n;
             /* We don't save the consumer name: we'll save the pending IDs
              * for each consumer in the consumer PEL, and resolve the consumer
@@ -734,20 +742,27 @@ size_t rdbSaveStreamConsumers(rio *rdb, streamCG *cg) {
         streamConsumer *consumer = ri.data;
 
         /* Consumer name. */
-        if ((n = rdbSaveRawString(rdb,ri.key,ri.key_len)) == -1) return -1;
+        if ((n = rdbSaveRawString(rdb,ri.key,ri.key_len)) == -1) {
+            raxStop(&ri);
+            return -1;
+        }
         nwritten += n;
 
         /* Last seen time. */
-        if ((n = rdbSaveMillisecondTime(rdb,consumer->seen_time)) == -1)
+        if ((n = rdbSaveMillisecondTime(rdb,consumer->seen_time)) == -1) {
+            raxStop(&ri);
             return -1;
+        }
         nwritten += n;
 
         /* Consumer PEL, without the ACKs (see last parameter of the function
          * passed with value of 0), at loading time we'll lookup the ID
          * in the consumer group global PEL and will put a reference in the
          * consumer local PEL. */
-        if ((n = rdbSaveStreamPEL(rdb,consumer->pel,0)) == -1)
+        if ((n = rdbSaveStreamPEL(rdb,consumer->pel,0)) == -1) {
+            raxStop(&ri);
             return -1;
+        }
         nwritten += n;
     }
     raxStop(&ri);
@@ -912,9 +927,15 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key) {
         while (raxNext(&ri)) {
             unsigned char *lp = ri.data;
             size_t lp_bytes = lpBytes(lp);
-            if ((n = rdbSaveRawString(rdb,ri.key,ri.key_len)) == -1) return -1;
+            if ((n = rdbSaveRawString(rdb,ri.key,ri.key_len)) == -1) {
+                raxStop(&ri);
+                return -1;
+            }
             nwritten += n;
-            if ((n = rdbSaveRawString(rdb,lp,lp_bytes)) == -1) return -1;
+            if ((n = rdbSaveRawString(rdb,lp,lp_bytes)) == -1) {
+                raxStop(&ri);
+                return -1;
+            }
             nwritten += n;
         }
         raxStop(&ri);
@@ -946,22 +967,36 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key) {
                 streamCG *cg = ri.data;
 
                 /* Save the group name. */
-                if ((n = rdbSaveRawString(rdb,ri.key,ri.key_len)) == -1)
+                if ((n = rdbSaveRawString(rdb,ri.key,ri.key_len)) == -1) {
+                    raxStop(&ri);
                     return -1;
+                }
                 nwritten += n;
 
                 /* Last ID. */
-                if ((n = rdbSaveLen(rdb,cg->last_id.ms)) == -1) return -1;
+                if ((n = rdbSaveLen(rdb,cg->last_id.ms)) == -1) {
+                    raxStop(&ri);
+                    return -1;
+                }
                 nwritten += n;
-                if ((n = rdbSaveLen(rdb,cg->last_id.seq)) == -1) return -1;
+                if ((n = rdbSaveLen(rdb,cg->last_id.seq)) == -1) {
+                    raxStop(&ri);
+                    return -1;
+                }
                 nwritten += n;
 
                 /* Save the global PEL. */
-                if ((n = rdbSaveStreamPEL(rdb,cg->pel,1)) == -1) return -1;
+                if ((n = rdbSaveStreamPEL(rdb,cg->pel,1)) == -1) {
+                    raxStop(&ri);
+                    return -1;
+                }
                 nwritten += n;
 
                 /* Save the consumers of this group. */
-                if ((n = rdbSaveStreamConsumers(rdb,cg)) == -1) return -1;
+                if ((n = rdbSaveStreamConsumers(rdb,cg)) == -1) {
+                    raxStop(&ri);
+                    return -1;
+                }
                 nwritten += n;
             }
             raxStop(&ri);
diff --git a/src/timeout.c b/src/timeout.c
index 7787a049f..d4c4690e5 100644
--- a/src/timeout.c
+++ b/src/timeout.c
@@ -150,6 +150,7 @@ void handleBlockedClientsTimeout(void) {
         raxRemove(server.clients_timeout_table,ri.key,ri.key_len,NULL);
         raxSeek(&ri,"^",NULL,0);
     }
+    raxStop(&ri);
 }
 
 /* Get a timeout value from an object and store it into 'timeout'.

From 3f2fbc4c614ff718dce7d55fd971d7ed36062c24 Mon Sep 17 00:00:00 2001
From: Remi Collet <remi@remirepo.net>
Date: Tue, 21 Jul 2020 08:07:54 +0200
Subject: [PATCH 0501/1098] Fix deprecated tail syntax in tests (#7543)

---
 tests/support/util.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index fce3ffd18..69ea675dc 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -119,7 +119,7 @@ proc wait_for_log_message {srv_idx pattern from_line maxtries delay} {
     set retry $maxtries
     set stdout [srv $srv_idx stdout]
     while {$retry} {
-        set result [exec tail +$from_line < $stdout]
+        set result [exec tail -n +$from_line < $stdout]
         set result [split $result "\n"]
         foreach line $result {
             if {[string match $pattern $line]} {

From f7f77a746ab458566654dc159925528daaaf80da Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Tue, 21 Jul 2020 14:41:44 +0800
Subject: [PATCH 0502/1098] Clarification on the bug that was fixed in PR
 #7539. (#7541)

Before that PR, processCommand() did not notice that cmd could be a module
command in which case getkeys_proc member has a different meaning.

The outcome was that a module command which doesn't take any key names in its
arguments (similar to SLOWLOG) would be handled as if it might have key name arguments
(similar to MEMORY), would consider cluster redirect but will end up with 0 keys
after an excessive call to getKeysFromCommand, and eventually do the right thing.
---
 src/server.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index eee3391d9..d984a7edc 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3430,7 +3430,8 @@ void rejectCommandFormat(client *c, const char *fmt, ...) {
     sdsfree(s);
 }
 
-/* A command that uses keys but has no pre-determined key position arguments. */
+/* Returns 1 for commands that may have key names in their arguments, but have
+ * no pre-determined key positions. */
 static int cmdHasMovableKeys(struct redisCommand *cmd) {
     return (cmd->getkeys_proc && !(cmd->flags & CMD_MODULE)) ||
             cmd->flags & CMD_MODULE_GETKEYS;

From 343dd9bcce0ce28d131510e28bac411599da90b0 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 21 Jul 2020 14:07:06 +0300
Subject: [PATCH 0503/1098] Fixes to release scripts (#7547)

---
 utils/releasetools/03_test_release.sh | 1 +
 utils/releasetools/04_release_hash.sh | 3 +--
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/utils/releasetools/03_test_release.sh b/utils/releasetools/03_test_release.sh
index 169e965d5..493d0b74c 100755
--- a/utils/releasetools/03_test_release.sh
+++ b/utils/releasetools/03_test_release.sh
@@ -14,6 +14,7 @@ echo "Doing sanity test on the actual tarball"
 
 cd /tmp
 rm -rf test_release_tmp_dir
+mkdir test_release_tmp_dir
 cd test_release_tmp_dir
 rm -f $TARNAME
 rm -rf redis-${TAG}
diff --git a/utils/releasetools/04_release_hash.sh b/utils/releasetools/04_release_hash.sh
index bc1ebb66c..d93292803 100755
--- a/utils/releasetools/04_release_hash.sh
+++ b/utils/releasetools/04_release_hash.sh
@@ -7,8 +7,7 @@ fi
 
 SHA=$(curl -s http://download.redis.io/releases/redis-${1}.tar.gz | shasum -a 256 | cut -f 1 -d' ')
 ENTRY="hash redis-${1}.tar.gz sha256 $SHA http://download.redis.io/releases/redis-${1}.tar.gz"
-echo $ENTRY >> ~/hack/redis-hashes/README
-vi ../redis-hashes/README
+echo $ENTRY >> ../redis-hashes/README
 echo "Press any key to commit, Ctrl-C to abort)."
 read yes
 (cd ../redis-hashes; git commit -a -m "${1} hash."; git push)

From f57e844b2edbb86a5df2f3436045814812c0a3ae Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@users.noreply.github.com>
Date: Tue, 21 Jul 2020 14:17:14 +0300
Subject: [PATCH 0504/1098] Tests: drop TCL 8.6 dependency. (#7548)

This re-implements the redis-cli --pipe test so it no longer depends on a close feature available only in TCL 8.6.

Basically what this test does is run redis-cli --pipe, generates a bunch of commands and pipes them through redis-cli, and inspects the result in both Redis and the redis-cli output.

To do that, we need to close stdin for redis-cli to indicate we're done so it can flush its buffers and exit. TCL has bi-directional channels can only offers a way to "one-way close" a channel with TCL 8.6. To work around that, we now generate the commands into a file and feed that file to redis-cli directly.

As we're writing to an actual file, the number of commands is now reduced.
---
 tests/integration/redis-cli.tcl | 51 ++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 23 deletions(-)

diff --git a/tests/integration/redis-cli.tcl b/tests/integration/redis-cli.tcl
index c70d14eeb..44ff430e2 100644
--- a/tests/integration/redis-cli.tcl
+++ b/tests/integration/redis-cli.tcl
@@ -1,10 +1,16 @@
 source tests/support/cli.tcl
 
 start_server {tags {"cli"}} {
-    proc open_cli {{opts "-n 9"}} {
+    proc open_cli {{opts "-n 9"} {infile ""}} {
         set ::env(TERM) dumb
         set cmdline [rediscli [srv port] $opts]
-        set fd [open "|$cmdline" "r+"]
+        if {$infile ne ""} {
+            set cmdline "$cmdline < $infile"
+            set mode "r"
+        } else {
+            set mode "r+"
+        }
+        set fd [open "|$cmdline" $mode]
         fconfigure $fd -buffering none
         fconfigure $fd -blocking false
         fconfigure $fd -translation binary
@@ -228,7 +234,7 @@ start_server {tags {"cli"}} {
         assert_equal {} [r get should-not-exist]
     }
 
-    test_nontty_cli "Dumping an RDB" {
+    test "Dumping an RDB" {
         # Disk-based master
         assert_match "OK" [r config set repl-diskless-sync no]
         test_redis_cli_rdb_dump
@@ -239,7 +245,7 @@ start_server {tags {"cli"}} {
         test_redis_cli_rdb_dump
     }
 
-    test_nontty_cli "Connecting as a replica" {
+    test "Connecting as a replica" {
         set fd [open_cli "--replica"]
         wait_for_condition 500 500 {
             [string match {*slave0:*state=online*} [r info]]
@@ -258,31 +264,30 @@ start_server {tags {"cli"}} {
         close_cli $fd
     }
 
-    test_nontty_cli "Piping raw protocol" {
-        set fd [open_cli "--pipe"]
-        fconfigure $fd -blocking true
+    test "Piping raw protocol" {
+        set cmds [tmpfile "cli_cmds"]
+        set cmds_fd [open $cmds "w"]
 
-        # Create a new deferring client and overwrite its fd
-        set client [redis [srv 0 "host"] [srv 0 "port"] 1 0]
-        set ::redis::fd($::redis::id) $fd
-        $client select 9
-
-        r del test-counter
-        for {set i 0} {$i < 10000} {incr i} {
-            $client incr test-counter
-            $client set large-key [string repeat "x" 20000]
-        }
+        puts $cmds_fd [formatCommand select 9]
+        puts $cmds_fd [formatCommand del test-counter]
 
         for {set i 0} {$i < 1000} {incr i} {
-            $client set very-large-key [string repeat "x" 512000]
+            puts $cmds_fd [formatCommand incr test-counter]
+            puts $cmds_fd [formatCommand set large-key [string repeat "x" 20000]]
         }
 
-        close $fd write
-        set output [read_cli $fd]
+        for {set i 0} {$i < 100} {incr i} {
+            puts $cmds_fd [formatCommand set very-large-key [string repeat "x" 512000]]
+        }
+        close $cmds_fd
 
-        assert_equal {10000} [r get test-counter]
-        assert_match {*All data transferred*errors: 0*replies: 21001*} $output
+        set cli_fd [open_cli "--pipe" $cmds]
+        fconfigure $cli_fd -blocking true
+        set output [read_cli $cli_fd]
 
-        close_cli $fd
+        assert_equal {1000} [r get test-counter]
+        assert_match {*All data transferred*errors: 0*replies: 2102*} $output
+
+        file delete $cmds
     }
 }

From 36b949438547eb5bf8555fcac2c5040528fd7854 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 21 Jul 2020 16:56:19 +0300
Subject: [PATCH 0505/1098] testsuite may leave servers alive on error (#7549)

in cases where you have
test name {
  start_server {
    start_server {
      assert
    }
  }
}

the exception will be thrown to the test proc, and the servers are
supposed to be killed on the way out. but it seems there was always a
bug of not cleaning the server stack, and recently (#7404) we started
relying on that stack in order to kill them, so with that bug sometimes
we would have tried to kill the same server twice, and leave one alive.

luckly, in most cases the pattern is:
start_server {
  test name {
  }
}
---
 tests/support/server.tcl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index ea7d0b13c..0afe89f7c 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -396,6 +396,9 @@ proc start_server {options {code undefined}} {
             # fetch srv back from the server list, in case it was restarted by restart_server (new PID)
             set srv [lindex $::servers end]
 
+            # pop the server object
+            set ::servers [lrange $::servers 0 end-1]
+
             # Kill the server without checking for leaks
             dict set srv "skipleaks" 1
             kill_server $srv

From 818dc3a0898125d621288df671994eed34f849f2 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Tue, 21 Jul 2020 17:00:13 -0700
Subject: [PATCH 0506/1098] Properly reset errno for rdbLoad (#7542)

---
 src/server.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/server.c b/src/server.c
index d984a7edc..28abf5f50 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4881,6 +4881,7 @@ void loadDataFromDisk(void) {
             serverLog(LL_NOTICE,"DB loaded from append only file: %.3f seconds",(float)(ustime()-start)/1000000);
     } else {
         rdbSaveInfo rsi = RDB_SAVE_INFO_INIT;
+        errno = 0; /* Prevent a stale value from affecting error checking */
         if (rdbLoad(server.rdb_filename,&rsi,RDBFLAGS_NONE) == C_OK) {
             serverLog(LL_NOTICE,"DB loaded from disk: %.3f seconds",
                 (float)(ustime()-start)/1000000);

From 9fc8a5efe5541a4461cd889340fda46184a4b0f2 Mon Sep 17 00:00:00 2001
From: Sungho Hwang <sgc109@naver.com>
Date: Wed, 22 Jul 2020 17:33:57 +0900
Subject: [PATCH 0507/1098] Fix typo in deps/README.md (#7553)

---
 deps/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deps/README.md b/deps/README.md
index f923c06ad..a7f6e1834 100644
--- a/deps/README.md
+++ b/deps/README.md
@@ -17,7 +17,7 @@ active defragmentation logic. However this feature of Redis is not mandatory
 and Redis is able to understand if the Jemalloc version it is compiled
 against supports such Redis-specific modifications. So in theory, if you
 are not interested in the active defragmentation, you can replace Jemalloc
-just following tose steps:
+just following these steps:
 
 1. Remove the jemalloc directory.
 2. Substitute it with the new jemalloc source tree.

From 3e35ac9d7f2d2fc326443061c3323c0538a85ed0 Mon Sep 17 00:00:00 2001
From: Brian P O'Rourke <brian@orourke.io>
Date: Sat, 11 Jul 2020 09:36:28 -0700
Subject: [PATCH 0508/1098] Outdent github issues guidelines

---
 CONTRIBUTING | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/CONTRIBUTING b/CONTRIBUTING
index 000edbeaf..800d7bd21 100644
--- a/CONTRIBUTING
+++ b/CONTRIBUTING
@@ -7,18 +7,18 @@ each source file that you contribute.
 
 # IMPORTANT: HOW TO USE REDIS GITHUB ISSUES
 
-* Github issues SHOULD ONLY BE USED to report bugs, and for DETAILED feature
-  requests. Everything else belongs to the Redis Google Group:
+Github issues SHOULD ONLY BE USED to report bugs, and for DETAILED feature
+requests. Everything else belongs to the Redis Google Group:
       
-      https://groups.google.com/forum/m/#!forum/Redis-db
+    https://groups.google.com/forum/m/#!forum/Redis-db
 
-  PLEASE DO NOT POST GENERAL QUESTIONS that are not about bugs or suspected
-  bugs in the Github issues system. We'll be very happy to help you and provide
-  all the support in the mailing list.
+PLEASE DO NOT POST GENERAL QUESTIONS that are not about bugs or suspected
+bugs in the Github issues system. We'll be very happy to help you and provide
+all the support in the mailing list.
 
-  There is also an active community of Redis users at Stack Overflow:
+There is also an active community of Redis users at Stack Overflow:
 
-      http://stackoverflow.com/questions/tagged/redis
+    http://stackoverflow.com/questions/tagged/redis
 
 # How to provide a patch for a new feature
 

From 99e6e732352c483bcbd9efae7aaf0560f3e88173 Mon Sep 17 00:00:00 2001
From: Brian P O'Rourke <brian@orourke.io>
Date: Sat, 11 Jul 2020 09:37:41 -0700
Subject: [PATCH 0509/1098] Add contribution guidelines for vulnerability
 reports

---
 CONTRIBUTING | 20 ++++++++++++++++++++
 README.md    |  2 +-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/CONTRIBUTING b/CONTRIBUTING
index 800d7bd21..394fe02ad 100644
--- a/CONTRIBUTING
+++ b/CONTRIBUTING
@@ -20,6 +20,26 @@ There is also an active community of Redis users at Stack Overflow:
 
     http://stackoverflow.com/questions/tagged/redis
 
+# Reporting Security Bugs
+
+*If you are reporting a security bug*, please contact the core team privately
+by emailing redis@redis.io. Your report will be acknowledged by a core team
+member and once the report has been reviewed you will receive a more detailed
+response including next steps.
+
+If you do not receive a reply you can escalate to the Redis Google Group,
+linked above. Because this group is a public space please do not disclose the
+issue in detail, only say that you are trying to reach the core team for a
+security issue.
+
+Redis follows a responsible disclosure process:
+
+1. Reports are reviewed and analyzed privately
+2. Patches are prepared for supported versions of Redis
+3. Vendor lists are notified with an embargo date to reduce the public impact
+4. We push a fix release and your bug can be posted publicly with credit in
+   release notes and the version history (and our thanks!)
+
 # How to provide a patch for a new feature
 
 1. If it is a major feature or a semantical change, please don't start coding
diff --git a/README.md b/README.md
index 55537e01f..a3c5def9f 100644
--- a/README.md
+++ b/README.md
@@ -203,7 +203,7 @@ of the BSD license that you can find in the [COPYING][1] file included in the Re
 source distribution.
 
 Please see the [CONTRIBUTING][2] file in this source distribution for more
-information.
+information, including details on our process for security bugs/vulnerabilities.
 
 [1]: https://github.com/redis/redis/blob/unstable/COPYING
 [2]: https://github.com/redis/redis/blob/unstable/CONTRIBUTING

From 40d7fca3685d8439bae8480ddbd59775a2390411 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 23 Jul 2020 12:37:43 +0300
Subject: [PATCH 0510/1098] Fix harmless bug in rioConnRead (#7557)

this code is in use only if the master is disk-based, and the replica is
diskless. In this case we use a buffered reader, but we must avoid reading
past the rdb file, into the command stream. which Luckly rdb.c doesn't
really attempt to do (it knows how much it should read).

When rioConnRead detects that the extra buffering attempt reaches beyond
the read limit it should read less, but if the caller actually requested
more, then it should return with an error rather than a short read. the
bug would have resulted in short read.

in order to fix it, the code must consider the real requested size, and
not the extra buffering size.
---
 src/rio.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/rio.c b/src/rio.c
index c8c924380..77ba69fdc 100644
--- a/src/rio.c
+++ b/src/rio.c
@@ -190,15 +190,18 @@ static size_t rioConnRead(rio *r, void *buf, size_t len) {
     /* If we don't already have all the data in the sds, read more */
     while (len > sdslen(r->io.conn.buf) - r->io.conn.pos) {
         size_t buffered = sdslen(r->io.conn.buf) - r->io.conn.pos;
-        size_t toread = len - buffered;
+        size_t needs = len - buffered;
         /* Read either what's missing, or PROTO_IOBUF_LEN, the bigger of
          * the two. */
-        if (toread < PROTO_IOBUF_LEN) toread = PROTO_IOBUF_LEN;
+        size_t toread = needs < PROTO_IOBUF_LEN ? PROTO_IOBUF_LEN: needs;
         if (toread > sdsavail(r->io.conn.buf)) toread = sdsavail(r->io.conn.buf);
         if (r->io.conn.read_limit != 0 &&
             r->io.conn.read_so_far + buffered + toread > r->io.conn.read_limit)
         {
-            if (r->io.conn.read_limit >= r->io.conn.read_so_far - buffered)
+            /* Make sure the caller didn't request to read past the limit.
+             * If they didn't we'll buffer till the limit, if they did, we'll
+             * return an error. */
+            if (r->io.conn.read_limit >= r->io.conn.read_so_far + needs)
                 toread = r->io.conn.read_limit - r->io.conn.read_so_far - buffered;
             else {
                 errno = EOVERFLOW;

From 8d826393191399e132bd9e56fb51ed83223cc5ca Mon Sep 17 00:00:00 2001
From: "Meir Shpilraien (Spielrein)" <meir@redislabs.com>
Date: Thu, 23 Jul 2020 12:38:51 +0300
Subject: [PATCH 0511/1098] This PR introduces a new loaded keyspace event
 (#7536)

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: Itamar Haber <itamar@redislabs.com>
---
 runtest-moduleapi                        |  1 +
 src/module.c                             |  5 ++
 src/rdb.c                                |  6 +-
 src/redismodule.h                        |  2 +-
 src/server.h                             |  1 +
 tests/modules/Makefile                   |  3 +-
 tests/modules/keyspace_events.c          | 99 ++++++++++++++++++++++++
 tests/unit/moduleapi/keyspace_events.tcl | 22 ++++++
 8 files changed, 135 insertions(+), 4 deletions(-)
 create mode 100644 tests/modules/keyspace_events.c
 create mode 100644 tests/unit/moduleapi/keyspace_events.tcl

diff --git a/runtest-moduleapi b/runtest-moduleapi
index f6cc0a258..71db27e5e 100755
--- a/runtest-moduleapi
+++ b/runtest-moduleapi
@@ -25,4 +25,5 @@ $TCLSH tests/test_helper.tcl \
 --single unit/moduleapi/scan \
 --single unit/moduleapi/datatype \
 --single unit/moduleapi/auth \
+--single unit/moduleapi/keyspace_events \
 "${@}"
diff --git a/src/module.c b/src/module.c
index 9316e004d..0ccd5bbaa 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4850,6 +4850,11 @@ void moduleReleaseGIL(void) {
  *  - REDISMODULE_NOTIFY_STREAM: Stream events
  *  - REDISMODULE_NOTIFY_KEYMISS: Key-miss events
  *  - REDISMODULE_NOTIFY_ALL: All events (Excluding REDISMODULE_NOTIFY_KEYMISS)
+ *  - REDISMODULE_NOTIFY_LOADED: A special notification available only for modules,
+ *                               indicates that the key was loaded from persistence.
+ *                               Notice, when this event fires, the given key
+ *                               can not be retained, use RM_CreateStringFromString
+ *                               instead.
  *
  * We do not distinguish between key events and keyspace events, and it is up
  * to the module to filter the actions taken based on the key.
diff --git a/src/rdb.c b/src/rdb.c
index ac1985d24..54a169cd8 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2307,6 +2307,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
             decrRefCount(val);
         } else {
             robj keyobj;
+            initStaticStringObject(keyobj,key);
 
             /* Add the new object in the hash table */
             int added = dbAddRDBLoad(db,key,val);
@@ -2315,7 +2316,6 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
                     /* This flag is useful for DEBUG RELOAD special modes.
                      * When it's set we allow new keys to replace the current
                      * keys with the same name. */
-                    initStaticStringObject(keyobj,key);
                     dbSyncDelete(db,&keyobj);
                     dbAddRDBLoad(db,key,val);
                 } else {
@@ -2327,12 +2327,14 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
 
             /* Set the expire time if needed */
             if (expiretime != -1) {
-                initStaticStringObject(keyobj,key);
                 setExpire(NULL,db,&keyobj,expiretime);
             }
 
             /* Set usage information (for eviction). */
             objectSetLRUOrLFU(val,lfu_freq,lru_idle,lru_clock,1000);
+
+            /* call key space notification on key loaded for modules only */
+            moduleNotifyKeyspaceEvent(NOTIFY_LOADED, "loaded", &keyobj, db->id);
         }
 
         /* Loading the database more slowly is useful in order to test
diff --git a/src/redismodule.h b/src/redismodule.h
index 23b4d26e0..2599d3129 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -128,9 +128,9 @@
 #define REDISMODULE_NOTIFY_EVICTED (1<<9)     /* e */
 #define REDISMODULE_NOTIFY_STREAM (1<<10)     /* t */
 #define REDISMODULE_NOTIFY_KEY_MISS (1<<11)   /* m (Note: This one is excluded from REDISMODULE_NOTIFY_ALL on purpose) */
+#define REDISMODULE_NOTIFY_LOADED (1<<12)     /* module only key space notification, indicate a key loaded from rdb */
 #define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM)      /* A */
 
-
 /* A special pointer that we can use between the core and the module to signal
  * field deletion, and that is impossible to be a valid pointer. */
 #define REDISMODULE_HASH_DELETE ((RedisModuleString*)(long)1)
diff --git a/src/server.h b/src/server.h
index 3f471efcb..1ca99b5ea 100644
--- a/src/server.h
+++ b/src/server.h
@@ -426,6 +426,7 @@ typedef long long ustime_t; /* microsecond time type. */
 #define NOTIFY_EVICTED (1<<9)     /* e */
 #define NOTIFY_STREAM (1<<10)     /* t */
 #define NOTIFY_KEY_MISS (1<<11)   /* m (Note: This one is excluded from NOTIFY_ALL on purpose) */
+#define NOTIFY_LOADED (1<<12)     /* module only key space notification, indicate a key loaded from rdb */
 #define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED | NOTIFY_STREAM) /* A flag */
 
 /* Get the first bind addr or NULL */
diff --git a/tests/modules/Makefile b/tests/modules/Makefile
index 39b8e6efa..de7407a84 100644
--- a/tests/modules/Makefile
+++ b/tests/modules/Makefile
@@ -22,7 +22,8 @@ TEST_MODULES = \
     blockonkeys.so \
     scan.so \
     datatype.so \
-    auth.so
+    auth.so \
+    keyspace_events.so
 
 .PHONY: all
 
diff --git a/tests/modules/keyspace_events.c b/tests/modules/keyspace_events.c
new file mode 100644
index 000000000..b2296c1cb
--- /dev/null
+++ b/tests/modules/keyspace_events.c
@@ -0,0 +1,99 @@
+/* This module is used to test the server keyspace events API.
+ *
+ * -----------------------------------------------------------------------------
+ *
+ * Copyright (c) 2020, Meir Shpilraien <meir at redislabs dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define REDISMODULE_EXPERIMENTAL_API
+
+#include "redismodule.h"
+#include <stdio.h>
+#include <string.h>
+
+/** strores all the keys on which we got 'loaded' keyspace notification **/
+RedisModuleDict *loaded_event_log = NULL;
+
+static int KeySpace_Notification(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key){
+    REDISMODULE_NOT_USED(ctx);
+    REDISMODULE_NOT_USED(type);
+
+    if(strcmp(event, "loaded") == 0){
+        const char* keyName = RedisModule_StringPtrLen(key, NULL);
+        int nokey;
+        RedisModule_DictGetC(loaded_event_log, (void*)keyName, strlen(keyName), &nokey);
+        if(nokey){
+            RedisModule_DictSetC(loaded_event_log, (void*)keyName, strlen(keyName), NULL);
+        }
+    }
+
+    return REDISMODULE_OK;
+}
+
+static int cmdIsKeyLoaded(RedisModuleCtx *ctx, RedisModuleString **argv, int argc){
+    if(argc != 2){
+        return RedisModule_WrongArity(ctx);
+    }
+
+    const char* key  = RedisModule_StringPtrLen(argv[1], NULL);
+
+    int nokey;
+    RedisModule_DictGetC(loaded_event_log, (void*)key, strlen(key), &nokey);
+
+    RedisModule_ReplyWithLongLong(ctx, !nokey);
+    return REDISMODULE_OK;
+}
+
+/* This function must be present on each Redis module. It is used in order to
+ * register the commands into the Redis server. */
+int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+
+    if (RedisModule_Init(ctx,"testkeyspace",1,REDISMODULE_APIVER_1) == REDISMODULE_ERR){
+        return REDISMODULE_ERR;
+    }
+
+    loaded_event_log = RedisModule_CreateDict(ctx);
+
+    if(RedisModule_SubscribeToKeyspaceEvents(ctx, REDISMODULE_NOTIFY_LOADED, KeySpace_Notification) != REDISMODULE_OK){
+        return REDISMODULE_ERR;
+    }
+
+    if (RedisModule_CreateCommand(ctx,"keyspace.is_key_loaded", cmdIsKeyLoaded,"",0,0,0) == REDISMODULE_ERR){
+        return REDISMODULE_ERR;
+    }
+
+    return REDISMODULE_OK;
+}
+
+int RedisModule_OnUnload(RedisModuleCtx *ctx) {
+    RedisModule_FreeDict(ctx, loaded_event_log);
+    loaded_event_log = NULL;
+    return REDISMODULE_OK;
+}
diff --git a/tests/unit/moduleapi/keyspace_events.tcl b/tests/unit/moduleapi/keyspace_events.tcl
new file mode 100644
index 000000000..cb959ab52
--- /dev/null
+++ b/tests/unit/moduleapi/keyspace_events.tcl
@@ -0,0 +1,22 @@
+set testmodule [file normalize tests/modules/keyspace_events.so]
+
+tags "modules" {
+    start_server [list overrides [list loadmodule "$testmodule"]] {
+
+        test {Test loaded key space event} {
+            r set x 1
+            r hset y f v
+            r lpush z 1 2 3
+            r sadd p 1 2 3
+            r zadd t 1 f1 2 f2
+            r xadd s * f v
+            r debug reload
+            assert_equal 1 [r keyspace.is_key_loaded x]
+            assert_equal 1 [r keyspace.is_key_loaded y]
+            assert_equal 1 [r keyspace.is_key_loaded z]
+            assert_equal 1 [r keyspace.is_key_loaded p]
+            assert_equal 1 [r keyspace.is_key_loaded t]
+            assert_equal 1 [r keyspace.is_key_loaded s]
+        }
+	}
+}
\ No newline at end of file

From 8a57969fd75db01b881d438200911d95bdead293 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 23 Jul 2020 13:06:24 +0300
Subject: [PATCH 0512/1098] Stabilize bgsave test that sometimes fails with
 valgrind (#7559)

on ci.redis.io the test fails a lot, reporting that bgsave didn't end.
increaseing the timeout we wait for that bgsave to get aborted.
in addition to that, i also verify that it indeed got aborted by
checking that the save counter wasn't reset.

add another test to verify that a successful bgsave indeed resets the
change counter.
---
 tests/integration/rdb.tcl | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl
index b176bf199..6535cd089 100644
--- a/tests/integration/rdb.tcl
+++ b/tests/integration/rdb.tcl
@@ -118,16 +118,34 @@ start_server_and_kill_it [list "dir" $server_path] {
 
 start_server {} {
     test {Test FLUSHALL aborts bgsave} {
+        # 1000 keys with 1ms sleep per key shuld take 1 second
         r config set rdb-key-save-delay 1000
         r debug populate 1000
         r bgsave
         assert_equal [s rdb_bgsave_in_progress] 1
         r flushall
-        after 200
-        assert_equal [s rdb_bgsave_in_progress] 0
+        # wait half a second max
+        wait_for_condition 5 100 {
+            [s rdb_bgsave_in_progress] == 0
+        } else {
+            fail "bgsave not aborted"
+        }
+        # veirfy that bgsave failed, by checking that the change counter is still high
+        assert_lessthan 999 [s rdb_changes_since_last_save]
         # make sure the server is still writable
         r set x xx
     }
+
+    test {bgsave resets the change counter} {
+        r config set rdb-key-save-delay 0
+        r bgsave
+        wait_for_condition 5 100 {
+            [s rdb_bgsave_in_progress] == 0
+        } else {
+            fail "bgsave not aborted"
+        }
+        assert_equal [s rdb_changes_since_last_save] 0
+    }
 }
 
 test {client freed during loading} {

From da840e9851bab8d1674e245a812b2105be111208 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <276441700@qq.com>
Date: Fri, 24 Jul 2020 14:40:19 +0800
Subject: [PATCH 0513/1098] more strict check in rioConnRead (#7564)

---
 src/rio.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rio.c b/src/rio.c
index 77ba69fdc..bbef81b68 100644
--- a/src/rio.c
+++ b/src/rio.c
@@ -201,7 +201,7 @@ static size_t rioConnRead(rio *r, void *buf, size_t len) {
             /* Make sure the caller didn't request to read past the limit.
              * If they didn't we'll buffer till the limit, if they did, we'll
              * return an error. */
-            if (r->io.conn.read_limit >= r->io.conn.read_so_far + needs)
+            if (r->io.conn.read_limit >= r->io.conn.read_so_far + len)
                 toread = r->io.conn.read_limit - r->io.conn.read_so_far - buffered;
             else {
                 errno = EOVERFLOW;

From 4126ca466fb9fbf0d84a66e31c9f59f28e2fbff6 Mon Sep 17 00:00:00 2001
From: grishaf <grisha@drivenets.com>
Date: Sun, 26 Jul 2020 08:27:30 +0300
Subject: [PATCH 0514/1098] Fix prepareForShutdown function declaration (#7566)

---
 src/server.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.h b/src/server.h
index 1ca99b5ea..afd020759 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1977,7 +1977,7 @@ void forceCommandPropagation(client *c, int flags);
 void preventCommandPropagation(client *c);
 void preventCommandAOF(client *c);
 void preventCommandReplication(client *c);
-int prepareForShutdown();
+int prepareForShutdown(int flags);
 #ifdef __GNUC__
 void serverLog(int level, const char *fmt, ...)
     __attribute__((format(printf, 2, 3)));

From c75512d89d1697cf782cdc826acffab5b6adc1c7 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 27 Jul 2020 12:52:13 +0300
Subject: [PATCH 0515/1098] TLS: support cluster/replication without tls-port.

Initialize and configure OpenSSL even when tls-port is not used, because
we may still have tls-cluster or tls-replication.

Also, make sure to reconfigure OpenSSL when these parameters are changed
as TLS could have been enabled for the first time.
---
 src/config.c | 6 +++---
 src/server.c | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/config.c b/src/config.c
index acf1b069f..9b6dc459c 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2221,11 +2221,11 @@ standardConfig configs[] = {
     createOffTConfig("auto-aof-rewrite-min-size", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.aof_rewrite_min_size, 64*1024*1024, MEMORY_CONFIG, NULL, NULL),
 
 #ifdef USE_OPENSSL
-    createIntConfig("tls-port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG, NULL, NULL), /* TCP port. */
+    createIntConfig("tls-port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG, NULL, updateTlsCfgInt), /* TCP port. */
     createIntConfig("tls-session-cache-size", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tls_ctx_config.session_cache_size, 20*1024, INTEGER_CONFIG, NULL, updateTlsCfgInt),
     createIntConfig("tls-session-cache-timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tls_ctx_config.session_cache_timeout, 300, INTEGER_CONFIG, NULL, updateTlsCfgInt),
-    createBoolConfig("tls-cluster", NULL, MODIFIABLE_CONFIG, server.tls_cluster, 0, NULL, NULL),
-    createBoolConfig("tls-replication", NULL, MODIFIABLE_CONFIG, server.tls_replication, 0, NULL, NULL),
+    createBoolConfig("tls-cluster", NULL, MODIFIABLE_CONFIG, server.tls_cluster, 0, NULL, updateTlsCfgBool),
+    createBoolConfig("tls-replication", NULL, MODIFIABLE_CONFIG, server.tls_replication, 0, NULL, updateTlsCfgBool),
     createBoolConfig("tls-auth-clients", NULL, MODIFIABLE_CONFIG, server.tls_auth_clients, 1, NULL, NULL),
     createBoolConfig("tls-prefer-server-ciphers", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.prefer_server_ciphers, 0, NULL, updateTlsCfgBool),
     createBoolConfig("tls-session-caching", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.session_caching, 1, NULL, updateTlsCfgBool),
diff --git a/src/server.c b/src/server.c
index 28abf5f50..22a910e1c 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2774,7 +2774,8 @@ void initServer(void) {
     server.events_processed_while_blocked = 0;
     server.system_memory_size = zmalloc_get_memory_size();
 
-    if (server.tls_port && tlsConfigure(&server.tls_ctx_config) == C_ERR) {
+    if ((server.tls_port || server.tls_replication || server.tls_cluster)
+                && tlsConfigure(&server.tls_ctx_config) == C_ERR) {
         serverLog(LL_WARNING, "Failed to configure TLS. Check logs for more info.");
         exit(1);
     }

From 6d92eee69b8e693b9bf0a34320dd108b5a81a81a Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 27 Jul 2020 15:30:36 +0300
Subject: [PATCH 0516/1098] Daily github action: run cluster and sentinel tests
 with tls (#7575)

---
 .github/workflows/daily.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 5b5f3f7d4..acc811d3a 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -84,9 +84,9 @@ jobs:
     - name: module api test
       run: ./runtest-moduleapi --verbose --tls
     - name: sentinel tests
-      run: ./runtest-sentinel
+      run: ./runtest-sentinel --tls
     - name: cluster tests
-      run: ./runtest-cluster
+      run: ./runtest-cluster --tls
 
   test-valgrind:
     runs-on: ubuntu-latest

From f31260b0445f5649449da41555e1272a40ae4af7 Mon Sep 17 00:00:00 2001
From: Jiayuan Chen <mrpre@163.com>
Date: Tue, 28 Jul 2020 15:45:21 +0800
Subject: [PATCH 0517/1098] Add optional tls verification (#7502)

Adds an `optional` value to the previously boolean `tls-auth-clients` configuration keyword.

Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
---
 redis.conf         |  5 ++++-
 src/cluster.c      |  3 ++-
 src/config.c       |  8 +++++++-
 src/server.h       |  5 +++++
 src/tls.c          | 12 ++++++++++--
 tests/unit/tls.tcl | 12 ++++++++++++
 6 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/redis.conf b/redis.conf
index 8c53f015a..d4e3e47f0 100644
--- a/redis.conf
+++ b/redis.conf
@@ -159,9 +159,12 @@ tcp-keepalive 300
 # By default, clients (including replica servers) on a TLS port are required
 # to authenticate using valid client side certificates.
 #
-# It is possible to disable authentication using this directive.
+# If "no" is specified, client certificates are not required and not accepted.
+# If "optional" is specified, client certificates are accepted and must be
+# valid if provided, but are not required.
 #
 # tls-auth-clients no
+# tls-auth-clients optional
 
 # By default, a Redis replica does not attempt to establish a TLS connection
 # with its master.
diff --git a/src/cluster.c b/src/cluster.c
index 5dcd69ff8..485683cf0 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -670,7 +670,8 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
             return;
         }
 
-        connection *conn = server.tls_cluster ? connCreateAcceptedTLS(cfd,1) : connCreateAcceptedSocket(cfd);
+        connection *conn = server.tls_cluster ?
+            connCreateAcceptedTLS(cfd, TLS_CLIENT_AUTH_YES) : connCreateAcceptedSocket(cfd);
         connNonBlock(conn);
         connEnableTcpNoDelay(conn);
 
diff --git a/src/config.c b/src/config.c
index 9b6dc459c..10faa3bea 100644
--- a/src/config.c
+++ b/src/config.c
@@ -98,6 +98,12 @@ configEnum repl_diskless_load_enum[] = {
     {NULL, 0}
 };
 
+configEnum tls_auth_clients_enum[] = {
+    {"no", TLS_CLIENT_AUTH_NO},
+    {"yes", TLS_CLIENT_AUTH_YES},
+    {"optional", TLS_CLIENT_AUTH_OPTIONAL},
+    {NULL, 0}
+};
 /* Output buffer limits presets. */
 clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = {
     {0, 0, 0}, /* normal */
@@ -2226,7 +2232,7 @@ standardConfig configs[] = {
     createIntConfig("tls-session-cache-timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tls_ctx_config.session_cache_timeout, 300, INTEGER_CONFIG, NULL, updateTlsCfgInt),
     createBoolConfig("tls-cluster", NULL, MODIFIABLE_CONFIG, server.tls_cluster, 0, NULL, updateTlsCfgBool),
     createBoolConfig("tls-replication", NULL, MODIFIABLE_CONFIG, server.tls_replication, 0, NULL, updateTlsCfgBool),
-    createBoolConfig("tls-auth-clients", NULL, MODIFIABLE_CONFIG, server.tls_auth_clients, 1, NULL, NULL),
+    createEnumConfig("tls-auth-clients", NULL, MODIFIABLE_CONFIG, tls_auth_clients_enum, server.tls_auth_clients, TLS_CLIENT_AUTH_YES, NULL, NULL),
     createBoolConfig("tls-prefer-server-ciphers", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.prefer_server_ciphers, 0, NULL, updateTlsCfgBool),
     createBoolConfig("tls-session-caching", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.session_caching, 1, NULL, updateTlsCfgBool),
     createStringConfig("tls-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.cert_file, NULL, NULL, updateTlsCfg),
diff --git a/src/server.h b/src/server.h
index afd020759..745ae76a2 100644
--- a/src/server.h
+++ b/src/server.h
@@ -358,6 +358,11 @@ typedef long long ustime_t; /* microsecond time type. */
 #define REPL_DISKLESS_LOAD_WHEN_DB_EMPTY 1
 #define REPL_DISKLESS_LOAD_SWAPDB 2
 
+/* TLS Client Authentication */
+#define TLS_CLIENT_AUTH_NO 0
+#define TLS_CLIENT_AUTH_YES 1
+#define TLS_CLIENT_AUTH_OPTIONAL 2
+
 /* Sets operations codes */
 #define SET_OP_UNION 0
 #define SET_OP_DIFF 1
diff --git a/src/tls.c b/src/tls.c
index 8b2bb58e1..07d1775f8 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -342,8 +342,16 @@ connection *connCreateAcceptedTLS(int fd, int require_auth) {
     conn->c.fd = fd;
     conn->c.state = CONN_STATE_ACCEPTING;
 
-    if (!require_auth) {
-        SSL_set_verify(conn->ssl, SSL_VERIFY_NONE, NULL);
+    switch (require_auth) {
+        case TLS_CLIENT_AUTH_NO:
+            SSL_set_verify(conn->ssl, SSL_VERIFY_NONE, NULL);
+            break;
+        case TLS_CLIENT_AUTH_OPTIONAL:
+            SSL_set_verify(conn->ssl, SSL_VERIFY_PEER, NULL);
+            break;
+        default: /* TLS_CLIENT_AUTH_YES, also fall-secure */
+            SSL_set_verify(conn->ssl, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);
+            break;
     }
 
     SSL_set_fd(conn->ssl, conn->c.fd);
diff --git a/tests/unit/tls.tcl b/tests/unit/tls.tcl
index 2b04590cd..bb5b6d034 100644
--- a/tests/unit/tls.tcl
+++ b/tests/unit/tls.tcl
@@ -21,7 +21,19 @@ start_server {tags {"tls"}} {
             catch {$s PING} e
             assert_match {PONG} $e
 
+            r CONFIG SET tls-auth-clients optional
+
+            set s [redis [srv 0 host] [srv 0 port]]
+            ::tls::import [$s channel]
+            catch {$s PING} e
+            assert_match {PONG} $e
+
             r CONFIG SET tls-auth-clients yes
+
+            set s [redis [srv 0 host] [srv 0 port]]
+            ::tls::import [$s channel]
+            catch {$s PING} e
+            assert_match {*error*} $e
         }
 
         test {TLS: Verify tls-protocols behaves as expected} {

From 109b5ccdcd6e6b8cecdaeb13a246bc49ce7a61f4 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 28 Jul 2020 11:15:29 +0300
Subject: [PATCH 0518/1098] Fix failing tests due to issues with
 wait_for_log_message (#7572)

- the test now waits for specific set of log messages rather than wait for
  timeout looking for just one message.
- we don't wanna sample the current length of the log after an action, due
  to a race, we need to start the search from the line number of the last
  message we where waiting for.
- when attempting to trigger a full sync, use multi-exec to avoid a race
  where the replica manages to re-connect before we completed the set of
  actions that should force a full sync.
- fix verify_log_message which was broken and unused
---
 tests/integration/replication.tcl | 30 +++++++++++++++---------------
 tests/support/util.tcl            | 20 ++++++++++++--------
 tests/unit/moduleapi/testrdb.tcl  | 22 +++++++++++-----------
 3 files changed, 38 insertions(+), 34 deletions(-)

diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl
index d47ec4fe4..1052fbdd5 100644
--- a/tests/integration/replication.tcl
+++ b/tests/integration/replication.tcl
@@ -440,30 +440,30 @@ test {diskless loading short read} {
             for {set i 0} {$i < $attempts} {incr i} {
                 # wait for the replica to start reading the rdb
                 # using the log file since the replica only responds to INFO once in 2mb
-                wait_for_log_message -1 "*Loading DB in memory*" $loglines 2000 1
+                set res [wait_for_log_messages -1 {"*Loading DB in memory*"} $loglines 2000 1]
+                set loglines [lindex $res 1]
 
                 # add some additional random sleep so that we kill the master on a different place each time
-                after [expr {int(rand()*100)}]
+                after [expr {int(rand()*50)}]
 
                 # kill the replica connection on the master
                 set killed [$master client kill type replica]
 
-                if {[catch {
-                    set res [wait_for_log_message -1 "*Internal error in RDB*" $loglines 100 10]
-                    if {$::verbose} {
-                        puts $res
-                    }
-                }]} {
-                    puts "failed triggering short read"
+                set res [wait_for_log_messages -1 {"*Internal error in RDB*" "*Finished with success*" "*Successful partial resynchronization*"} $loglines 1000 1]
+                if {$::verbose} { puts $res }
+                set log_text [lindex $res 0]
+                set loglines [lindex $res 1]
+                if {![string match "*Internal error in RDB*" $log_text]} {
                     # force the replica to try another full sync
+                    $master multi
                     $master client kill type replica
                     $master set asdf asdf
                     # the side effect of resizing the backlog is that it is flushed (16k is the min size)
                     $master config set repl-backlog-size [expr {16384 + $i}]
+                    $master exec
                 }
                 # wait for loading to stop (fail)
-                set loglines [count_log_lines -1]
-                wait_for_condition 100 10 {
+                wait_for_condition 1000 1 {
                     [s -1 loading] eq 0
                 } else {
                     fail "Replica didn't disconnect"
@@ -545,7 +545,7 @@ start_server {tags {"repl"}} {
 
                     # wait for the replicas to start reading the rdb
                     # using the log file since the replica only responds to INFO once in 2mb
-                    wait_for_log_message -1 "*Loading DB in memory*" $loglines 800 10
+                    wait_for_log_messages -1 {"*Loading DB in memory*"} $loglines 800 10
 
                     if {$measure_time} {
                         set master_statfile "/proc/$master_pid/stat"
@@ -580,13 +580,13 @@ start_server {tags {"repl"}} {
 
                     # make sure we got what we were aiming for, by looking for the message in the log file
                     if {$all_drop == "all"} {
-                        wait_for_log_message -2 "*Diskless rdb transfer, last replica dropped, killing fork child*" $loglines 1 1
+                        wait_for_log_messages -2 {"*Diskless rdb transfer, last replica dropped, killing fork child*"} $loglines 1 1
                     }
                     if {$all_drop == "no"} {
-                        wait_for_log_message -2 "*Diskless rdb transfer, done reading from pipe, 2 replicas still up*" $loglines 1 1
+                        wait_for_log_messages -2 {"*Diskless rdb transfer, done reading from pipe, 2 replicas still up*"} $loglines 1 1
                     }
                     if {$all_drop == "slow" || $all_drop == "fast"} {
-                        wait_for_log_message -2 "*Diskless rdb transfer, done reading from pipe, 1 replicas still up*" $loglines 1 1
+                        wait_for_log_messages -2 {"*Diskless rdb transfer, done reading from pipe, 1 replicas still up*"} $loglines 1 1
                     }
 
                     # make sure we don't have a busy loop going thought epoll_wait
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index 69ea675dc..8340ad207 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -106,31 +106,35 @@ proc count_log_lines {srv_idx} {
 
 # verify pattern exists in server's sdtout after a certain line number
 proc verify_log_message {srv_idx pattern from_line} {
-    set lines_after [count_log_lines]
-    set lines [expr $lines_after - $from_line]
-    set result [exec tail -$lines < [srv $srv_idx stdout]]
+    incr from_line
+    set result [exec tail -n +$from_line < [srv $srv_idx stdout]]
     if {![string match $pattern $result]} {
         error "assertion:expected message not found in log file: $pattern"
     }
 }
 
 # wait for pattern to be found in server's stdout after certain line number
-proc wait_for_log_message {srv_idx pattern from_line maxtries delay} {
+# return value is a list containing the line that matched the pattern and the line number
+proc wait_for_log_messages {srv_idx patterns from_line maxtries delay} {
     set retry $maxtries
+    set next_line [expr $from_line + 1] ;# searching form the line after
     set stdout [srv $srv_idx stdout]
     while {$retry} {
-        set result [exec tail -n +$from_line < $stdout]
+        set result [exec tail -n +$next_line < $stdout]
         set result [split $result "\n"]
         foreach line $result {
-            if {[string match $pattern $line]} {
-                return $line
+            foreach pattern $patterns {
+                if {[string match $pattern $line]} {
+                    return [list $line $next_line]
+                }
             }
+            incr next_line
         }
         incr retry -1
         after $delay
     }
     if {$retry == 0} {
-        fail "log message of '$pattern' not found in $stdout after line: $from_line"
+        fail "log message of '$patterns' not found in $stdout after line: $from_line till line: [expr $next_line -1]"
     }
 }
 
diff --git a/tests/unit/moduleapi/testrdb.tcl b/tests/unit/moduleapi/testrdb.tcl
index 98641ae0a..02c82c7c3 100644
--- a/tests/unit/moduleapi/testrdb.tcl
+++ b/tests/unit/moduleapi/testrdb.tcl
@@ -77,30 +77,30 @@ tags "modules" {
                     for {set i 0} {$i < $attempts} {incr i} {
                         # wait for the replica to start reading the rdb
                         # using the log file since the replica only responds to INFO once in 2mb
-                        wait_for_log_message -1 "*Loading DB in memory*" $loglines 2000 1
+                        set res [wait_for_log_messages -1 {"*Loading DB in memory*"} $loglines 2000 1]
+                        set loglines [lindex $res 1]
 
                         # add some additional random sleep so that we kill the master on a different place each time
-                        after [expr {int(rand()*100)}]
+                        after [expr {int(rand()*50)}]
 
                         # kill the replica connection on the master
                         set killed [$master client kill type replica]
 
-                        if {[catch {
-                            set res [wait_for_log_message -1 "*Internal error in RDB*" $loglines 100 10]
-                            if {$::verbose} {
-                                puts $res
-                            }
-                        }]} {
-                            puts "failed triggering short read"
+                        set res [wait_for_log_messages -1 {"*Internal error in RDB*" "*Finished with success*" "*Successful partial resynchronization*"} $loglines 1000 1]
+                        if {$::verbose} { puts $res }
+                        set log_text [lindex $res 0]
+                        set loglines [lindex $res 1]
+                        if {![string match "*Internal error in RDB*" $log_text]} {
                             # force the replica to try another full sync
+                            $master multi
                             $master client kill type replica
                             $master set asdf asdf
                             # the side effect of resizing the backlog is that it is flushed (16k is the min size)
                             $master config set repl-backlog-size [expr {16384 + $i}]
+                            $master exec
                         }
                         # wait for loading to stop (fail)
-                        set loglines [count_log_lines -1]
-                        wait_for_condition 100 10 {
+                        wait_for_condition 1000 1 {
                             [s -1 loading] eq 0
                         } else {
                             fail "Replica didn't disconnect"

From 784ceeb90d84bbc49fc2f2e2e6c7b9fae2524bd5 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 28 Jul 2020 11:32:47 +0300
Subject: [PATCH 0519/1098] TLS: Propagate and handle SSL_new() failures.
 (#7576)

The connection API may create an accepted connection object in an error
state, and callers are expected to check it before attempting to use it.

Co-authored-by: mrpre <mrpre@163.com>
---
 src/cluster.c    |  9 +++++++++
 src/connection.c |  6 +++++-
 src/networking.c | 11 ++++++++++-
 src/tls.c        | 28 ++++++++++++++++++++++++----
 4 files changed, 48 insertions(+), 6 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 485683cf0..350aa7b6a 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -672,6 +672,15 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
 
         connection *conn = server.tls_cluster ?
             connCreateAcceptedTLS(cfd, TLS_CLIENT_AUTH_YES) : connCreateAcceptedSocket(cfd);
+
+        /* Make sure connection is not in an error state */
+        if (connGetState(conn) != CONN_STATE_ACCEPTING) {
+            serverLog(LL_VERBOSE,
+                "Error creating an accepting connection for cluster node: %s",
+                    connGetLastError(conn));
+            connClose(conn);
+            return;
+        }
         connNonBlock(conn);
         connEnableTcpNoDelay(conn);
 
diff --git a/src/connection.c b/src/connection.c
index 2015c9195..09fa12f2a 100644
--- a/src/connection.c
+++ b/src/connection.c
@@ -85,8 +85,12 @@ connection *connCreateSocket() {
 /* Create a new socket-type connection that is already associated with
  * an accepted connection.
  *
- * The socket is not read for I/O until connAccept() was called and
+ * The socket is not ready for I/O until connAccept() was called and
  * invoked the connection-level accept handler.
+ *
+ * Callers should use connGetState() and verify the created connection
+ * is not in an error state (which is not possible for a socket connection,
+ * but could but possible with other protocols).
  */
 connection *connCreateAcceptedSocket(int fd) {
     connection *conn = connCreateSocket();
diff --git a/src/networking.c b/src/networking.c
index e3b62f151..a3c04efa6 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -895,8 +895,18 @@ void clientAcceptHandler(connection *conn) {
 #define MAX_ACCEPTS_PER_CALL 1000
 static void acceptCommonHandler(connection *conn, int flags, char *ip) {
     client *c;
+    char conninfo[100];
     UNUSED(ip);
 
+    if (connGetState(conn) != CONN_STATE_ACCEPTING) {
+        serverLog(LL_VERBOSE,
+            "Accepted client connection in error state: %s (conn: %s)",
+            connGetLastError(conn),
+            connGetInfo(conn, conninfo, sizeof(conninfo)));
+        connClose(conn);
+        return;
+    }
+
     /* Limit the number of connections we take at the same time.
      *
      * Admission control will happen before a client is created and connAccept()
@@ -925,7 +935,6 @@ static void acceptCommonHandler(connection *conn, int flags, char *ip) {
 
     /* Create connection and client */
     if ((c = createClient(conn)) == NULL) {
-        char conninfo[100];
         serverLog(LL_WARNING,
             "Error registering fd event for the new client: %s (conn: %s)",
             connGetLastError(conn),
diff --git a/src/tls.c b/src/tls.c
index 07d1775f8..4f0ea4d65 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -337,11 +337,34 @@ connection *connCreateTLS(void) {
     return (connection *) conn;
 }
 
+/* Fetch the latest OpenSSL error and store it in the connection */
+static void updateTLSError(tls_connection *conn) {
+    conn->c.last_errno = 0;
+    if (conn->ssl_error) zfree(conn->ssl_error);
+    conn->ssl_error = zmalloc(512);
+    ERR_error_string_n(ERR_get_error(), conn->ssl_error, 512);
+}
+
+/* Create a new TLS connection that is already associated with
+ * an accepted underlying file descriptor.
+ *
+ * The socket is not ready for I/O until connAccept() was called and
+ * invoked the connection-level accept handler.
+ *
+ * Callers should use connGetState() and verify the created connection
+ * is not in an error state.
+ */
 connection *connCreateAcceptedTLS(int fd, int require_auth) {
     tls_connection *conn = (tls_connection *) connCreateTLS();
     conn->c.fd = fd;
     conn->c.state = CONN_STATE_ACCEPTING;
 
+    if (!conn->ssl) {
+        updateTLSError(conn);
+        conn->c.state = CONN_STATE_ERROR;
+        return (connection *) conn;
+    }
+
     switch (require_auth) {
         case TLS_CLIENT_AUTH_NO:
             SSL_set_verify(conn->ssl, SSL_VERIFY_NONE, NULL);
@@ -384,10 +407,7 @@ static int handleSSLReturnCode(tls_connection *conn, int ret_value, WantIOType *
                 break;
             default:
                 /* Error! */
-                conn->c.last_errno = 0;
-                if (conn->ssl_error) zfree(conn->ssl_error);
-                conn->ssl_error = zmalloc(512);
-                ERR_error_string_n(ERR_get_error(), conn->ssl_error, 512);
+                updateTLSError(conn);
                 break;
         }
 

From bedf1b21269dfb8afb584bc24585023af8b2a208 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 28 Jul 2020 14:04:06 +0300
Subject: [PATCH 0520/1098] Fix TLS cluster tests. (#7578)

Fix consistency test added in af5167b7f without considering TLS
redis-cli configuration.
---
 tests/cluster/tests/14-consistency-check.tcl | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/cluster/tests/14-consistency-check.tcl b/tests/cluster/tests/14-consistency-check.tcl
index a43725ebc..d3b3052f3 100644
--- a/tests/cluster/tests/14-consistency-check.tcl
+++ b/tests/cluster/tests/14-consistency-check.tcl
@@ -64,7 +64,10 @@ proc test_slave_load_expired_keys {aof} {
         kill_instance redis $replica_id
 
         set master_port [get_instance_attrib redis $master_id port]
-        exec ../../../src/redis-cli -h 127.0.0.1 -p $master_port debug sleep [expr $data_ttl+3] > /dev/null &
+        exec ../../../src/redis-cli \
+            -h 127.0.0.1 -p $master_port \
+            {*}[rediscli_tls_config "../../../tests"] \
+            debug sleep [expr $data_ttl+3] > /dev/null &
 
         while {[clock seconds] <= $end_time} {
             #wait for $data_ttl seconds

From c69a9b2f61e0747fabb8120f03c9e2a29b43b472 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Tue, 28 Jul 2020 15:05:48 -0400
Subject: [PATCH 0521/1098] fix leak in error handling of debug populate
 command (#7062)

valsize was not modified during the for loop below instead of getting from c->argv[4], therefore there is no need to put inside the for loop.. Moreover, putting the check outside loop will also avoid memory leaking, decrRefCount(key) should be called in the original code if we put the check in for loop
---
 src/debug.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/debug.c b/src/debug.c
index 60cc2a1fa..0bea69876 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -591,14 +591,13 @@ NULL
         if (getLongFromObjectOrReply(c, c->argv[2], &keys, NULL) != C_OK)
             return;
         dictExpand(c->db->dict,keys);
+        long valsize = 0;
+        if ( c->argc == 5 && getLongFromObjectOrReply(c, c->argv[4], &valsize, NULL) != C_OK ) 
+            return;
         for (j = 0; j < keys; j++) {
-            long valsize = 0;
             snprintf(buf,sizeof(buf),"%s:%lu",
                 (c->argc == 3) ? "key" : (char*)c->argv[3]->ptr, j);
             key = createStringObject(buf,strlen(buf));
-            if (c->argc == 5)
-                if (getLongFromObjectOrReply(c, c->argv[4], &valsize, NULL) != C_OK)
-                    return;
             if (lookupKeyWrite(c->db,key) != NULL) {
                 decrRefCount(key);
                 continue;

From f33acb3f029d7f5aa68e22e612036e04417e3e91 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Wed, 29 Jul 2020 01:22:54 -0400
Subject: [PATCH 0522/1098] Add SignalModifiedKey hook in XGROUP CREATE with
 MKSTREAM option (#7562)

---
 src/t_stream.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/t_stream.c b/src/t_stream.c
index 19b20ee56..79265a6fb 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1845,6 +1845,7 @@ NULL
             o = createStreamObject();
             dbAdd(c->db,c->argv[2],o);
             s = o->ptr;
+            signalModifiedKey(c,c->db,c->argv[2]);
         }
 
         streamCG *cg = streamCreateCG(s,grpname,sdslen(grpname),&id);

From 63dae5232415d216dfc1acce8b5335e20aa3b178 Mon Sep 17 00:00:00 2001
From: namtsui <384455+namtsui@users.noreply.github.com>
Date: Tue, 28 Jul 2020 22:25:56 -0700
Subject: [PATCH 0523/1098] Avoid an out-of-bounds read in the redis-sentinel
 (#7443)

The Redis sentinel would crash with a segfault after a few minutes
because it tried to read from a page without read permissions. Check up
front whether the sds is long enough to contain redis:slave or
redis:master before memcmp() as is done everywhere else in
sentinelRefreshInstanceInfo().

Bug report and commit message from Theo Buehler. Fix from Nam Nguyen.

Co-authored-by: Nam Nguyen <namn@berkeley.edu>
---
 src/sentinel.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/sentinel.c b/src/sentinel.c
index 5be4193dc..5bd594955 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -2218,8 +2218,8 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) {
         }
 
         /* role:<role> */
-        if (!memcmp(l,"role:master",11)) role = SRI_MASTER;
-        else if (!memcmp(l,"role:slave",10)) role = SRI_SLAVE;
+        if (sdslen(l) >= 11 && !memcmp(l,"role:master",11)) role = SRI_MASTER;
+        else if (sdslen(l) >= 10 && !memcmp(l,"role:slave",10)) role = SRI_SLAVE;
 
         if (role == SRI_SLAVE) {
             /* master_host:<host> */

From f6cad30bb69b2ad35bb0a870077fac2d4605d727 Mon Sep 17 00:00:00 2001
From: Arun Ranganathan <arun.ranga@hotmail.ca>
Date: Wed, 29 Jul 2020 01:46:44 -0400
Subject: [PATCH 0524/1098] Show threading configuration in INFO output (#7446)

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/networking.c | 33 +++++++++++++++++++++++----------
 src/server.c     | 22 ++++++++++++++++++----
 src/server.h     |  5 +++++
 3 files changed, 46 insertions(+), 14 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index a3c04efa6..0da9bbc9c 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1295,6 +1295,9 @@ client *lookupClientByID(uint64_t id) {
  * set to 0. So when handler_installed is set to 0 the function must be
  * thread safe. */
 int writeToClient(client *c, int handler_installed) {
+    /* Update total number of writes on server */
+    server.stat_total_writes_processed++;
+
     ssize_t nwritten = 0, totwritten = 0;
     size_t objlen;
     clientReplyBlock *o;
@@ -1910,6 +1913,9 @@ void readQueryFromClient(connection *conn) {
      * the event loop. This is the case if threaded I/O is enabled. */
     if (postponeClientRead(c)) return;
 
+    /* Update total number of reads on server */
+    server.stat_total_reads_processed++;
+
     readlen = PROTO_IOBUF_LEN;
     /* If this is a multi bulk request, and we are processing a bulk reply
      * that is large enough, try to maximize the probability that the query
@@ -2907,7 +2913,6 @@ int tio_debug = 0;
 pthread_t io_threads[IO_THREADS_MAX_NUM];
 pthread_mutex_t io_threads_mutex[IO_THREADS_MAX_NUM];
 _Atomic unsigned long io_threads_pending[IO_THREADS_MAX_NUM];
-int io_threads_active;  /* Are the threads currently spinning waiting I/O? */
 int io_threads_op;      /* IO_THREADS_OP_WRITE or IO_THREADS_OP_READ. */
 
 /* This is the list of clients each thread will serve when threaded I/O is
@@ -2966,7 +2971,7 @@ void *IOThreadMain(void *myid) {
 
 /* Initialize the data structures needed for threaded I/O. */
 void initThreadedIO(void) {
-    io_threads_active = 0; /* We start with threads not active. */
+    server.io_threads_active = 0; /* We start with threads not active. */
 
     /* Don't spawn any thread if the user selected a single thread:
      * we'll handle I/O directly from the main thread. */
@@ -3000,10 +3005,10 @@ void initThreadedIO(void) {
 void startThreadedIO(void) {
     if (tio_debug) { printf("S"); fflush(stdout); }
     if (tio_debug) printf("--- STARTING THREADED IO ---\n");
-    serverAssert(io_threads_active == 0);
+    serverAssert(server.io_threads_active == 0);
     for (int j = 1; j < server.io_threads_num; j++)
         pthread_mutex_unlock(&io_threads_mutex[j]);
-    io_threads_active = 1;
+    server.io_threads_active = 1;
 }
 
 void stopThreadedIO(void) {
@@ -3014,10 +3019,10 @@ void stopThreadedIO(void) {
     if (tio_debug) printf("--- STOPPING THREADED IO [R%d] [W%d] ---\n",
         (int) listLength(server.clients_pending_read),
         (int) listLength(server.clients_pending_write));
-    serverAssert(io_threads_active == 1);
+    serverAssert(server.io_threads_active == 1);
     for (int j = 1; j < server.io_threads_num; j++)
         pthread_mutex_lock(&io_threads_mutex[j]);
-    io_threads_active = 0;
+    server.io_threads_active = 0;
 }
 
 /* This function checks if there are not enough pending clients to justify
@@ -3036,7 +3041,7 @@ int stopThreadedIOIfNeeded(void) {
     if (server.io_threads_num == 1) return 1;
 
     if (pending < (server.io_threads_num*2)) {
-        if (io_threads_active) stopThreadedIO();
+        if (server.io_threads_active) stopThreadedIO();
         return 1;
     } else {
         return 0;
@@ -3054,7 +3059,7 @@ int handleClientsWithPendingWritesUsingThreads(void) {
     }
 
     /* Start threads if needed. */
-    if (!io_threads_active) startThreadedIO();
+    if (!server.io_threads_active) startThreadedIO();
 
     if (tio_debug) printf("%d TOTAL WRITE pending clients\n", processed);
 
@@ -3111,6 +3116,10 @@ int handleClientsWithPendingWritesUsingThreads(void) {
         }
     }
     listEmpty(server.clients_pending_write);
+
+    /* Update processed count on server */
+    server.stat_io_writes_processed += processed;
+
     return processed;
 }
 
@@ -3119,7 +3128,7 @@ int handleClientsWithPendingWritesUsingThreads(void) {
  * As a side effect of calling this function the client is put in the
  * pending read clients and flagged as such. */
 int postponeClientRead(client *c) {
-    if (io_threads_active &&
+    if (server.io_threads_active &&
         server.io_threads_do_reads &&
         !ProcessingEventsWhileBlocked &&
         !(c->flags & (CLIENT_MASTER|CLIENT_SLAVE|CLIENT_PENDING_READ)))
@@ -3139,7 +3148,7 @@ int postponeClientRead(client *c) {
  * the reads in the buffers, and also parse the first command available
  * rendering it in the client structures. */
 int handleClientsWithPendingReadsUsingThreads(void) {
-    if (!io_threads_active || !server.io_threads_do_reads) return 0;
+    if (!server.io_threads_active || !server.io_threads_do_reads) return 0;
     int processed = listLength(server.clients_pending_read);
     if (processed == 0) return 0;
 
@@ -3200,5 +3209,9 @@ int handleClientsWithPendingReadsUsingThreads(void) {
         }
         processInputBuffer(c);
     }
+
+    /* Update processed count on server */
+    server.stat_io_reads_processed += processed;
+
     return processed;
 }
diff --git a/src/server.c b/src/server.c
index 22a910e1c..afe5094c5 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2726,6 +2726,10 @@ void resetServerStats(void) {
     server.stat_sync_full = 0;
     server.stat_sync_partial_ok = 0;
     server.stat_sync_partial_err = 0;
+    server.stat_io_reads_processed = 0;
+    server.stat_total_reads_processed = 0;
+    server.stat_io_writes_processed = 0;
+    server.stat_total_writes_processed = 0;
     for (j = 0; j < STATS_METRIC_COUNT; j++) {
         server.inst_metric[j].idx = 0;
         server.inst_metric[j].last_sample_time = mstime();
@@ -4071,7 +4075,8 @@ sds genRedisInfoString(const char *section) {
             "configured_hz:%i\r\n"
             "lru_clock:%u\r\n"
             "executable:%s\r\n"
-            "config_file:%s\r\n",
+            "config_file:%s\r\n"
+            "io_threads_active:%d\r\n",
             REDIS_VERSION,
             redisGitSHA1(),
             strtol(redisGitDirty(),NULL,10) > 0,
@@ -4095,7 +4100,8 @@ sds genRedisInfoString(const char *section) {
             server.config_hz,
             server.lruclock,
             server.executable ? server.executable : "",
-            server.configfile ? server.configfile : "");
+            server.configfile ? server.configfile : "",
+            server.io_threads_active);
     }
 
     /* Clients */
@@ -4367,7 +4373,11 @@ sds genRedisInfoString(const char *section) {
             "tracking_total_keys:%lld\r\n"
             "tracking_total_items:%lld\r\n"
             "tracking_total_prefixes:%lld\r\n"
-            "unexpected_error_replies:%lld\r\n",
+            "unexpected_error_replies:%lld\r\n"
+            "total_reads_processed:%lld\r\n"
+            "total_writes_processed:%lld\r\n"
+            "io_threaded_reads_processed:%lld\r\n"
+            "io_threaded_writes_processed:%lld\r\n",
             server.stat_numconnections,
             server.stat_numcommands,
             getInstantaneousMetric(STATS_METRIC_COMMAND),
@@ -4398,7 +4408,11 @@ sds genRedisInfoString(const char *section) {
             (unsigned long long) trackingGetTotalKeys(),
             (unsigned long long) trackingGetTotalItems(),
             (unsigned long long) trackingGetTotalPrefixes(),
-            server.stat_unexpected_error_replies);
+            server.stat_unexpected_error_replies,
+            server.stat_total_reads_processed,
+            server.stat_total_writes_processed,
+            server.stat_io_reads_processed,
+            server.stat_io_writes_processed);
     }
 
     /* Replication */
diff --git a/src/server.h b/src/server.h
index 745ae76a2..adfac3293 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1108,6 +1108,7 @@ struct redisServer {
                                    queries. Will still serve RESP2 queries. */
     int io_threads_num;         /* Number of IO threads to use. */
     int io_threads_do_reads;    /* Read and parse from IO threads? */
+    int io_threads_active;      /* Is IO threads currently active? */
     long long events_processed_while_blocked; /* processEventsWhileBlocked() */
 
     /* RDB / AOF loading information */
@@ -1157,6 +1158,10 @@ struct redisServer {
     size_t stat_module_cow_bytes;   /* Copy on write bytes during module fork. */
     uint64_t stat_clients_type_memory[CLIENT_TYPE_COUNT];/* Mem usage by type */
     long long stat_unexpected_error_replies; /* Number of unexpected (aof-loading, replica to master, etc.) error replies */
+    long long stat_io_reads_processed; /* Number of read events processed by IO / Main threads */
+    long long stat_io_writes_processed; /* Number of write events processed by IO / Main threads */
+    _Atomic long long stat_total_reads_processed; /* Total number of read events processed */
+    _Atomic long long stat_total_writes_processed; /* Total number of write events processed */
     /* The following two are used to track instantaneous metrics, like
      * number of operations per second, network traffic. */
     struct {

From 7af05f07ff72a4246ddaf986a1e815eadd0cbfef Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 29 Jul 2020 17:03:38 +0300
Subject: [PATCH 0525/1098] Clarify  RM_BlockClient() error condition. (#6093)

---
 src/module.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/module.c b/src/module.c
index 0ccd5bbaa..b008929f7 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4347,6 +4347,7 @@ void unblockClientFromModule(client *c) {
  * Even when blocking on keys, RM_UnblockClient() can be called however, but
  * in that case the privdata argument is disregarded, because we pass the
  * reply callback the privdata that is set here while blocking.
+ *
  */
 RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata) {
     client *c = ctx->client;
@@ -4439,6 +4440,14 @@ int moduleTryServeClientBlockedOnKey(client *c, robj *key) {
  * Note: RedisModule_UnblockClient should be called for every blocked client,
  *       even if client was killed, timed-out or disconnected. Failing to do so
  *       will result in memory leaks.
+ *
+ * There are some cases where RedisModule_BlockClient() cannot be used:
+ *
+ * 1. If the client is a Lua script.
+ * 2. If the client is executing a MULTI block.
+ *
+ * In these cases, a call to RedisModule_BlockClient() will **not** block the
+ * client, but instead produce a specific error reply.
  */
 RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms) {
     return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, NULL,0,NULL);

From b55c1602b564d67ccd61a16f70eeefbe39600fba Mon Sep 17 00:00:00 2001
From: Kevin McGehee <mcgehee@amazon.com>
Date: Wed, 29 Jul 2020 17:54:37 -0700
Subject: [PATCH 0526/1098] Call propagate instead of writing directly to
 AOF/replicas (#6658)

Use higher-level API to funnel all generic propagation through
single function call.
---
 src/db.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/db.c b/src/db.c
index 19b2c48e4..529b57c7b 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1228,9 +1228,7 @@ void propagateExpire(redisDb *db, robj *key, int lazy) {
     incrRefCount(argv[0]);
     incrRefCount(argv[1]);
 
-    if (server.aof_state != AOF_OFF)
-        feedAppendOnlyFile(server.delCommand,db->id,argv,2);
-    replicationFeedSlaves(server.slaves,db->id,argv,2);
+    propagate(server.delCommand,db->id,argv,2,PROPAGATE_AOF|PROPAGATE_REPL);
 
     decrRefCount(argv[0]);
     decrRefCount(argv[1]);

From 4faad81f50f044b1ed92daae58e1e01e23c864fc Mon Sep 17 00:00:00 2001
From: fayadexinqing <2418967201@qq.com>
Date: Thu, 30 Jul 2020 09:05:27 +0800
Subject: [PATCH 0527/1098] broadcast a PONG message when slot's migration is
 over, which may reduce the moved request of clients (#7571)

---
 src/cluster.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/cluster.c b/src/cluster.c
index 350aa7b6a..8860bfc14 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4522,6 +4522,9 @@ NULL
                         "configEpoch updated after importing slot %d", slot);
                 }
                 server.cluster->importing_slots_from[slot] = NULL;
+                /* After importing this slot, let the other nodes know as
+                 * soon as possible. */
+                clusterBroadcastPong(CLUSTER_BROADCAST_ALL);
             }
             clusterDelSlot(slot);
             clusterAddSlot(n,slot);

From f3352daf4f4826e1cad4c163fd6e35b81a72e21b Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Thu, 30 Jul 2020 13:56:21 +0800
Subject: [PATCH 0528/1098] Fix running single test 14-consistency-check.tcl
 (#7587)

---
 tests/cluster/tests/14-consistency-check.tcl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/cluster/tests/14-consistency-check.tcl b/tests/cluster/tests/14-consistency-check.tcl
index d3b3052f3..5a80dd0df 100644
--- a/tests/cluster/tests/14-consistency-check.tcl
+++ b/tests/cluster/tests/14-consistency-check.tcl
@@ -1,4 +1,5 @@
 source "../tests/includes/init-tests.tcl"
+source "../../../tests/support/cli.tcl"
 
 test "Create a 5 nodes cluster" {
     create_cluster 5 5

From 4ac1f9ac55b841094861e84cdec5da0f4a294a86 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 30 Jul 2020 13:25:10 +0300
Subject: [PATCH 0529/1098] CI: Add daily CentOS 7.x jobs. (#7582)

---
 .github/workflows/daily.yml | 54 ++++++++++++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index acc811d3a..458f07c97 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -10,7 +10,7 @@ on:
 
 jobs:
 
-  test-jemalloc:
+  test-ubuntu-jemalloc:
     runs-on: ubuntu-latest
     timeout-minutes: 14400
     steps:
@@ -28,7 +28,7 @@ jobs:
     - name: cluster tests
       run: ./runtest-cluster
 
-  test-libc-malloc:
+  test-ubuntu-libc-malloc:
     runs-on: ubuntu-latest
     timeout-minutes: 14400
     steps:
@@ -46,7 +46,7 @@ jobs:
     - name: cluster tests
       run: ./runtest-cluster
 
-  test-32bit:
+  test-ubuntu-32bit:
     runs-on: ubuntu-latest
     timeout-minutes: 14400
     steps:
@@ -68,7 +68,7 @@ jobs:
     - name: cluster tests
       run: ./runtest-cluster
 
-  test-tls:
+  test-ubuntu-tls:
     runs-on: ubuntu-latest
     timeout-minutes: 14400
     steps:
@@ -101,3 +101,49 @@ jobs:
         ./runtest --valgrind --verbose --clients 1
     - name: module api test
       run: ./runtest-moduleapi --valgrind --verbose --clients 1
+
+  test-centos7-jemalloc:
+    runs-on: ubuntu-latest
+    container: centos:7
+    timeout-minutes: 14400
+    steps:
+    - uses: actions/checkout@v2
+    - name: make
+      run: |
+        yum -y install centos-release-scl
+        yum -y install devtoolset-7
+        scl enable devtoolset-7 make
+    - name: test
+      run: |
+        yum -y install tcl
+        ./runtest --accurate --verbose
+    - name: module api test
+      run: ./runtest-moduleapi --verbose
+    - name: sentinel tests
+      run: ./runtest-sentinel
+    - name: cluster tests
+      run: ./runtest-cluster
+
+  test-centos7-tls:
+    runs-on: ubuntu-latest
+    container: centos:7
+    timeout-minutes: 14400
+    steps:
+    - uses: actions/checkout@v2
+    - name: make
+      run: |
+        yum -y install centos-release-scl epel-release
+        yum -y install devtoolset-7 openssl-devel openssl
+        scl enable devtoolset-7 make BUILD_TLS=yes
+    - name: test
+      run: |
+        yum -y install tcl tcltls
+        ./utils/gen-test-certs.sh
+        ./runtest --accurate --verbose --tls
+    - name: module api test
+      run: ./runtest-moduleapi --verbose --tls
+    - name: sentinel tests
+      run: ./runtest-sentinel --tls
+    - name: cluster tests
+      run: ./runtest-cluster --tls
+

From 50f5181488d9c3a5557a0b32249a4acdd70684ce Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 31 Jul 2020 13:01:39 +0300
Subject: [PATCH 0530/1098] Remove dead code from update_zmalloc_stat_alloc
 (#7589)

this seems like leftover from before 6eb51bf
---
 src/zmalloc.c | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/src/zmalloc.c b/src/zmalloc.c
index 639a5fe2b..e36319241 100644
--- a/src/zmalloc.c
+++ b/src/zmalloc.c
@@ -71,17 +71,8 @@ void zlibc_free(void *ptr) {
 #define dallocx(ptr,flags) je_dallocx(ptr,flags)
 #endif
 
-#define update_zmalloc_stat_alloc(__n) do { \
-    size_t _n = (__n); \
-    if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \
-    atomicIncr(used_memory,__n); \
-} while(0)
-
-#define update_zmalloc_stat_free(__n) do { \
-    size_t _n = (__n); \
-    if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \
-    atomicDecr(used_memory,__n); \
-} while(0)
+#define update_zmalloc_stat_alloc(__n) atomicIncr(used_memory,(__n))
+#define update_zmalloc_stat_free(__n) atomicDecr(used_memory,(__n))
 
 static size_t used_memory = 0;
 pthread_mutex_t used_memory_mutex = PTHREAD_MUTEX_INITIALIZER;

From f7e77759902aa19cfa537ed454e6bc987498e8c5 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 31 Jul 2020 13:14:29 +0300
Subject: [PATCH 0531/1098] module hook for master link up missing on
 successful psync (#7584)

besides, hooks test was time sensitive. when the replica managed to
reconnect quickly after the client kill, the test would fail
---
 src/replication.c              |  5 +++++
 tests/unit/moduleapi/hooks.tcl | 19 +++++++++++++++++--
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index 197acde79..a8f46be95 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2824,6 +2824,11 @@ void replicationResurrectCachedMaster(connection *conn) {
     server.repl_state = REPL_STATE_CONNECTED;
     server.repl_down_since = 0;
 
+    /* Fire the master link modules event. */
+    moduleFireServerEvent(REDISMODULE_EVENT_MASTER_LINK_CHANGE,
+                          REDISMODULE_SUBEVENT_MASTER_LINK_UP,
+                          NULL);
+
     /* Re-add to the list of clients. */
     linkClient(server.master);
     if (connSetReadHandler(server.master->conn, readQueryFromClient)) {
diff --git a/tests/unit/moduleapi/hooks.tcl b/tests/unit/moduleapi/hooks.tcl
index bf2b9010b..da0307ce6 100644
--- a/tests/unit/moduleapi/hooks.tcl
+++ b/tests/unit/moduleapi/hooks.tcl
@@ -114,6 +114,21 @@ tags "modules" {
             test {Test master link down hook} {
                 r client kill type master
                 assert_equal [r hooks.event_count masterlink-down] 1
+
+                wait_for_condition 50 100 {
+                    [string match {*master_link_status:up*} [r info replication]]
+                } else {
+                    fail "Replica didn't reconnect"
+                }
+
+                assert_equal [r hooks.event_count masterlink-down] 1
+                assert_equal [r hooks.event_count masterlink-up] 2
+            }
+
+            wait_for_condition 50 10 {
+                [string match {*master_link_status:up*} [r info replication]]
+            } else {
+                fail "Can't turn the instance into a replica"
             }
 
             $replica replicaof no one
@@ -125,8 +140,8 @@ tags "modules" {
             }
 
             test {Test replica-offline hook} {
-                assert_equal [r -1 hooks.event_count replica-online] 1
-                assert_equal [r -1 hooks.event_count replica-offline] 1
+                assert_equal [r -1 hooks.event_count replica-online] 2
+                assert_equal [r -1 hooks.event_count replica-offline] 2
             }
             # get the replica stdout, to be used by the next test
             set replica_stdout [srv 0 stdout]

From f2da3efc78fe39c52fd46a2b9b5ed05e29647119 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Fri, 31 Jul 2020 13:55:57 +0300
Subject: [PATCH 0532/1098] Fix test-centos7-tls daily job. (#7598)

---
 .github/workflows/daily.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 458f07c97..986bd223b 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -112,7 +112,7 @@ jobs:
       run: |
         yum -y install centos-release-scl
         yum -y install devtoolset-7
-        scl enable devtoolset-7 make
+        scl enable devtoolset-7 "make"
     - name: test
       run: |
         yum -y install tcl
@@ -134,7 +134,7 @@ jobs:
       run: |
         yum -y install centos-release-scl epel-release
         yum -y install devtoolset-7 openssl-devel openssl
-        scl enable devtoolset-7 make BUILD_TLS=yes
+        scl enable devtoolset-7 "make BUILD_TLS=yes"
     - name: test
       run: |
         yum -y install tcl tcltls

From 2f454e5ae81769ec6670a54bbe4722edecb1a3bf Mon Sep 17 00:00:00 2001
From: hujie <hujiecs@qq.com>
Date: Sun, 2 Aug 2020 18:59:51 +0800
Subject: [PATCH 0533/1098] remove duplicate semicolon (#7438)

---
 src/config.c | 2 +-
 src/object.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/config.c b/src/config.c
index 10faa3bea..a68fddf1d 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1058,7 +1058,7 @@ struct rewriteConfigState *rewriteConfigReadOldFile(char *path) {
         char *p = strstr(argv[0],"slave");
         if (p) {
             sds alt = sdsempty();
-            alt = sdscatlen(alt,argv[0],p-argv[0]);;
+            alt = sdscatlen(alt,argv[0],p-argv[0]);
             alt = sdscatlen(alt,"replica",7);
             alt = sdscatlen(alt,p+5,strlen(p+5));
             sdsfree(argv[0]);
diff --git a/src/object.c b/src/object.c
index 1bc400e85..6ba18d6a5 100644
--- a/src/object.c
+++ b/src/object.c
@@ -1298,7 +1298,7 @@ NULL
                     addReply(c,shared.syntaxerr);
                     return;
                 }
-                if (samples == 0) samples = LLONG_MAX;;
+                if (samples == 0) samples = LLONG_MAX;
                 j++; /* skip option argument. */
             } else {
                 addReply(c,shared.syntaxerr);

From 824bd2ac11472b7a3fce9fcf3189a8e6c6048115 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 4 Aug 2020 08:53:50 +0300
Subject: [PATCH 0534/1098] fix new rdb test failing on timing issues (#7604)

apparenlty on github actions sometimes 500ms is not enough
---
 tests/integration/rdb.tcl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl
index 6535cd089..9cd970350 100644
--- a/tests/integration/rdb.tcl
+++ b/tests/integration/rdb.tcl
@@ -139,10 +139,10 @@ start_server {} {
     test {bgsave resets the change counter} {
         r config set rdb-key-save-delay 0
         r bgsave
-        wait_for_condition 5 100 {
+        wait_for_condition 50 100 {
             [s rdb_bgsave_in_progress] == 0
         } else {
-            fail "bgsave not aborted"
+            fail "bgsave not done"
         }
         assert_equal [s rdb_changes_since_last_save] 0
     }

From f11f26cc53f25767c26b98ceb21e1b1fc3aef2d2 Mon Sep 17 00:00:00 2001
From: Tyson Andre <tyson.andre@uwaterloo.ca>
Date: Tue, 4 Aug 2020 10:49:33 -0400
Subject: [PATCH 0535/1098] Add a ZMSCORE command returning an array of scores.
 (#7593)

Syntax: `ZMSCORE KEY MEMBER [MEMBER ...]`

This is an extension of #2359
amended by Tyson Andre to work with the changed unstable API,
add more tests, and consistently return an array.

- It seemed as if it would be more likely to get reviewed
  after updating the implementation.

Currently, multi commands or lua scripting to call zscore multiple times
would almost definitely be less efficient than a native ZMSCORE
for the following reasons:

- Need to fetch the set from the string every time instead of reusing the C
  pointer.
- Using pipelining or multi-commands would result in more bytes sent by
  the client for the repeated `ZMSCORE KEY` sections.
- Need to specially encode the data and decode it from the client
  for lua-based solutions.
- The fastest solution I've seen for large sets(thousands or millions)
  involves lua and a variadic ZADD, then a ZINTERSECT, then a ZRANGE 0 -1,
  then UNLINK of a temporary set (or lua). This is still inefficient.

Co-authored-by: Tyson Andre <tysonandre775@hotmail.com>
---
 src/server.c              |  4 +++
 src/server.h              |  1 +
 src/t_zset.c              | 18 +++++++++++++
 tests/support/cluster.tcl |  2 +-
 tests/unit/type/zset.tcl  | 53 +++++++++++++++++++++++++++++++++++++++
 5 files changed, 77 insertions(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index afe5094c5..9d915ac3b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -470,6 +470,10 @@ struct redisCommand redisCommandTable[] = {
      "read-only fast @sortedset",
      0,NULL,1,1,1,0,0,0},
 
+    {"zmscore",zmscoreCommand,-3,
+     "read-only fast @sortedset",
+     0,NULL,1,1,1,0,0,0},
+
     {"zrank",zrankCommand,3,
      "read-only fast @sortedset",
      0,NULL,1,1,1,0,0,0},
diff --git a/src/server.h b/src/server.h
index adfac3293..018dd4607 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2321,6 +2321,7 @@ void zrevrangeCommand(client *c);
 void zcardCommand(client *c);
 void zremCommand(client *c);
 void zscoreCommand(client *c);
+void zmscoreCommand(client *c);
 void zremrangebyscoreCommand(client *c);
 void zremrangebylexCommand(client *c);
 void zpopminCommand(client *c);
diff --git a/src/t_zset.c b/src/t_zset.c
index 16131ee85..87a5a882e 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -3082,6 +3082,24 @@ void zscoreCommand(client *c) {
     }
 }
 
+void zmscoreCommand(client *c) {
+    robj *key = c->argv[1];
+    robj *zobj;
+    double score;
+    zobj = lookupKeyRead(c->db,key);
+    if (zobj != NULL && checkType(c,zobj,OBJ_ZSET)) return;
+
+    addReplyArrayLen(c,c->argc - 2);
+    for (int j = 2; j < c->argc; j++) {
+        /* Treat a missing set the same way as an empty set */
+        if (zobj == NULL || zsetScore(zobj,c->argv[j]->ptr,&score) == C_ERR) {
+            addReplyNull(c);
+        } else {
+            addReplyDouble(c,score);
+        }
+    }
+}
+
 void zrankGenericCommand(client *c, int reverse) {
     robj *key = c->argv[1];
     robj *ele = c->argv[2];
diff --git a/tests/support/cluster.tcl b/tests/support/cluster.tcl
index 64b079ff8..dced8d864 100644
--- a/tests/support/cluster.tcl
+++ b/tests/support/cluster.tcl
@@ -27,7 +27,7 @@ set ::redis_cluster::plain_commands {
     sadd srem sismember scard spop srandmember smembers sscan zadd
     zincrby zrem zremrangebyscore zremrangebyrank zremrangebylex zrange
     zrangebyscore zrevrangebyscore zrangebylex zrevrangebylex zcount
-    zlexcount zrevrange zcard zscore zrank zrevrank zscan hset hsetnx
+    zlexcount zrevrange zcard zscore zmscore zrank zrevrank zscan hset hsetnx
     hget hmset hmget hincrby hincrbyfloat hdel hlen hkeys hvals
     hgetall hexists hscan incrby decrby incrbyfloat getset move
     expire expireat pexpire pexpireat type ttl pttl persist restore
diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl
index a8c817f6e..d098840b0 100644
--- a/tests/unit/type/zset.tcl
+++ b/tests/unit/type/zset.tcl
@@ -780,6 +780,44 @@ start_server {tags {"zset"}} {
             set oldscore $score
         }
     }
+    
+    test {ZMSCORE retrieve} {
+        r del zmscoretest
+        r zadd zmscoretest 10 x
+        r zadd zmscoretest 20 y
+        
+        r zmscore zmscoretest x y
+    } {10 20}
+
+    test {ZMSCORE retrieve from empty set} {
+        r del zmscoretest
+        
+        r zmscore zmscoretest x y
+    } {{} {}}
+    
+    test {ZMSCORE retrieve with missing member} {
+        r del zmscoretest
+        r zadd zmscoretest 10 x
+        
+        r zmscore zmscoretest x y
+    } {10 {}}
+
+    test {ZMSCORE retrieve single member} {
+        r del zmscoretest
+        r zadd zmscoretest 10 x
+        r zadd zmscoretest 20 y
+        
+        r zmscore zmscoretest x
+    } {10}
+
+    test {ZMSCORE retrieve requires one or more members} {
+        r del zmscoretest
+        r zadd zmscoretest 10 x
+        r zadd zmscoretest 20 y
+        
+        catch {r zmscore zmscoretest} e
+        assert_match {*ERR*wrong*number*arg*} $e
+    }
 
     test "ZSET commands don't accept the empty strings as valid score" {
         assert_error "*not*float*" {r zadd myzset "" abc}
@@ -815,6 +853,21 @@ start_server {tags {"zset"}} {
             }
         }
 
+        test "ZMSCORE - $encoding" {
+            r del zscoretest
+            set aux {}
+            for {set i 0} {$i < $elements} {incr i} {
+                set score [expr rand()]
+                lappend aux $score
+                r zadd zscoretest $score $i
+            }
+
+            assert_encoding $encoding zscoretest
+            for {set i 0} {$i < $elements} {incr i} {
+                assert_equal [lindex $aux $i] [r zmscore zscoretest $i]
+            }
+        }
+
         test "ZSCORE after a DEBUG RELOAD - $encoding" {
             r del zscoretest
             set aux {}

From 40c7628fc815d24b43494bbf4ff59450926190bb Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Tue, 4 Aug 2020 23:00:58 +0800
Subject: [PATCH 0536/1098] Fix tests/cluster/cluster.tcl about wrong usage of
 lrange. (#6702)

---
 tests/cluster/cluster.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/cluster/cluster.tcl b/tests/cluster/cluster.tcl
index f9a0e180d..e894823de 100644
--- a/tests/cluster/cluster.tcl
+++ b/tests/cluster/cluster.tcl
@@ -21,7 +21,7 @@ proc get_cluster_nodes id {
             pong_recv [lindex $args 5] \
             config_epoch [lindex $args 6] \
             linkstate [lindex $args 7] \
-            slots [lrange $args 8 -1] \
+            slots [lrange $args 8 end] \
         ]
         lappend nodes $node
     }

From 3244fae7d4f2e77f6d0d3f8280e852bd63753462 Mon Sep 17 00:00:00 2001
From: Frank Meier <40453138+meierfra-ergon@users.noreply.github.com>
Date: Wed, 5 Aug 2020 11:06:33 +0200
Subject: [PATCH 0537/1098] add force option to 'create-cluster create' script
 call (#7612)

---
 utils/create-cluster/create-cluster | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster
index 931f6f521..d9a3b8c35 100755
--- a/utils/create-cluster/create-cluster
+++ b/utils/create-cluster/create-cluster
@@ -38,7 +38,11 @@ then
         PORT=$((PORT+1))
         HOSTS="$HOSTS $CLUSTER_HOST:$PORT"
     done
-    $BIN_PATH/redis-cli --cluster create $HOSTS --cluster-replicas $REPLICAS
+    OPT_ARG=""
+    if [ "$2" == "-f" ]; then
+        OPT_ARG="--cluster-yes"
+    fi
+    $BIN_PATH/redis-cli --cluster create $HOSTS --cluster-replicas $REPLICAS $OPT_ARG
     exit 0
 fi
 
@@ -104,7 +108,7 @@ fi
 
 echo "Usage: $0 [start|create|stop|watch|tail|clean|call]"
 echo "start       -- Launch Redis Cluster instances."
-echo "create      -- Create a cluster using redis-cli --cluster create."
+echo "create [-f] -- Create a cluster using redis-cli --cluster create."
 echo "stop        -- Stop Redis Cluster instances."
 echo "watch       -- Show CLUSTER NODES output (first 30 lines) of first node."
 echo "tail <id>   -- Run tail -f of instance at base port + ID."

From 51077c821236eba5223b1e624f2462214a0e5062 Mon Sep 17 00:00:00 2001
From: Frank Meier <frank.meier@ergon.ch>
Date: Thu, 28 Feb 2019 10:59:06 +0100
Subject: [PATCH 0538/1098] reintroduce REDISCLI_CLUSTER_YES env variable in
 redis-cli

the variable was introduced only in the 5.0 branch in #5879 bc6c1c40db
---
 src/redis-cli.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 0148964bf..ec7153d15 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -73,6 +73,7 @@
 #define REDIS_CLI_RCFILE_ENV "REDISCLI_RCFILE"
 #define REDIS_CLI_RCFILE_DEFAULT ".redisclirc"
 #define REDIS_CLI_AUTH_ENV "REDISCLI_AUTH"
+#define REDIS_CLI_CLUSTER_YES_ENV "REDISCLI_CLUSTER_YES"
 
 #define CLUSTER_MANAGER_SLOTS               16384
 #define CLUSTER_MANAGER_MIGRATE_TIMEOUT     60000
@@ -1668,6 +1669,11 @@ static void parseEnv() {
     if (auth != NULL && config.auth == NULL) {
         config.auth = auth;
     }
+
+    char *cluster_yes = getenv(REDIS_CLI_CLUSTER_YES_ENV);
+    if (cluster_yes != NULL && !strcmp(cluster_yes, "1")) {
+        config.cluster_manager_command.flags |= CLUSTER_MANAGER_CMD_FLAG_YES;
+    }
 }
 
 static sds readArgFromStdin(void) {

From 1aa31e4da9a2daf9058dd923738e331bc5c91e31 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 5 Aug 2020 12:18:44 +0300
Subject: [PATCH 0539/1098] redis-cli --cluster-yes - negate force flag for
 clarity

this internal flag is there so that some commands do not comply to `--cluster-yes`
---
 src/redis-cli.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index ec7153d15..de5d08149 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -1803,10 +1803,10 @@ static void usage(void) {
     exit(1);
 }
 
-static int confirmWithYes(char *msg, int force) {
-    /* if force is true and --cluster-yes option is on,
+static int confirmWithYes(char *msg, int ignore_force) {
+    /* if --cluster-yes option is set and ignore_force is false,
      * do not prompt for an answer */
-    if (force &&
+    if (!ignore_force &&
         (config.cluster_manager_command.flags & CLUSTER_MANAGER_CMD_FLAG_YES)) {
         return 1;
     }
@@ -4500,7 +4500,7 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) {
     dictReleaseIterator(iter);
 
     /* we want explicit manual confirmation from users for all the fix cases */
-    int force = 0;
+    int ignore_force = 1;
 
     /*  Handle case "1": keys in no node. */
     if (listLength(none) > 0) {
@@ -4508,7 +4508,7 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) {
                "across the cluster:\n");
         clusterManagerPrintSlotsList(none);
         if (confirmWithYes("Fix these slots by covering with a random node?",
-                           force)) {
+                           ignore_force)) {
             listIter li;
             listNode *ln;
             listRewind(none, &li);
@@ -4535,7 +4535,7 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) {
         printf("The following uncovered slots have keys in just one node:\n");
         clusterManagerPrintSlotsList(single);
         if (confirmWithYes("Fix these slots by covering with those nodes?",
-                           force)) {
+                           ignore_force)) {
             listIter li;
             listNode *ln;
             listRewind(single, &li);
@@ -4567,7 +4567,7 @@ static int clusterManagerFixSlotsCoverage(char *all_slots) {
         printf("The following uncovered slots have keys in multiple nodes:\n");
         clusterManagerPrintSlotsList(multi);
         if (confirmWithYes("Fix these slots by moving keys "
-                           "into a single node?", force)) {
+                           "into a single node?", ignore_force)) {
             listIter li;
             listNode *ln;
             listRewind(multi, &li);
@@ -5530,8 +5530,8 @@ assign_replicas:
     }
     clusterManagerOptimizeAntiAffinity(ip_nodes, ip_count);
     clusterManagerShowNodes();
-    int force = 1;
-    if (confirmWithYes("Can I set the above configuration?", force)) {
+    int ignore_force = 0;
+    if (confirmWithYes("Can I set the above configuration?", ignore_force)) {
         listRewind(cluster_manager.nodes, &li);
         while ((ln = listNext(&li)) != NULL) {
             clusterManagerNode *node = ln->value;

From e5a50ed3c49c211b413d92b1f45f950c830c58c0 Mon Sep 17 00:00:00 2001
From: ShooterIT <wangyuancode@163.com>
Date: Thu, 6 Aug 2020 15:36:28 +0800
Subject: [PATCH 0540/1098] [Redis-benchmark] Support zset type

---
 src/redis-benchmark.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index f47cbe333..ef43275b9 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -1733,6 +1733,22 @@ int main(int argc, const char **argv) {
             free(cmd);
         }
 
+        if (test_is_selected("zadd")) {
+            char *score = "0";
+            if (config.randomkeys) score = "__rand_int__";
+            len = redisFormatCommand(&cmd,
+                "ZADD myzset:{tag} %s element:__rand_int__",score);
+            benchmark("ZADD",cmd,len);
+            free(cmd);
+        }
+
+        if (test_is_selected("zrem")) {
+            len = redisFormatCommand(&cmd,
+                "ZREM myzset:{tag} element:__rand_int__");
+            benchmark("ZREM",cmd,len);
+            free(cmd);
+        }
+
         if (test_is_selected("lrange") ||
             test_is_selected("lrange_100") ||
             test_is_selected("lrange_300") ||

From 90b717e72340081ee87f0b85b4ef00b2a5bd2bf2 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 29 Jul 2020 17:05:14 +0300
Subject: [PATCH 0541/1098] Assertion and panic, print crash log without
 generating SIGSEGV

This makes it possible to add tests that generate assertions, and run
them with valgrind, making sure that there are no memory violations
prior to the assertion.

New config options:
- crash-log-enabled - can be disabled for cleaner core dumps
- crash-memcheck-enabled - useful for faster termination after a crash
- use-exit-on-panic - to be used by the test suite so that valgrind can
  detect leaks and memory corruptions

Other changes:
- Crash log is printed even on system that dont HAVE_BACKTRACE, i.e. in
  both SIGSEGV and assert / panic
- Assertion and panic won't print registers and code around EIP (which
  was useless), but will do fast memory test (which may still indicate
  that the assertion was due to memory corrpution)

I had to reshuffle code in order to re-use it, so i extracted come code
into function without actually doing any changes to the code:
- logServerInfo
- logModulesInfo
- doFastMemoryTest (with the exception of it being conditional)
- dumpCodeAroundEIP

changes to the crash report on segfault:
- logRegisters is called right after the stack trace (before info) done
  just in order to have more re-usable code
- stack trace skips the first two items on the stack (the crash log and
  signal handler functions)
---
 redis.conf   |  10 ++
 src/config.c |  14 ++-
 src/debug.c  | 261 ++++++++++++++++++++++++++++++++-------------------
 src/server.c |  21 +++--
 src/server.h |  11 ++-
 5 files changed, 205 insertions(+), 112 deletions(-)

diff --git a/redis.conf b/redis.conf
index d4e3e47f0..8440b4750 100644
--- a/redis.conf
+++ b/redis.conf
@@ -269,6 +269,16 @@ logfile ""
 # Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7.
 # syslog-facility local0
 
+# To disable the built in crash log, which will possibly produce cleaner core
+# dumps when they are needed, uncomment the following:
+#
+# crash-log-enabled no
+
+# To disable the fast memory check that's run as part of the crash log, which
+# will possibly let redis terminate sooner, uncomment the following:
+#
+# crash-memcheck-enabled no
+
 # Set the number of databases. The default database is DB 0, you can select
 # a different one on a per-connection basis using SELECT <dbid> where
 # dbid is a number between 0 and 'databases'-1
diff --git a/src/config.c b/src/config.c
index a68fddf1d..52a859a77 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2043,6 +2043,16 @@ static int updateAppendonly(int val, int prev, char **err) {
     return 1;
 }
 
+static int updateSighandlerEnabled(int val, int prev, char **err) {
+    UNUSED(err);
+    UNUSED(prev);
+    if (val)
+        setupSignalHandlers();
+    else
+        removeSignalHandlers();
+    return 1;
+}
+
 static int updateMaxclients(long long val, long long prev, char **err) {
     /* Try to check if the OS is capable of supporting so many FDs. */
     if (val > prev) {
@@ -2132,7 +2142,9 @@ standardConfig configs[] = {
     createBoolConfig("cluster-enabled", NULL, IMMUTABLE_CONFIG, server.cluster_enabled, 0, NULL, NULL),
     createBoolConfig("appendonly", NULL, MODIFIABLE_CONFIG, server.aof_enabled, 0, NULL, updateAppendonly),
     createBoolConfig("cluster-allow-reads-when-down", NULL, MODIFIABLE_CONFIG, server.cluster_allow_reads_when_down, 0, NULL, NULL),
-
+    createBoolConfig("crash-log-enabled", NULL, MODIFIABLE_CONFIG, server.crashlog_enabled, 1, NULL, updateSighandlerEnabled),
+    createBoolConfig("crash-memcheck-enabled", NULL, MODIFIABLE_CONFIG, server.memcheck_enabled, 1, NULL, NULL),
+    createBoolConfig("use-exit-on-panic", NULL, MODIFIABLE_CONFIG, server.use_exit_on_panic, 0, NULL, NULL),
 
     /* String Configs */
     createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, "", NULL, NULL),
diff --git a/src/debug.c b/src/debug.c
index 0bea69876..3b40e685b 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1,5 +1,6 @@
 /*
- * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2009-2020, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2020, Redis Labs, Inc
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -30,10 +31,13 @@
 #include "server.h"
 #include "sha1.h"   /* SHA1 is used for DEBUG DIGEST */
 #include "crc64.h"
+#include "bio.h"
 
 #include <arpa/inet.h>
 #include <signal.h>
 #include <dlfcn.h>
+#include <fcntl.h>
+#include <unistd.h>
 
 #ifdef HAVE_BACKTRACE
 #include <execinfo.h>
@@ -42,9 +46,6 @@
 #else
 typedef ucontext_t sigcontext_t;
 #endif
-#include <fcntl.h>
-#include "bio.h"
-#include <unistd.h>
 #endif /* HAVE_BACKTRACE */
 
 #ifdef __CYGWIN__
@@ -53,6 +54,15 @@ typedef ucontext_t sigcontext_t;
 #endif
 #endif
 
+/* Globals */
+static int bug_report_start = 0; /* True if bug report header was already logged. */
+
+/* Forward declarations */
+void bugReportStart(void);
+void printCrashReport(void);
+void bugReportEnd(int killViaSignal, int sig);
+void logStackTrace(void *eip, int uplevel);
+
 /* ================================= Debugging ============================== */
 
 /* Compute the sha1 of string at 's' with 'len' bytes long.
@@ -814,13 +824,14 @@ void _serverAssert(const char *estr, const char *file, int line) {
     bugReportStart();
     serverLog(LL_WARNING,"=== ASSERTION FAILED ===");
     serverLog(LL_WARNING,"==> %s:%d '%s' is not true",file,line,estr);
+
+    if (server.crashlog_enabled) {
 #ifdef HAVE_BACKTRACE
-    server.assert_failed = estr;
-    server.assert_file = file;
-    server.assert_line = line;
-    serverLog(LL_WARNING,"(forcing SIGSEGV to print the bug report.)");
+        logStackTrace(NULL, 1);
 #endif
-    *((char*)-1) = 'x';
+        printCrashReport();
+    }
+    bugReportEnd(0, 0);
 }
 
 void _serverAssertPrintClientInfo(const client *c) {
@@ -897,18 +908,21 @@ void _serverPanic(const char *file, int line, const char *msg, ...) {
     serverLog(LL_WARNING,"------------------------------------------------");
     serverLog(LL_WARNING,"!!! Software Failure. Press left mouse button to continue");
     serverLog(LL_WARNING,"Guru Meditation: %s #%s:%d",fmtmsg,file,line);
+
+    if (server.crashlog_enabled) {
 #ifdef HAVE_BACKTRACE
-    serverLog(LL_WARNING,"(forcing SIGSEGV in order to print the stack trace)");
+        logStackTrace(NULL, 1);
 #endif
-    serverLog(LL_WARNING,"------------------------------------------------");
-    *((char*)-1) = 'x';
+        printCrashReport();
+    }
+    bugReportEnd(0, 0);
 }
 
 void bugReportStart(void) {
-    if (server.bug_report_start == 0) {
+    if (bug_report_start == 0) {
         serverLogRaw(LL_WARNING|LL_RAW,
         "\n\n=== REDIS BUG REPORT START: Cut & paste starting from here ===\n");
-        server.bug_report_start = 1;
+        bug_report_start = 1;
     }
 }
 
@@ -980,6 +994,7 @@ void logStackContent(void **sp) {
     }
 }
 
+/* Log dump of processor registers */
 void logRegisters(ucontext_t *uc) {
     serverLog(LL_WARNING|LL_RAW, "\n------ REGISTERS ------\n");
 
@@ -1352,6 +1367,8 @@ void logRegisters(ucontext_t *uc) {
 #endif
 }
 
+#endif /* HAVE_BACKTRACE */
+
 /* Return a file descriptor to write directly to the Redis log with the
  * write(2) syscall, that can be used in critical sections of the code
  * where the rest of Redis can't be trusted (for example during the memory
@@ -1372,33 +1389,66 @@ void closeDirectLogFiledes(int fd) {
     if (!log_to_stdout) close(fd);
 }
 
+#ifdef HAVE_BACKTRACE
+
 /* Logs the stack trace using the backtrace() call. This function is designed
- * to be called from signal handlers safely. */
-void logStackTrace(ucontext_t *uc) {
-    void *trace[101];
+ * to be called from signal handlers safely.
+ * The eip argument is optional (can take NULL).
+ * The uplevel argument indicates how many of the calling functions to skip.
+ */
+void logStackTrace(void *eip, int uplevel) {
+    void *trace[100];
     int trace_size = 0, fd = openDirectLogFiledes();
+    char *msg;
+    uplevel++; /* skip this function */
 
     if (fd == -1) return; /* If we can't log there is anything to do. */
 
-    /* Generate the stack trace */
-    trace_size = backtrace(trace+1, 100);
+    /* Get the stack trace first! */
+    trace_size = backtrace(trace, 100);
 
-    if (getMcontextEip(uc) != NULL) {
-        char *msg1 = "EIP:\n";
-        char *msg2 = "\nBacktrace:\n";
-        if (write(fd,msg1,strlen(msg1)) == -1) {/* Avoid warning. */};
-        trace[0] = getMcontextEip(uc);
-        backtrace_symbols_fd(trace, 1, fd);
-        if (write(fd,msg2,strlen(msg2)) == -1) {/* Avoid warning. */};
+    msg = "\n------ STACK TRACE ------\n";
+    if (write(fd,msg,strlen(msg)) == -1) {/* Avoid warning. */};
+
+    if (eip) {
+        /* Write EIP to the log file*/
+        msg = "EIP:\n";
+        if (write(fd,msg,strlen(msg)) == -1) {/* Avoid warning. */};
+        backtrace_symbols_fd(&eip, 1, fd);
     }
 
     /* Write symbols to log file */
-    backtrace_symbols_fd(trace+1, trace_size, fd);
+    msg = "\nBacktrace:\n";
+    if (write(fd,msg,strlen(msg)) == -1) {/* Avoid warning. */};
+    backtrace_symbols_fd(trace+uplevel, trace_size-uplevel, fd);
 
     /* Cleanup */
     closeDirectLogFiledes(fd);
 }
 
+#endif /* HAVE_BACKTRACE */
+
+/* Log global server info */
+void logServerInfo(void) {
+    sds infostring, clients;
+    serverLogRaw(LL_WARNING|LL_RAW, "\n------ INFO OUTPUT ------\n");
+    infostring = genRedisInfoString("all");
+    serverLogRaw(LL_WARNING|LL_RAW, infostring);
+    serverLogRaw(LL_WARNING|LL_RAW, "\n------ CLIENT LIST OUTPUT ------\n");
+    clients = getAllClientsInfoString(-1);
+    serverLogRaw(LL_WARNING|LL_RAW, clients);
+    sdsfree(infostring);
+    sdsfree(clients);
+}
+
+/* Log modules info. Something we wanna do last since we fear it may crash. */
+void logModulesInfo(void) {
+    serverLogRaw(LL_WARNING|LL_RAW, "\n------ MODULES INFO OUTPUT ------\n");
+    sds infostring = modulesCollectInfo(sdsempty(), NULL, 1, 0);
+    serverLogRaw(LL_WARNING|LL_RAW, infostring);
+    sdsfree(infostring);
+}
+
 /* Log information about the "current" client, that is, the client that is
  * currently being served by Redis. May be NULL if Redis is not serving a
  * client right now. */
@@ -1505,6 +1555,23 @@ int memtest_test_linux_anonymous_maps(void) {
 }
 #endif
 
+void doFastMemoryTest(void) {
+#if defined(HAVE_PROC_MAPS)
+    if (server.memcheck_enabled) {
+        /* Test memory */
+        serverLogRaw(LL_WARNING|LL_RAW, "\n------ FAST MEMORY TEST ------\n");
+        bioKillThreads();
+        if (memtest_test_linux_anonymous_maps()) {
+            serverLogRaw(LL_WARNING|LL_RAW,
+                "!!! MEMORY ERROR DETECTED! Check your memory ASAP !!!\n");
+        } else {
+            serverLogRaw(LL_WARNING|LL_RAW,
+                "Fast memory test PASSED, however your memory can still be broken. Please run a memory test for several hours if possible.\n");
+        }
+    }
+#endif
+}
+
 /* Scans the (assumed) x86 code starting at addr, for a max of `len`
  * bytes, searching for E8 (callq) opcodes, and dumping the symbols
  * and the call offset if they appear to be valid. */
@@ -1531,95 +1598,86 @@ void dumpX86Calls(void *addr, size_t len) {
     }
 }
 
+void dumpCodeAroundEIP(void *eip) {
+    Dl_info info;
+    if (dladdr(eip, &info) != 0) {
+        serverLog(LL_WARNING|LL_RAW,
+            "\n------ DUMPING CODE AROUND EIP ------\n"
+            "Symbol: %s (base: %p)\n"
+            "Module: %s (base %p)\n"
+            "$ xxd -r -p /tmp/dump.hex /tmp/dump.bin\n"
+            "$ objdump --adjust-vma=%p -D -b binary -m i386:x86-64 /tmp/dump.bin\n"
+            "------\n",
+            info.dli_sname, info.dli_saddr, info.dli_fname, info.dli_fbase,
+            info.dli_saddr);
+        size_t len = (long)eip - (long)info.dli_saddr;
+        unsigned long sz = sysconf(_SC_PAGESIZE);
+        if (len < 1<<13) { /* we don't have functions over 8k (verified) */
+            /* Find the address of the next page, which is our "safety"
+             * limit when dumping. Then try to dump just 128 bytes more
+             * than EIP if there is room, or stop sooner. */
+            unsigned long next = ((unsigned long)eip + sz) & ~(sz-1);
+            unsigned long end = (unsigned long)eip + 128;
+            if (end > next) end = next;
+            len = end - (unsigned long)info.dli_saddr;
+            serverLogHexDump(LL_WARNING, "dump of function",
+                info.dli_saddr ,len);
+            dumpX86Calls(info.dli_saddr,len);
+        }
+    }
+}
+
 void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
-    ucontext_t *uc = (ucontext_t*) secret;
-    void *eip = getMcontextEip(uc);
-    sds infostring, clients;
-    struct sigaction act;
+    UNUSED(secret);
     UNUSED(info);
 
     bugReportStart();
     serverLog(LL_WARNING,
         "Redis %s crashed by signal: %d", REDIS_VERSION, sig);
-    if (eip != NULL) {
-        serverLog(LL_WARNING,
-        "Crashed running the instruction at: %p", eip);
-    }
     if (sig == SIGSEGV || sig == SIGBUS) {
         serverLog(LL_WARNING,
         "Accessing address: %p", (void*)info->si_addr);
     }
-    serverLog(LL_WARNING,
-        "Failed assertion: %s (%s:%d)", server.assert_failed,
-                        server.assert_file, server.assert_line);
 
-    /* Log the stack trace */
-    serverLogRaw(LL_WARNING|LL_RAW, "\n------ STACK TRACE ------\n");
-    logStackTrace(uc);
+#ifdef HAVE_BACKTRACE
+    ucontext_t *uc = (ucontext_t*) secret;
+    void *eip = getMcontextEip(uc);
+    if (eip != NULL) {
+        serverLog(LL_WARNING,
+        "Crashed running the instruction at: %p", eip);
+    }
 
+    logStackTrace(getMcontextEip(uc), 1);
+
+    logRegisters(uc);
+#endif
+
+    printCrashReport();
+
+#ifdef HAVE_BACKTRACE
+    if (eip != NULL)
+        dumpCodeAroundEIP(eip);
+#endif
+
+    bugReportEnd(1, sig);
+}
+
+void printCrashReport(void) {
     /* Log INFO and CLIENT LIST */
-    serverLogRaw(LL_WARNING|LL_RAW, "\n------ INFO OUTPUT ------\n");
-    infostring = genRedisInfoString("all");
-    serverLogRaw(LL_WARNING|LL_RAW, infostring);
-    serverLogRaw(LL_WARNING|LL_RAW, "\n------ CLIENT LIST OUTPUT ------\n");
-    clients = getAllClientsInfoString(-1);
-    serverLogRaw(LL_WARNING|LL_RAW, clients);
-    sdsfree(infostring);
-    sdsfree(clients);
+    logServerInfo();
 
     /* Log the current client */
     logCurrentClient();
 
-    /* Log dump of processor registers */
-    logRegisters(uc);
+    /* Log modules info. Something we wanna do last since we fear it may crash. */
+    logModulesInfo();
 
-    /* Log Modules INFO */
-    serverLogRaw(LL_WARNING|LL_RAW, "\n------ MODULES INFO OUTPUT ------\n");
-    infostring = modulesCollectInfo(sdsempty(), NULL, 1, 0);
-    serverLogRaw(LL_WARNING|LL_RAW, infostring);
-    sdsfree(infostring);
+    /* Run memory test in case the crash was triggered by memory corruption. */
+    doFastMemoryTest();
+}
 
-#if defined(HAVE_PROC_MAPS)
-    /* Test memory */
-    serverLogRaw(LL_WARNING|LL_RAW, "\n------ FAST MEMORY TEST ------\n");
-    bioKillThreads();
-    if (memtest_test_linux_anonymous_maps()) {
-        serverLogRaw(LL_WARNING|LL_RAW,
-            "!!! MEMORY ERROR DETECTED! Check your memory ASAP !!!\n");
-    } else {
-        serverLogRaw(LL_WARNING|LL_RAW,
-            "Fast memory test PASSED, however your memory can still be broken. Please run a memory test for several hours if possible.\n");
-    }
-#endif
-
-    if (eip != NULL) {
-        Dl_info info;
-        if (dladdr(eip, &info) != 0) {
-            serverLog(LL_WARNING|LL_RAW,
-                "\n------ DUMPING CODE AROUND EIP ------\n"
-                "Symbol: %s (base: %p)\n"
-                "Module: %s (base %p)\n"
-                "$ xxd -r -p /tmp/dump.hex /tmp/dump.bin\n"
-                "$ objdump --adjust-vma=%p -D -b binary -m i386:x86-64 /tmp/dump.bin\n"
-                "------\n",
-                info.dli_sname, info.dli_saddr, info.dli_fname, info.dli_fbase,
-                info.dli_saddr);
-            size_t len = (long)eip - (long)info.dli_saddr;
-            unsigned long sz = sysconf(_SC_PAGESIZE);
-            if (len < 1<<13) { /* we don't have functions over 8k (verified) */
-                /* Find the address of the next page, which is our "safety"
-                 * limit when dumping. Then try to dump just 128 bytes more
-                 * than EIP if there is room, or stop sooner. */
-                unsigned long next = ((unsigned long)eip + sz) & ~(sz-1);
-                unsigned long end = (unsigned long)eip + 128;
-                if (end > next) end = next;
-                len = end - (unsigned long)info.dli_saddr;
-                serverLogHexDump(LL_WARNING, "dump of function",
-                    info.dli_saddr ,len);
-                dumpX86Calls(info.dli_saddr,len);
-            }
-        }
-    }
+void bugReportEnd(int killViaSignal, int sig) {
+    struct sigaction act;
 
     serverLogRaw(LL_WARNING|LL_RAW,
 "\n=== REDIS BUG REPORT END. Make sure to include from START to END. ===\n\n"
@@ -1631,6 +1689,12 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
     /* free(messages); Don't call free() with possibly corrupted memory. */
     if (server.daemonize && server.supervised == 0) unlink(server.pidfile);
 
+    if (!killViaSignal) {
+        if (server.use_exit_on_panic)
+            exit(1);
+        abort();
+    }
+
     /* Make sure we exit with the right signal at the end. So for instance
      * the core will be dumped if enabled. */
     sigemptyset (&act.sa_mask);
@@ -1639,7 +1703,6 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
     sigaction (sig, &act, NULL);
     kill(getpid(),sig);
 }
-#endif /* HAVE_BACKTRACE */
 
 /* ==================== Logging functions for debugging ===================== */
 
@@ -1679,7 +1742,7 @@ void watchdogSignalHandler(int sig, siginfo_t *info, void *secret) {
 
     serverLogFromHandler(LL_WARNING,"\n--- WATCHDOG TIMER EXPIRED ---");
 #ifdef HAVE_BACKTRACE
-    logStackTrace(uc);
+    logStackTrace(getMcontextEip(uc), 1);
 #else
     serverLogFromHandler(LL_WARNING,"Sorry: no support for backtrace().");
 #endif
diff --git a/src/server.c b/src/server.c
index 3b45bea59..45db1cbcf 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2442,10 +2442,6 @@ void initServerConfig(void) {
     server.rpoplpushCommand = lookupCommandByCString("rpoplpush");
 
     /* Debugging */
-    server.assert_failed = "<no assertion failed>";
-    server.assert_file = "<no file>";
-    server.assert_line = 0;
-    server.bug_report_start = 0;
     server.watchdog_period = 0;
 
     /* By default we want scripts to be always replicated by effects
@@ -4812,16 +4808,27 @@ void setupSignalHandlers(void) {
     sigaction(SIGTERM, &act, NULL);
     sigaction(SIGINT, &act, NULL);
 
-#ifdef HAVE_BACKTRACE
     sigemptyset(&act.sa_mask);
     act.sa_flags = SA_NODEFER | SA_RESETHAND | SA_SIGINFO;
     act.sa_sigaction = sigsegvHandler;
+    if(server.crashlog_enabled) {
+        sigaction(SIGSEGV, &act, NULL);
+        sigaction(SIGBUS, &act, NULL);
+        sigaction(SIGFPE, &act, NULL);
+        sigaction(SIGILL, &act, NULL);
+    }
+    return;
+}
+
+void removeSignalHandlers(void) {
+    struct sigaction act;
+    sigemptyset(&act.sa_mask);
+    act.sa_flags = SA_NODEFER | SA_RESETHAND;
+    act.sa_handler = SIG_DFL;
     sigaction(SIGSEGV, &act, NULL);
     sigaction(SIGBUS, &act, NULL);
     sigaction(SIGFPE, &act, NULL);
     sigaction(SIGILL, &act, NULL);
-#endif
-    return;
 }
 
 /* This is the signal handler for children process. It is currently useful
diff --git a/src/server.h b/src/server.h
index 018dd4607..efb7d2ee8 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1274,6 +1274,11 @@ struct redisServer {
     int syslog_enabled;             /* Is syslog enabled? */
     char *syslog_ident;             /* Syslog ident */
     int syslog_facility;            /* Syslog facility */
+    int crashlog_enabled;           /* Enable signal handler for crashlog.
+                                     * disable for clean core dumps. */
+    int memcheck_enabled;           /* Enable memory check on crash. */
+    int use_exit_on_panic;          /* Use exit() on panic and assert rather than
+                                     * abort(). useful for Valgrind. */
     /* Replication (master) */
     char replid[CONFIG_RUN_ID_SIZE+1];  /* My current replication ID. */
     char replid2[CONFIG_RUN_ID_SIZE+1]; /* replid inherited from master*/
@@ -1439,10 +1444,6 @@ struct redisServer {
                                old "requirepass" directive for backward
                                compatibility with Redis <= 5. */
     /* Assert & bug reporting */
-    const char *assert_failed;
-    const char *assert_file;
-    int assert_line;
-    int bug_report_start; /* True if bug report header was already logged. */
     int watchdog_period;  /* Software watchdog period in ms. 0 = off */
     /* System hardware info */
     size_t system_memory_size;  /* Total memory in system as reported by OS */
@@ -1975,6 +1976,7 @@ int freeMemoryIfNeeded(void);
 int freeMemoryIfNeededAndSafe(void);
 int processCommand(client *c);
 void setupSignalHandlers(void);
+void removeSignalHandlers(void);
 struct redisCommand *lookupCommand(sds name);
 struct redisCommand *lookupCommandByCString(char *s);
 struct redisCommand *lookupCommandOrOriginal(sds name);
@@ -2432,7 +2434,6 @@ void *realloc(void *ptr, size_t size) __attribute__ ((deprecated));
 void _serverAssertWithInfo(const client *c, const robj *o, const char *estr, const char *file, int line);
 void _serverAssert(const char *estr, const char *file, int line);
 void _serverPanic(const char *file, int line, const char *msg, ...);
-void bugReportStart(void);
 void serverLogObjectDebugInfo(const robj *o);
 void sigsegvHandler(int sig, siginfo_t *info, void *secret);
 sds genRedisInfoString(const char *section);

From 81f8524a1282d4ea192fd0b0ee5e3f6cd2e673d3 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 4 Aug 2020 12:24:45 +0300
Subject: [PATCH 0542/1098] Fix potential race in bugReportStart

this race would only happen when two threads paniced at the same time,
and even then the only consequence is some extra log lines.

race reported in #7391
---
 src/debug.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/debug.c b/src/debug.c
index 3b40e685b..5cfda59c9 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -55,7 +55,7 @@ typedef ucontext_t sigcontext_t;
 #endif
 
 /* Globals */
-static int bug_report_start = 0; /* True if bug report header was already logged. */
+static _Atomic int bug_report_start = 0; /* True if bug report header was already logged. */
 
 /* Forward declarations */
 void bugReportStart(void);

From c17e597d05262e9c2dfb8f7045a02f2f5d0c5d8a Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 6 Aug 2020 16:53:06 +0300
Subject: [PATCH 0543/1098] Accelerate diskless master connections, and general
 re-connections (#6271)

Diskless master has some inherent latencies.
1) fork starts with delay from cron rather than immediately
2) replica is put online only after an ACK. but the ACK
   was sent only once a second.
3) but even if it would arrive immediately, it will not
   register in case cron didn't yet detect that the fork is done.

Besides that, when a replica disconnects, it doesn't immediately
attempts to re-connect, it waits for replication cron (one per second).
in case it was already online, it may be important to try to re-connect
as soon as possible, so that the backlog at the master doesn't vanish.

In case it disconnected during rdb transfer, one can argue that it's
not very important to re-connect immediately, but this is needed for the
"diskless loading short read" test to be able to run 100 iterations in 5
seconds, rather than 3 (waiting for replication cron re-connection)

changes in this commit:
1) sync command starts a fork immediately if no sync_delay is configured
2) replica sends REPLCONF ACK when done reading the rdb (rather than on 1s cron)
3) when a replica unexpectedly disconnets, it immediately tries to
   re-connect rather than waiting 1s
4) when when a child exits, if there is another replica waiting, we spawn a new
   one right away, instead of waiting for 1s replicationCron.
5) added a call to connectWithMaster from replicationSetMaster. which is called
   from the REPLICAOF command but also in 3 places in cluster.c, in all of
   these the connection attempt will now be immediate instead of delayed by 1
   second.

side note:
we can add a call to rdbPipeReadHandler in replconfCommand when getting
a REPLCONF ACK from the replica to solve a race where the replica got
the entire rdb and EOF marker before we detected that the pipe was
closed.
in the test i did see this race happens in one about of some 300 runs,
but i concluded that this race is unlikely in real life (where the
replica is on another host and we're more likely to first detect the
pipe was closed.
the test runs 100 iterations in 3 seconds, so in some cases it'll take 4
seconds instead (waiting for another REPLCONF ACK).

Removing unneeded startBgsaveForReplication from updateSlavesWaitingForBgsave
Now that CheckChildrenDone is calling the new replicationStartPendingFork
(extracted from serverCron) there's actually no need to call
startBgsaveForReplication from updateSlavesWaitingForBgsave anymore,
since as soon as updateSlavesWaitingForBgsave returns, CheckChildrenDone is
calling replicationStartPendingFork that handles that anyway.
The code in updateSlavesWaitingForBgsave had a bug in which it ignored
repl-diskless-sync-delay, but removing that code shows that this bug was
hiding another bug, which is that the max_idle should have used >= and
not >, this one second delay has a big impact on my new test.
---
 src/replication.c                 | 97 ++++++++++++++++++++-----------
 src/server.c                      |  3 +
 src/server.h                      |  1 +
 tests/integration/replication.tcl | 21 ++++++-
 tests/unit/moduleapi/testrdb.tcl  | 21 ++++++-
 5 files changed, 105 insertions(+), 38 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index a8f46be95..05d450558 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -43,7 +43,7 @@ void replicationDiscardCachedMaster(void);
 void replicationResurrectCachedMaster(connection *conn);
 void replicationSendAck(void);
 void putSlaveOnline(client *slave);
-int cancelReplicationHandshake(void);
+int cancelReplicationHandshake(int reconnect);
 
 /* We take a global flag to remember if this instance generated an RDB
  * because of replication, so that we can remove the RDB file in case
@@ -832,6 +832,8 @@ void syncCommand(client *c) {
              * few seconds to wait for more slaves to arrive. */
             if (server.repl_diskless_sync_delay)
                 serverLog(LL_NOTICE,"Delay next BGSAVE for diskless SYNC");
+            else
+                startBgsaveForReplication(c->slave_capa);
         } else {
             /* Target is disk (or the slave is not capable of supporting
              * diskless replication) and we don't have a BGSAVE in progress,
@@ -1225,19 +1227,13 @@ void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData,
  * (if it had a disk or socket target). */
 void updateSlavesWaitingBgsave(int bgsaveerr, int type) {
     listNode *ln;
-    int startbgsave = 0;
-    int mincapa = -1;
     listIter li;
 
     listRewind(server.slaves,&li);
     while((ln = listNext(&li))) {
         client *slave = ln->value;
 
-        if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START) {
-            startbgsave = 1;
-            mincapa = (mincapa == -1) ? slave->slave_capa :
-                                        (mincapa & slave->slave_capa);
-        } else if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END) {
+        if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END) {
             struct redis_stat buf;
 
             if (bgsaveerr != C_OK) {
@@ -1304,7 +1300,6 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type) {
             }
         }
     }
-    if (startbgsave) startBgsaveForReplication(mincapa);
 }
 
 /* Change the current instance replication ID with a new, random one.
@@ -1568,7 +1563,7 @@ void readSyncBulkPayload(connection *conn) {
             }
             serverLog(LL_WARNING,"I/O error trying to sync with MASTER: %s",
                 (nread == -1) ? strerror(errno) : "connection lost");
-            cancelReplicationHandshake();
+            cancelReplicationHandshake(1);
             return;
         }
         server.stat_net_input_bytes += nread;
@@ -1695,7 +1690,7 @@ void readSyncBulkPayload(connection *conn) {
             serverLog(LL_WARNING,
                 "Failed trying to load the MASTER synchronization DB "
                 "from socket");
-            cancelReplicationHandshake();
+            cancelReplicationHandshake(1);
             rioFreeConn(&rdb, NULL);
             if (server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB) {
                 /* Restore the backed up databases. */
@@ -1728,7 +1723,7 @@ void readSyncBulkPayload(connection *conn) {
                 memcmp(buf,eofmark,CONFIG_RUN_ID_SIZE) != 0)
             {
                 serverLog(LL_WARNING,"Replication stream EOF marker is broken");
-                cancelReplicationHandshake();
+                cancelReplicationHandshake(1);
                 rioFreeConn(&rdb, NULL);
                 return;
             }
@@ -1758,7 +1753,7 @@ void readSyncBulkPayload(connection *conn) {
                 "Failed trying to rename the temp DB into %s in "
                 "MASTER <-> REPLICA synchronization: %s",
                 server.rdb_filename, strerror(errno));
-            cancelReplicationHandshake();
+            cancelReplicationHandshake(1);
             if (old_rdb_fd != -1) close(old_rdb_fd);
             return;
         }
@@ -1769,7 +1764,7 @@ void readSyncBulkPayload(connection *conn) {
             serverLog(LL_WARNING,
                 "Failed trying to load the MASTER synchronization "
                 "DB from disk");
-            cancelReplicationHandshake();
+            cancelReplicationHandshake(1);
             if (server.rdb_del_sync_files && allPersistenceDisabled()) {
                 serverLog(LL_NOTICE,"Removing the RDB file obtained from "
                                     "the master. This replica has persistence "
@@ -1824,6 +1819,9 @@ void readSyncBulkPayload(connection *conn) {
         redisCommunicateSystemd("READY=1\n");
     }
 
+    /* Send the initial ACK immediately to put this replica in online state. */
+    if (usemark) replicationSendAck();
+
     /* Restart the AOF subsystem now that we finished the sync. This
      * will trigger an AOF rewrite, and when done will start appending
      * to the new file. */
@@ -1831,7 +1829,7 @@ void readSyncBulkPayload(connection *conn) {
     return;
 
 error:
-    cancelReplicationHandshake();
+    cancelReplicationHandshake(1);
     return;
 }
 
@@ -2421,6 +2419,7 @@ int connectWithMaster(void) {
 
     server.repl_transfer_lastio = server.unixtime;
     server.repl_state = REPL_STATE_CONNECTING;
+    serverLog(LL_NOTICE,"MASTER <-> REPLICA sync started");
     return C_OK;
 }
 
@@ -2456,7 +2455,7 @@ void replicationAbortSyncTransfer(void) {
  * the replication state (server.repl_state) set to REPL_STATE_CONNECT.
  *
  * Otherwise zero is returned and no operation is perforemd at all. */
-int cancelReplicationHandshake(void) {
+int cancelReplicationHandshake(int reconnect) {
     if (server.repl_state == REPL_STATE_TRANSFER) {
         replicationAbortSyncTransfer();
         server.repl_state = REPL_STATE_CONNECT;
@@ -2468,6 +2467,16 @@ int cancelReplicationHandshake(void) {
     } else {
         return 0;
     }
+
+    if (!reconnect)
+        return 1;
+
+    /* try to re-connect without waiting for replicationCron, this is needed
+     * for the "diskless loading short read" test. */
+    serverLog(LL_NOTICE,"Reconnecting to MASTER %s:%d after failure",
+        server.masterhost, server.masterport);
+    connectWithMaster();
+
     return 1;
 }
 
@@ -2476,17 +2485,22 @@ void replicationSetMaster(char *ip, int port) {
     int was_master = server.masterhost == NULL;
 
     sdsfree(server.masterhost);
-    server.masterhost = sdsnew(ip);
-    server.masterport = port;
+    server.masterhost = NULL;
     if (server.master) {
         freeClient(server.master);
     }
     disconnectAllBlockedClients(); /* Clients blocked in master, now slave. */
 
+    /* Setting masterhost only after the call to freeClient since it calls
+     * replicationHandleMasterDisconnection which can trigger a re-connect
+     * directly from within that call. */
+    server.masterhost = sdsnew(ip);
+    server.masterport = port;
+
     /* Force our slaves to resync with us as well. They may hopefully be able
      * to partially resync with us, but we can notify the replid change. */
     disconnectSlaves();
-    cancelReplicationHandshake();
+    cancelReplicationHandshake(0);
     /* Before destroying our master state, create a cached master using
      * our own parameters, to later PSYNC with the new master. */
     if (was_master) {
@@ -2506,6 +2520,9 @@ void replicationSetMaster(char *ip, int port) {
                               NULL);
 
     server.repl_state = REPL_STATE_CONNECT;
+    serverLog(LL_NOTICE,"Connecting to MASTER %s:%d",
+        server.masterhost, server.masterport);
+    connectWithMaster();
 }
 
 /* Cancel replication, setting the instance as a master itself. */
@@ -2518,11 +2535,13 @@ void replicationUnsetMaster(void) {
                               REDISMODULE_SUBEVENT_MASTER_LINK_DOWN,
                               NULL);
 
+    /* Clear masterhost first, since the freeClient calls
+     * replicationHandleMasterDisconnection which can attempt to re-connect. */
     sdsfree(server.masterhost);
     server.masterhost = NULL;
     if (server.master) freeClient(server.master);
     replicationDiscardCachedMaster();
-    cancelReplicationHandshake();
+    cancelReplicationHandshake(0);
     /* When a slave is turned into a master, the current replication ID
      * (that was inherited from the master at synchronization time) is
      * used as secondary ID up to the current offset, and a new replication
@@ -2576,6 +2595,14 @@ void replicationHandleMasterDisconnection(void) {
     /* We lost connection with our master, don't disconnect slaves yet,
      * maybe we'll be able to PSYNC with our master later. We'll disconnect
      * the slaves only if we'll have to do a full resync with our master. */
+
+    /* Try to re-connect immediately rather than wait for replicationCron
+     * waiting 1 second may risk backlog being recycled. */
+    if (server.masterhost) {
+        serverLog(LL_NOTICE,"Reconnecting to MASTER %s:%d",
+            server.masterhost, server.masterport);
+        connectWithMaster();
+    }
 }
 
 void replicaofCommand(client *c) {
@@ -3121,7 +3148,7 @@ void replicationCron(void) {
          (time(NULL)-server.repl_transfer_lastio) > server.repl_timeout)
     {
         serverLog(LL_WARNING,"Timeout connecting to the MASTER...");
-        cancelReplicationHandshake();
+        cancelReplicationHandshake(1);
     }
 
     /* Bulk transfer I/O timeout? */
@@ -3129,7 +3156,7 @@ void replicationCron(void) {
         (time(NULL)-server.repl_transfer_lastio) > server.repl_timeout)
     {
         serverLog(LL_WARNING,"Timeout receiving bulk data from MASTER... If the problem persists try to set the 'repl-timeout' parameter in redis.conf to a larger value.");
-        cancelReplicationHandshake();
+        cancelReplicationHandshake(1);
     }
 
     /* Timed out master when we are an already connected slave? */
@@ -3144,9 +3171,7 @@ void replicationCron(void) {
     if (server.repl_state == REPL_STATE_CONNECT) {
         serverLog(LL_NOTICE,"Connecting to MASTER %s:%d",
             server.masterhost, server.masterport);
-        if (connectWithMaster() == C_OK) {
-            serverLog(LL_NOTICE,"MASTER <-> REPLICA sync started");
-        }
+        connectWithMaster();
     }
 
     /* Send ACK to master from time to time.
@@ -3280,6 +3305,18 @@ void replicationCron(void) {
         replicationScriptCacheFlush();
     }
 
+    replicationStartPendingFork();
+
+    /* Remove the RDB file used for replication if Redis is not running
+     * with any persistence. */
+    removeRDBUsedToSyncReplicas();
+
+    /* Refresh the number of slaves with lag <= min-slaves-max-lag. */
+    refreshGoodSlavesCount();
+    replication_cron_loops++; /* Incremented with frequency 1 HZ. */
+}
+
+void replicationStartPendingFork(void) {
     /* Start a BGSAVE good for replication if we have slaves in
      * WAIT_BGSAVE_START state.
      *
@@ -3307,7 +3344,7 @@ void replicationCron(void) {
 
         if (slaves_waiting &&
             (!server.repl_diskless_sync ||
-             max_idle > server.repl_diskless_sync_delay))
+             max_idle >= server.repl_diskless_sync_delay))
         {
             /* Start the BGSAVE. The called function may start a
              * BGSAVE with socket target or disk target depending on the
@@ -3315,12 +3352,4 @@ void replicationCron(void) {
             startBgsaveForReplication(mincapa);
         }
     }
-
-    /* Remove the RDB file used for replication if Redis is not running
-     * with any persistence. */
-    removeRDBUsedToSyncReplicas();
-
-    /* Refresh the number of slaves with lag <= min-slaves-max-lag. */
-    refreshGoodSlavesCount();
-    replication_cron_loops++; /* Incremented with frequency 1 HZ. */
 }
diff --git a/src/server.c b/src/server.c
index 45db1cbcf..c9d995e0b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1828,6 +1828,9 @@ void checkChildrenDone(void) {
         }
         updateDictResizePolicy();
         closeChildInfoPipe();
+
+        /* start any pending forks immediately. */
+        replicationStartPendingFork();
     }
 }
 
diff --git a/src/server.h b/src/server.h
index efb7d2ee8..71bbfec7b 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1807,6 +1807,7 @@ void replicationFeedSlavesFromMasterStream(list *slaves, char *buf, size_t bufle
 void replicationFeedMonitors(client *c, list *monitors, int dictid, robj **argv, int argc);
 void updateSlavesWaitingBgsave(int bgsaveerr, int type);
 void replicationCron(void);
+void replicationStartPendingFork(void);
 void replicationHandleMasterDisconnection(void);
 void replicationCacheMaster(client *c);
 void resizeReplicationBacklog(long long newsize);
diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl
index 1052fbdd5..a2f9e75bd 100644
--- a/tests/integration/replication.tcl
+++ b/tests/integration/replication.tcl
@@ -407,7 +407,12 @@ test {diskless loading short read} {
             $master config set repl-diskless-sync yes
             $master config set rdbcompression no
             $replica config set repl-diskless-load swapdb
+            $master config set hz 500
+            $replica config set hz 500
+            $master config set dynamic-hz no
+            $replica config set dynamic-hz no
             # Try to fill the master with all types of data types / encodings
+            set start [clock clicks -milliseconds]
             for {set k 0} {$k < 3} {incr k} {
                 for {set i 0} {$i < 10} {incr i} {
                     r set "$k int_$i" [expr {int(rand()*10000)}]
@@ -429,14 +434,21 @@ test {diskless loading short read} {
                 }
             }
 
+            if {$::verbose} {
+                set end [clock clicks -milliseconds]
+                set duration [expr $end - $start]
+                puts "filling took $duration ms (TODO: use pipeline)"
+                set start [clock clicks -milliseconds]
+            }
+
             # Start the replication process...
             set loglines [count_log_lines -1]
             $master config set repl-diskless-sync-delay 0
             $replica replicaof $master_host $master_port
 
             # kill the replication at various points
-            set attempts 3
-            if {$::accurate} { set attempts 10 }
+            set attempts 100
+            if {$::accurate} { set attempts 500 }
             for {set i 0} {$i < $attempts} {incr i} {
                 # wait for the replica to start reading the rdb
                 # using the log file since the replica only responds to INFO once in 2mb
@@ -469,6 +481,11 @@ test {diskless loading short read} {
                     fail "Replica didn't disconnect"
                 }
             }
+            if {$::verbose} {
+                set end [clock clicks -milliseconds]
+                set duration [expr $end - $start]
+                puts "test took $duration ms"
+            }
             # enable fast shutdown
             $master config set rdb-key-save-delay 0
         }
diff --git a/tests/unit/moduleapi/testrdb.tcl b/tests/unit/moduleapi/testrdb.tcl
index 02c82c7c3..0d26b0a26 100644
--- a/tests/unit/moduleapi/testrdb.tcl
+++ b/tests/unit/moduleapi/testrdb.tcl
@@ -62,18 +62,30 @@ tags "modules" {
                     $master config set repl-diskless-sync yes
                     $master config set rdbcompression no
                     $replica config set repl-diskless-load swapdb
+                    $master config set hz 500
+                    $replica config set hz 500
+                    $master config set dynamic-hz no
+                    $replica config set dynamic-hz no
+                    set start [clock clicks -milliseconds]
                     for {set k 0} {$k < 30} {incr k} {
                         r testrdb.set.key key$k [string repeat A [expr {int(rand()*1000000)}]]
                     }
 
+                    if {$::verbose} {
+                        set end [clock clicks -milliseconds]
+                        set duration [expr $end - $start]
+                        puts "filling took $duration ms (TODO: use pipeline)"
+                        set start [clock clicks -milliseconds]
+                    }
+
                     # Start the replication process...
                     set loglines [count_log_lines -1]
                     $master config set repl-diskless-sync-delay 0
                     $replica replicaof $master_host $master_port
 
                     # kill the replication at various points
-                    set attempts 3
-                    if {$::accurate} { set attempts 10 }
+                    set attempts 100
+                    if {$::accurate} { set attempts 500 }
                     for {set i 0} {$i < $attempts} {incr i} {
                         # wait for the replica to start reading the rdb
                         # using the log file since the replica only responds to INFO once in 2mb
@@ -106,6 +118,11 @@ tags "modules" {
                             fail "Replica didn't disconnect"
                         }
                     }
+                    if {$::verbose} {
+                        set end [clock clicks -milliseconds]
+                        set duration [expr $end - $start]
+                        puts "test took $duration ms"
+                    }
                     # enable fast shutdown
                     $master config set rdb-key-save-delay 0
                 }

From 3ddb6ffa34740b877c759e9f0bd422f33424cd3b Mon Sep 17 00:00:00 2001
From: michael-grunder <michael.grunder@gmail.com>
Date: Thu, 6 Aug 2020 12:41:46 -0700
Subject: [PATCH 0544/1098] Remove hiredis so we can add it as a subtree

---
 deps/hiredis/.gitignore                      |    8 -
 deps/hiredis/.travis.yml                     |   97 --
 deps/hiredis/CHANGELOG.md                    |  199 ---
 deps/hiredis/CMakeLists.txt                  |   90 --
 deps/hiredis/COPYING                         |   29 -
 deps/hiredis/Makefile                        |  273 ----
 deps/hiredis/README.md                       |  412 ------
 deps/hiredis/adapters/ae.h                   |  127 --
 deps/hiredis/adapters/glib.h                 |  153 ---
 deps/hiredis/adapters/ivykis.h               |   81 --
 deps/hiredis/adapters/libev.h                |  147 --
 deps/hiredis/adapters/libevent.h             |  172 ---
 deps/hiredis/adapters/libuv.h                |  119 --
 deps/hiredis/adapters/macosx.h               |  114 --
 deps/hiredis/adapters/qt.h                   |  135 --
 deps/hiredis/appveyor.yml                    |   24 -
 deps/hiredis/async.c                         |  766 -----------
 deps/hiredis/async.h                         |  142 --
 deps/hiredis/async_private.h                 |   72 -
 deps/hiredis/dict.c                          |  338 -----
 deps/hiredis/dict.h                          |  126 --
 deps/hiredis/examples/CMakeLists.txt         |   46 -
 deps/hiredis/examples/example-ae.c           |   62 -
 deps/hiredis/examples/example-glib.c         |   73 -
 deps/hiredis/examples/example-ivykis.c       |   58 -
 deps/hiredis/examples/example-libev.c        |   52 -
 deps/hiredis/examples/example-libevent-ssl.c |   73 -
 deps/hiredis/examples/example-libevent.c     |   64 -
 deps/hiredis/examples/example-libuv.c        |   53 -
 deps/hiredis/examples/example-macosx.c       |   66 -
 deps/hiredis/examples/example-qt.cpp         |   46 -
 deps/hiredis/examples/example-qt.h           |   32 -
 deps/hiredis/examples/example-ssl.c          |   97 --
 deps/hiredis/examples/example.c              |   91 --
 deps/hiredis/fmacros.h                       |   12 -
 deps/hiredis/hiredis.c                       | 1086 ---------------
 deps/hiredis/hiredis.h                       |  299 ----
 deps/hiredis/hiredis.pc.in                   |   11 -
 deps/hiredis/hiredis_ssl.h                   |   53 -
 deps/hiredis/hiredis_ssl.pc.in               |   12 -
 deps/hiredis/net.c                           |  571 --------
 deps/hiredis/net.h                           |   54 -
 deps/hiredis/read.c                          |  681 ---------
 deps/hiredis/read.h                          |  122 --
 deps/hiredis/sds.c                           | 1291 ------------------
 deps/hiredis/sds.h                           |  276 ----
 deps/hiredis/sdsalloc.h                      |   42 -
 deps/hiredis/sockcompat.c                    |  248 ----
 deps/hiredis/sockcompat.h                    |   91 --
 deps/hiredis/ssl.c                           |  448 ------
 deps/hiredis/test.c                          | 1012 --------------
 deps/hiredis/test.sh                         |   70 -
 deps/hiredis/win32.h                         |   56 -
 53 files changed, 10872 deletions(-)
 delete mode 100644 deps/hiredis/.gitignore
 delete mode 100644 deps/hiredis/.travis.yml
 delete mode 100644 deps/hiredis/CHANGELOG.md
 delete mode 100644 deps/hiredis/CMakeLists.txt
 delete mode 100644 deps/hiredis/COPYING
 delete mode 100644 deps/hiredis/Makefile
 delete mode 100644 deps/hiredis/README.md
 delete mode 100644 deps/hiredis/adapters/ae.h
 delete mode 100644 deps/hiredis/adapters/glib.h
 delete mode 100644 deps/hiredis/adapters/ivykis.h
 delete mode 100644 deps/hiredis/adapters/libev.h
 delete mode 100644 deps/hiredis/adapters/libevent.h
 delete mode 100644 deps/hiredis/adapters/libuv.h
 delete mode 100644 deps/hiredis/adapters/macosx.h
 delete mode 100644 deps/hiredis/adapters/qt.h
 delete mode 100644 deps/hiredis/appveyor.yml
 delete mode 100644 deps/hiredis/async.c
 delete mode 100644 deps/hiredis/async.h
 delete mode 100644 deps/hiredis/async_private.h
 delete mode 100644 deps/hiredis/dict.c
 delete mode 100644 deps/hiredis/dict.h
 delete mode 100644 deps/hiredis/examples/CMakeLists.txt
 delete mode 100644 deps/hiredis/examples/example-ae.c
 delete mode 100644 deps/hiredis/examples/example-glib.c
 delete mode 100644 deps/hiredis/examples/example-ivykis.c
 delete mode 100644 deps/hiredis/examples/example-libev.c
 delete mode 100644 deps/hiredis/examples/example-libevent-ssl.c
 delete mode 100644 deps/hiredis/examples/example-libevent.c
 delete mode 100644 deps/hiredis/examples/example-libuv.c
 delete mode 100644 deps/hiredis/examples/example-macosx.c
 delete mode 100644 deps/hiredis/examples/example-qt.cpp
 delete mode 100644 deps/hiredis/examples/example-qt.h
 delete mode 100644 deps/hiredis/examples/example-ssl.c
 delete mode 100644 deps/hiredis/examples/example.c
 delete mode 100644 deps/hiredis/fmacros.h
 delete mode 100644 deps/hiredis/hiredis.c
 delete mode 100644 deps/hiredis/hiredis.h
 delete mode 100644 deps/hiredis/hiredis.pc.in
 delete mode 100644 deps/hiredis/hiredis_ssl.h
 delete mode 100644 deps/hiredis/hiredis_ssl.pc.in
 delete mode 100644 deps/hiredis/net.c
 delete mode 100644 deps/hiredis/net.h
 delete mode 100644 deps/hiredis/read.c
 delete mode 100644 deps/hiredis/read.h
 delete mode 100644 deps/hiredis/sds.c
 delete mode 100644 deps/hiredis/sds.h
 delete mode 100644 deps/hiredis/sdsalloc.h
 delete mode 100644 deps/hiredis/sockcompat.c
 delete mode 100644 deps/hiredis/sockcompat.h
 delete mode 100644 deps/hiredis/ssl.c
 delete mode 100644 deps/hiredis/test.c
 delete mode 100755 deps/hiredis/test.sh
 delete mode 100644 deps/hiredis/win32.h

diff --git a/deps/hiredis/.gitignore b/deps/hiredis/.gitignore
deleted file mode 100644
index 8e50b5434..000000000
--- a/deps/hiredis/.gitignore
+++ /dev/null
@@ -1,8 +0,0 @@
-/hiredis-test
-/examples/hiredis-example*
-/*.o
-/*.so
-/*.dylib
-/*.a
-/*.pc
-*.dSYM
diff --git a/deps/hiredis/.travis.yml b/deps/hiredis/.travis.yml
deleted file mode 100644
index dd8e0e73d..000000000
--- a/deps/hiredis/.travis.yml
+++ /dev/null
@@ -1,97 +0,0 @@
-language: c
-sudo: false
-compiler:
-  - gcc
-  - clang
-
-os:
-  - linux
-  - osx
-
-branches:
-  only:
-    - staging
-    - trying
-    - master
-
-before_script:
-    - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then brew update; brew install redis; fi
-
-addons:
-  apt:
-    packages:
-    - libc6-dbg
-    - libc6-dev
-    - libc6:i386
-    - libc6-dev-i386
-    - libc6-dbg:i386
-    - gcc-multilib
-    - g++-multilib
-    - valgrind
-
-env:
-  - BITS="32"
-  - BITS="64"
-
-script:
-  - EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DHIREDIS_SSL:BOOL=ON";
-    if [ "$TRAVIS_OS_NAME" == "osx" ]; then
-      if [ "$BITS" == "32" ]; then
-        CFLAGS="-m32 -Werror";
-        CXXFLAGS="-m32 -Werror";
-        LDFLAGS="-m32";
-        EXTRA_CMAKE_OPTS=;
-      else
-        CFLAGS="-Werror";
-        CXXFLAGS="-Werror";
-      fi;
-    else
-      TEST_PREFIX="valgrind --track-origins=yes --leak-check=full";
-      if [ "$BITS" == "32" ]; then
-        CFLAGS="-m32 -Werror";
-        CXXFLAGS="-m32 -Werror";
-        LDFLAGS="-m32";
-        EXTRA_CMAKE_OPTS=;
-      else
-        CFLAGS="-Werror";
-        CXXFLAGS="-Werror";
-      fi;
-    fi;
-    export CFLAGS CXXFLAGS LDFLAGS TEST_PREFIX EXTRA_CMAKE_OPTS
-  - mkdir build/ && cd build/
-  - cmake .. ${EXTRA_CMAKE_OPTS}
-  - make VERBOSE=1
-  - ctest -V
-
-matrix:
-  include:
-    # Windows MinGW cross compile on Linux
-    - os: linux
-      dist: xenial
-      compiler: mingw
-      addons:
-        apt:
-          packages:
-            - ninja-build
-            - gcc-mingw-w64-x86-64
-            - g++-mingw-w64-x86-64
-      script:
-        - mkdir build && cd build
-        - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_WITH_INSTALL_RPATH=on
-        - ninja -v
-
-    # Windows MSVC 2017
-    - os: windows
-      compiler: msvc
-      env:
-        - MATRIX_EVAL="CC=cl.exe && CXX=cl.exe"
-      before_install:
-        - eval "${MATRIX_EVAL}"
-      install:
-        - choco install ninja
-      script:
-        - mkdir build && cd build
-        - cmd.exe /C '"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 &&
-          cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release &&
-          ninja -v'
-        - ctest -V
diff --git a/deps/hiredis/CHANGELOG.md b/deps/hiredis/CHANGELOG.md
deleted file mode 100644
index d1d37e515..000000000
--- a/deps/hiredis/CHANGELOG.md
+++ /dev/null
@@ -1,199 +0,0 @@
-### 1.0.0 (unreleased)
-
-**BREAKING CHANGES**:
-
-* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now
-  protocol errors. This is consistent with the RESP specification. On 32-bit
-  platforms, the upper bound is lowered to `SIZE_MAX`.
-
-* Change `redisReply.len` to `size_t`, as it denotes the the size of a string
-
-  User code should compare this to `size_t` values as well.  If it was used to
-  compare to other values, casting might be necessary or can be removed, if
-  casting was applied before.
-
-### 0.x.x (unreleased)
-**BREAKING CHANGES**:
-
-* Change `redisReply.len` to `size_t`, as it denotes the the size of a string
-
-User code should compare this to `size_t` values as well.
-If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before.
-
-* `redisReplyObjectFunctions.createArray` now takes `size_t` for its length parameter.
-
-### 0.14.0 (2018-09-25)
-
-* Make string2ll static to fix conflict with Redis (Tom Lee [c3188b])
-* Use -dynamiclib instead of -shared for OSX (Ryan Schmidt [a65537])
-* Use string2ll from Redis w/added tests (Michael Grunder [7bef04, 60f622])
-* Makefile - OSX compilation fixes (Ryan Schmidt [881fcb, 0e9af8])
-* Remove redundant NULL checks (Justin Brewer [54acc8, 58e6b8])
-* Fix bulk and multi-bulk length truncation (Justin Brewer [109197])
-* Fix SIGSEGV in OpenBSD by checking for NULL before calling freeaddrinfo (Justin Brewer [546d94])
-* Several POSIX compatibility fixes (Justin Brewer [bbeab8, 49bbaa, d1c1b6])
-* Makefile - Compatibility fixes (Dimitri Vorobiev [3238cf, 12a9d1])
-* Makefile - Fix make install on FreeBSD (Zach Shipko [a2ef2b])
-* Makefile - don't assume $(INSTALL) is cp (Igor Gnatenko [725a96])
-* Separate side-effect causing function from assert and small cleanup (amallia [b46413, 3c3234])
-* Don't send negative values to `__redisAsyncCommand` (Frederik Deweerdt [706129])
-* Fix leak if setsockopt fails (Frederik Deweerdt [e21c9c])
-* Fix libevent leak (zfz [515228])
-* Clean up GCC warning (Ichito Nagata [2ec774])
-* Keep track of errno in `__redisSetErrorFromErrno()` as snprintf may use it (Jin Qing [25cd88])
-* Solaris compilation fix (Donald Whyte [41b07d])
-* Reorder linker arguments when building examples (Tustfarm-heart [06eedd])
-* Keep track of subscriptions in case of rapid subscribe/unsubscribe (Hyungjin Kim [073dc8, be76c5, d46999])
-* libuv use after free fix (Paul Scott [cbb956])
-* Properly close socket fd on reconnect attempt (WSL [64d1ec])
-* Skip valgrind in OSX tests (Jan-Erik Rediger [9deb78])
-* Various updates for Travis testing OSX (Ted Nyman [fa3774, 16a459, bc0ea5])
-* Update libevent (Chris Xin [386802])
-* Change sds.h for building in C++ projects (Ali Volkan ATLI [f5b32e])
-* Use proper format specifier in redisFormatSdsCommandArgv (Paulino Huerta, Jan-Erik Rediger [360a06, 8655a6])
-* Better handling of NULL reply in example code (Jan-Erik Rediger [1b8ed3])
-* Prevent overflow when formatting an error (Jan-Erik Rediger [0335cb])
-* Compatibility fix for strerror_r (Tom Lee [bb1747])
-* Properly detect integer parse/overflow errors (Justin Brewer [93421f])
-* Adds CI for Windows and cygwin fixes (owent, [6c53d6, 6c3e40])
-* Catch a buffer overflow when formatting the error message
-* Import latest upstream sds. This breaks applications that are linked against the old hiredis v0.13
-* Fix warnings, when compiled with -Wshadow
-* Make hiredis compile in Cygwin on Windows, now CI-tested
-* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now
-  protocol errors. This is consistent with the RESP specification. On 32-bit
-  platforms, the upper bound is lowered to `SIZE_MAX`.
-
-* Remove backwards compatibility macro's
-
-This removes the following old function aliases, use the new name now:
-
-| Old                         | New                    |
-| --------------------------- | ---------------------- |
-| redisReplyReaderCreate      | redisReaderCreate      |
-| redisReplyReaderCreate      | redisReaderCreate      |
-| redisReplyReaderFree        | redisReaderFree        |
-| redisReplyReaderFeed        | redisReaderFeed        |
-| redisReplyReaderGetReply    | redisReaderGetReply    |
-| redisReplyReaderSetPrivdata | redisReaderSetPrivdata |
-| redisReplyReaderGetObject   | redisReaderGetObject   |
-| redisReplyReaderGetError    | redisReaderGetError    |
-
-* The `DEBUG` variable in the Makefile was renamed to `DEBUG_FLAGS`
-
-Previously it broke some builds for people that had `DEBUG` set to some arbitrary value,
-due to debugging other software.
-By renaming we avoid unintentional name clashes.
-
-Simply rename `DEBUG` to `DEBUG_FLAGS` in your environment to make it working again.
-
-### 0.13.3 (2015-09-16)
-
-* Revert "Clear `REDIS_CONNECTED` flag when connection is closed".
-* Make tests pass on FreeBSD (Thanks, Giacomo Olgeni)
-
-
-If the `REDIS_CONNECTED` flag is cleared,
-the async onDisconnect callback function will never be called.
-This causes problems as the disconnect is never reported back to the user.
-
-### 0.13.2 (2015-08-25)
-
-* Prevent crash on pending replies in async code (Thanks, @switch-st)
-* Clear `REDIS_CONNECTED` flag when connection is closed (Thanks, Jerry Jacobs)
-* Add MacOS X addapter (Thanks, @dizzus)
-* Add Qt adapter (Thanks, Pietro Cerutti)
-* Add Ivykis adapter (Thanks, Gergely Nagy)
-
-All adapters are provided as is and are only tested where possible.
-
-### 0.13.1 (2015-05-03)
-
-This is a bug fix release.
-The new `reconnect` method introduced new struct members, which clashed with pre-defined names in pre-C99 code.
-Another commit forced C99 compilation just to make it work, but of course this is not desirable for outside projects.
-Other non-C99 code can now use hiredis as usual again.
-Sorry for the inconvenience.
-
-* Fix memory leak in async reply handling (Salvatore Sanfilippo)
-* Rename struct member to avoid name clash with pre-c99 code (Alex Balashov, ncopa)
-
-### 0.13.0 (2015-04-16)
-
-This release adds a minimal Windows compatibility layer.
-The parser, standalone since v0.12.0, can now be compiled on Windows
-(and thus used in other client libraries as well)
-
-* Windows compatibility layer for parser code (tzickel)
-* Properly escape data printed to PKGCONF file (Dan Skorupski)
-* Fix tests when assert() undefined (Keith Bennett, Matt Stancliff)
-* Implement a reconnect method for the client context, this changes the structure of `redisContext` (Aaron Bedra)
-
-### 0.12.1 (2015-01-26)
-
-* Fix `make install`: DESTDIR support, install all required files, install PKGCONF in proper location
-* Fix `make test` as 32 bit build on 64 bit platform
-
-### 0.12.0 (2015-01-22)
-
-* Add optional KeepAlive support
-
-* Try again on EINTR errors
-
-* Add libuv adapter
-
-* Add IPv6 support
-
-* Remove possibility of multiple close on same fd
-
-* Add ability to bind source address on connect
-
-* Add redisConnectFd() and redisFreeKeepFd()
-
-* Fix getaddrinfo() memory leak
-
-* Free string if it is unused (fixes memory leak)
-
-* Improve redisAppendCommandArgv performance 2.5x
-
-* Add support for SO_REUSEADDR
-
-* Fix redisvFormatCommand format parsing
-
-* Add GLib 2.0 adapter
-
-* Refactor reading code into read.c
-
-* Fix errno error buffers to not clobber errors
-
-* Generate pkgconf during build
-
-* Silence _BSD_SOURCE warnings
-
-* Improve digit counting for multibulk creation
-
-
-### 0.11.0
-
-* Increase the maximum multi-bulk reply depth to 7.
-
-* Increase the read buffer size from 2k to 16k.
-
-* Use poll(2) instead of select(2) to support large fds (>= 1024).
-
-### 0.10.1
-
-* Makefile overhaul. Important to check out if you override one or more
-  variables using environment variables or via arguments to the "make" tool.
-
-* Issue #45: Fix potential memory leak for a multi bulk reply with 0 elements
-  being created by the default reply object functions.
-
-* Issue #43: Don't crash in an asynchronous context when Redis returns an error
-  reply after the connection has been made (this happens when the maximum
-  number of connections is reached).
-
-### 0.10.0
-
-* See commit log.
-
diff --git a/deps/hiredis/CMakeLists.txt b/deps/hiredis/CMakeLists.txt
deleted file mode 100644
index 9e78894f3..000000000
--- a/deps/hiredis/CMakeLists.txt
+++ /dev/null
@@ -1,90 +0,0 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 3.4.0)
-INCLUDE(GNUInstallDirs)
-PROJECT(hiredis)
-
-OPTION(ENABLE_SSL "Build hiredis_ssl for SSL support" OFF)
-
-MACRO(getVersionBit name)
-  SET(VERSION_REGEX "^#define ${name} (.+)$")
-  FILE(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/hiredis.h"
-    VERSION_BIT REGEX ${VERSION_REGEX})
-  STRING(REGEX REPLACE ${VERSION_REGEX} "\\1" ${name} "${VERSION_BIT}")
-ENDMACRO(getVersionBit)
-
-getVersionBit(HIREDIS_MAJOR)
-getVersionBit(HIREDIS_MINOR)
-getVersionBit(HIREDIS_PATCH)
-getVersionBit(HIREDIS_SONAME)
-SET(VERSION "${HIREDIS_MAJOR}.${HIREDIS_MINOR}.${HIREDIS_PATCH}")
-MESSAGE("Detected version: ${VERSION}")
-
-PROJECT(hiredis VERSION "${VERSION}")
-
-SET(ENABLE_EXAMPLES OFF CACHE BOOL "Enable building hiredis examples")
-
-ADD_LIBRARY(hiredis SHARED
-    async.c
-    dict.c
-    hiredis.c
-    net.c
-    read.c
-    sds.c
-    sockcompat.c)
-
-SET_TARGET_PROPERTIES(hiredis
-    PROPERTIES
-    VERSION "${HIREDIS_SONAME}")
-IF(WIN32 OR MINGW)
-    TARGET_LINK_LIBRARIES(hiredis PRIVATE ws2_32)
-ENDIF()
-TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC .)
-
-CONFIGURE_FILE(hiredis.pc.in hiredis.pc @ONLY)
-
-INSTALL(TARGETS hiredis
-    DESTINATION "${CMAKE_INSTALL_LIBDIR}")
-
-INSTALL(FILES hiredis.h read.h sds.h async.h
-    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis)
-    
-INSTALL(DIRECTORY adapters
-    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis)
-    
-INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis.pc
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
-
-IF(ENABLE_SSL)
-    IF (NOT OPENSSL_ROOT_DIR)
-        IF (APPLE)
-            SET(OPENSSL_ROOT_DIR "/usr/local/opt/openssl")
-        ENDIF()
-    ENDIF()
-    FIND_PACKAGE(OpenSSL REQUIRED)
-    ADD_LIBRARY(hiredis_ssl SHARED
-        ssl.c)
-    TARGET_INCLUDE_DIRECTORIES(hiredis_ssl PRIVATE "${OPENSSL_INCLUDE_DIR}")
-    TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE ${OPENSSL_LIBRARIES})
-    CONFIGURE_FILE(hiredis_ssl.pc.in hiredis_ssl.pc @ONLY)
-
-    INSTALL(TARGETS hiredis_ssl
-        DESTINATION "${CMAKE_INSTALL_LIBDIR}")
-
-    INSTALL(FILES hiredis_ssl.h
-        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis)
-    
-    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl.pc
-        DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
-ENDIF()
-
-IF(NOT (WIN32 OR MINGW))
-    ENABLE_TESTING()
-    ADD_EXECUTABLE(hiredis-test test.c)
-    TARGET_LINK_LIBRARIES(hiredis-test hiredis)
-    ADD_TEST(NAME hiredis-test
-        COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test.sh)
-ENDIF()
-
-# Add examples
-IF(ENABLE_EXAMPLES)
-  ADD_SUBDIRECTORY(examples)
-ENDIF(ENABLE_EXAMPLES)
diff --git a/deps/hiredis/COPYING b/deps/hiredis/COPYING
deleted file mode 100644
index a5fc97395..000000000
--- a/deps/hiredis/COPYING
+++ /dev/null
@@ -1,29 +0,0 @@
-Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
-Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
-
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice,
-  this list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice,
-  this list of conditions and the following disclaimer in the documentation
-  and/or other materials provided with the distribution.
-
-* Neither the name of Redis nor the names of its contributors may be used
-  to endorse or promote products derived from this software without specific
-  prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/deps/hiredis/Makefile b/deps/hiredis/Makefile
deleted file mode 100644
index 25ac15464..000000000
--- a/deps/hiredis/Makefile
+++ /dev/null
@@ -1,273 +0,0 @@
-# Hiredis Makefile
-# Copyright (C) 2010-2011 Salvatore Sanfilippo <antirez at gmail dot com>
-# Copyright (C) 2010-2011 Pieter Noordhuis <pcnoordhuis at gmail dot com>
-# This file is released under the BSD license, see the COPYING file
-
-OBJ=net.o hiredis.o sds.o async.o read.o sockcompat.o
-SSL_OBJ=ssl.o
-EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib
-ifeq ($(USE_SSL),1)
-EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl
-endif
-TESTS=hiredis-test
-LIBNAME=libhiredis
-SSL_LIBNAME=libhiredis_ssl
-PKGCONFNAME=hiredis.pc
-SSL_PKGCONFNAME=hiredis_ssl.pc
-
-HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}')
-HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}')
-HIREDIS_PATCH=$(shell grep HIREDIS_PATCH hiredis.h | awk '{print $$3}')
-HIREDIS_SONAME=$(shell grep HIREDIS_SONAME hiredis.h | awk '{print $$3}')
-
-# Installation related variables and target
-PREFIX?=/usr/local
-INCLUDE_PATH?=include/hiredis
-LIBRARY_PATH?=lib
-PKGCONF_PATH?=pkgconfig
-INSTALL_INCLUDE_PATH= $(DESTDIR)$(PREFIX)/$(INCLUDE_PATH)
-INSTALL_LIBRARY_PATH= $(DESTDIR)$(PREFIX)/$(LIBRARY_PATH)
-INSTALL_PKGCONF_PATH= $(INSTALL_LIBRARY_PATH)/$(PKGCONF_PATH)
-
-# redis-server configuration used for testing
-REDIS_PORT=56379
-REDIS_SERVER=redis-server
-define REDIS_TEST_CONFIG
-	daemonize yes
-	pidfile /tmp/hiredis-test-redis.pid
-	port $(REDIS_PORT)
-	bind 127.0.0.1
-	unixsocket /tmp/hiredis-test-redis.sock
-endef
-export REDIS_TEST_CONFIG
-
-# Fallback to gcc when $CC is not in $PATH.
-CC:=$(shell sh -c 'type $${CC%% *} >/dev/null 2>/dev/null && echo $(CC) || echo gcc')
-CXX:=$(shell sh -c 'type $${CXX%% *} >/dev/null 2>/dev/null && echo $(CXX) || echo g++')
-OPTIMIZATION?=-O3
-WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings -Wno-missing-field-initializers
-DEBUG_FLAGS?= -g -ggdb
-REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CPPFLAGS) $(CFLAGS) $(WARNINGS) $(DEBUG_FLAGS)
-REAL_LDFLAGS=$(LDFLAGS)
-
-DYLIBSUFFIX=so
-STLIBSUFFIX=a
-DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME)
-DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR)
-DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX)
-SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX)
-DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME)
-STLIBNAME=$(LIBNAME).$(STLIBSUFFIX)
-SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX)
-STLIB_MAKE_CMD=$(AR) rcs
-
-# Platform-specific overrides
-uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
-
-USE_SSL?=0
-
-# This is required for test.c only
-ifeq ($(USE_SSL),1)
-  CFLAGS+=-DHIREDIS_TEST_SSL
-endif
-
-ifeq ($(uname_S),Linux)
-  SSL_LDFLAGS=-lssl -lcrypto
-else
-  OPENSSL_PREFIX?=/usr/local/opt/openssl
-  CFLAGS+=-I$(OPENSSL_PREFIX)/include
-  SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto
-endif
-
-ifeq ($(uname_S),SunOS)
-  REAL_LDFLAGS+= -ldl -lnsl -lsocket
-  DYLIB_MAKE_CMD=$(CC) -G -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS)
-endif
-ifeq ($(uname_S),Darwin)
-  DYLIBSUFFIX=dylib
-  DYLIB_MINOR_NAME=$(LIBNAME).$(HIREDIS_SONAME).$(DYLIBSUFFIX)
-  DYLIB_MAKE_CMD=$(CC) -dynamiclib -Wl,-install_name,$(PREFIX)/$(LIBRARY_PATH)/$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS)
-endif
-
-all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME)
-ifeq ($(USE_SSL),1)
-all: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME)
-endif
-
-# Deps (use make dep to generate this)
-async.o: async.c fmacros.h async.h hiredis.h read.h sds.h net.h dict.c dict.h
-dict.o: dict.c fmacros.h dict.h
-hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h win32.h
-net.o: net.c fmacros.h net.h hiredis.h read.h sds.h sockcompat.h win32.h
-read.o: read.c fmacros.h read.h sds.h
-sds.o: sds.c sds.h
-sockcompat.o: sockcompat.c sockcompat.h
-ssl.o: ssl.c hiredis.h
-test.o: test.c fmacros.h hiredis.h read.h sds.h
-
-$(DYLIBNAME): $(OBJ)
-	$(DYLIB_MAKE_CMD) -o $(DYLIBNAME) $(OBJ) $(REAL_LDFLAGS)
-
-$(STLIBNAME): $(OBJ)
-	$(STLIB_MAKE_CMD) $(STLIBNAME) $(OBJ)
-
-$(SSL_DYLIBNAME): $(SSL_OBJ)
-	$(DYLIB_MAKE_CMD) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(SSL_LDFLAGS)
-
-$(SSL_STLIBNAME): $(SSL_OBJ)
-	$(STLIB_MAKE_CMD) $(SSL_STLIBNAME) $(SSL_OBJ)
-
-dynamic: $(DYLIBNAME)
-static: $(STLIBNAME)
-ifeq ($(USE_SSL),1)
-dynamic: $(SSL_DYLIBNAME)
-static: $(SSL_STLIBNAME)
-endif
-
-# Binaries:
-hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME)
-	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(REAL_LDFLAGS)
-
-hiredis-example-libevent-ssl: examples/example-libevent-ssl.c adapters/libevent.h $(STLIBNAME) $(SSL_STLIBNAME)
-	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS)
-
-hiredis-example-libev: examples/example-libev.c adapters/libev.h $(STLIBNAME)
-	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -lev $(STLIBNAME) $(REAL_LDFLAGS)
-
-hiredis-example-glib: examples/example-glib.c adapters/glib.h $(STLIBNAME)
-	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(shell pkg-config --cflags --libs glib-2.0) $(STLIBNAME) $(REAL_LDFLAGS)
-
-hiredis-example-ivykis: examples/example-ivykis.c adapters/ivykis.h $(STLIBNAME)
-	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -livykis $(STLIBNAME) $(REAL_LDFLAGS)
-
-hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME)
-	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) $(REAL_LDFLAGS)
-
-hiredis-example-ssl: examples/example-ssl.c $(STLIBNAME) $(SSL_STLIBNAME)
-	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS)
-
-ifndef AE_DIR
-hiredis-example-ae:
-	@echo "Please specify AE_DIR (e.g. <redis repository>/src)"
-	@false
-else
-hiredis-example-ae: examples/example-ae.c adapters/ae.h $(STLIBNAME)
-	$(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(AE_DIR) $< $(AE_DIR)/ae.o $(AE_DIR)/zmalloc.o $(AE_DIR)/../deps/jemalloc/lib/libjemalloc.a -pthread $(STLIBNAME)
-endif
-
-ifndef LIBUV_DIR
-hiredis-example-libuv:
-	@echo "Please specify LIBUV_DIR (e.g. ../libuv/)"
-	@false
-else
-hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME)
-	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) $(REAL_LDFLAGS)
-endif
-
-ifeq ($(and $(QT_MOC),$(QT_INCLUDE_DIR),$(QT_LIBRARY_DIR)),)
-hiredis-example-qt:
-	@echo "Please specify QT_MOC, QT_INCLUDE_DIR AND QT_LIBRARY_DIR"
-	@false
-else
-hiredis-example-qt: examples/example-qt.cpp adapters/qt.h $(STLIBNAME)
-	$(QT_MOC) adapters/qt.h -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore | \
-	    $(CXX) -x c++ -o qt-adapter-moc.o -c - $(REAL_CFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore
-	$(QT_MOC) examples/example-qt.h -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore | \
-	    $(CXX) -x c++ -o qt-example-moc.o -c - $(REAL_CFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore
-	$(CXX) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore -L$(QT_LIBRARY_DIR) qt-adapter-moc.o qt-example-moc.o $< -pthread $(STLIBNAME) -lQtCore
-endif
-
-hiredis-example: examples/example.c $(STLIBNAME)
-	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS)
-
-examples: $(EXAMPLES)
-
-TEST_LIBS = $(STLIBNAME)
-ifeq ($(USE_SSL),1)
-    TEST_LIBS += $(SSL_STLIBNAME) -lssl -lcrypto -lpthread
-endif
-hiredis-test: test.o $(TEST_LIBS)
-
-hiredis-%: %.o $(STLIBNAME)
-	$(CC) $(REAL_CFLAGS) -o $@ $< $(TEST_LIBS) $(REAL_LDFLAGS)
-
-test: hiredis-test
-	./hiredis-test
-
-check: hiredis-test
-	TEST_SSL=$(USE_SSL) ./test.sh
-
-.c.o:
-	$(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $<
-
-clean:
-	rm -rf $(DYLIBNAME) $(STLIBNAME) $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov
-
-dep:
-	$(CC) $(CPPFLAGS) $(CFLAGS) -MM *.c
-
-INSTALL?= cp -pPR
-
-$(PKGCONFNAME): hiredis.h
-	@echo "Generating $@ for pkgconfig..."
-	@echo prefix=$(PREFIX) > $@
-	@echo exec_prefix=\$${prefix} >> $@
-	@echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@
-	@echo includedir=$(PREFIX)/$(INCLUDE_PATH) >> $@
-	@echo >> $@
-	@echo Name: hiredis >> $@
-	@echo Description: Minimalistic C client library for Redis. >> $@
-	@echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@
-	@echo Libs: -L\$${libdir} -lhiredis >> $@
-	@echo Cflags: -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@
-
-$(SSL_PKGCONFNAME): hiredis.h
-	@echo "Generating $@ for pkgconfig..."
-	@echo prefix=$(PREFIX) > $@
-	@echo exec_prefix=\$${prefix} >> $@
-	@echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@
-	@echo includedir=$(PREFIX)/$(INCLUDE_PATH) >> $@
-	@echo >> $@
-	@echo Name: hiredis_ssl >> $@
-	@echo Description: SSL Support for hiredis. >> $@
-	@echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@
-	@echo Requires: hiredis >> $@
-	@echo Libs: -L\$${libdir} -lhiredis_ssl >> $@
-	@echo Libs.private: -lssl -lcrypto >> $@
-
-install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME)
-	mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL_LIBRARY_PATH)
-	$(INSTALL) hiredis.h async.h read.h sds.h $(INSTALL_INCLUDE_PATH)
-	$(INSTALL) adapters/*.h $(INSTALL_INCLUDE_PATH)/adapters
-	$(INSTALL) $(DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(DYLIB_MINOR_NAME)
-	cd $(INSTALL_LIBRARY_PATH) && ln -sf $(DYLIB_MINOR_NAME) $(DYLIBNAME)
-	$(INSTALL) $(STLIBNAME) $(INSTALL_LIBRARY_PATH)
-	mkdir -p $(INSTALL_PKGCONF_PATH)
-	$(INSTALL) $(PKGCONFNAME) $(INSTALL_PKGCONF_PATH)
-
-32bit:
-	@echo ""
-	@echo "WARNING: if this fails under Linux you probably need to install libc6-dev-i386"
-	@echo ""
-	$(MAKE) CFLAGS="-m32" LDFLAGS="-m32"
-
-32bit-vars:
-	$(eval CFLAGS=-m32)
-	$(eval LDFLAGS=-m32)
-
-gprof:
-	$(MAKE) CFLAGS="-pg" LDFLAGS="-pg"
-
-gcov:
-	$(MAKE) CFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS="-fprofile-arcs"
-
-coverage: gcov
-	make check
-	mkdir -p tmp/lcov
-	lcov -d . -c -o tmp/lcov/hiredis.info
-	genhtml --legend -o tmp/lcov/report tmp/lcov/hiredis.info
-
-noopt:
-	$(MAKE) OPTIMIZATION=""
-
-.PHONY: all test check clean dep install 32bit 32bit-vars gprof gcov noopt
diff --git a/deps/hiredis/README.md b/deps/hiredis/README.md
deleted file mode 100644
index c0b432f07..000000000
--- a/deps/hiredis/README.md
+++ /dev/null
@@ -1,412 +0,0 @@
-[![Build Status](https://travis-ci.org/redis/hiredis.png)](https://travis-ci.org/redis/hiredis)
-
-**This Readme reflects the latest changed in the master branch. See [v0.13.3](https://github.com/redis/hiredis/tree/v0.13.3) for the Readme and documentation for the latest release.**
-
-# HIREDIS
-
-Hiredis is a minimalistic C client library for the [Redis](http://redis.io/) database.
-
-It is minimalistic because it just adds minimal support for the protocol, but
-at the same time it uses a high level printf-alike API in order to make it
-much higher level than otherwise suggested by its minimal code base and the
-lack of explicit bindings for every Redis command.
-
-Apart from supporting sending commands and receiving replies, it comes with
-a reply parser that is decoupled from the I/O layer. It
-is a stream parser designed for easy reusability, which can for instance be used
-in higher level language bindings for efficient reply parsing.
-
-Hiredis only supports the binary-safe Redis protocol, so you can use it with any
-Redis version >= 1.2.0.
-
-The library comes with multiple APIs. There is the
-*synchronous API*, the *asynchronous API* and the *reply parsing API*.
-
-## Upgrading to `1.0.0`
-
-Version 1.0.0 marks a stable release of hiredis.
-It includes some minor breaking changes, mostly to make the exposed API more uniform and self-explanatory.
-It also bundles the updated `sds` library, to sync up with upstream and Redis.
-For most applications a recompile against the new hiredis should be enough.
-For code changes see the [Changelog](CHANGELOG.md).
-
-## Upgrading from `<0.9.0`
-
-Version 0.9.0 is a major overhaul of hiredis in every aspect. However, upgrading existing
-code using hiredis should not be a big pain. The key thing to keep in mind when
-upgrading is that hiredis >= 0.9.0 uses a `redisContext*` to keep state, in contrast to
-the stateless 0.0.1 that only has a file descriptor to work with.
-
-## Synchronous API
-
-To consume the synchronous API, there are only a few function calls that need to be introduced:
-
-```c
-redisContext *redisConnect(const char *ip, int port);
-void *redisCommand(redisContext *c, const char *format, ...);
-void freeReplyObject(void *reply);
-```
-
-### Connecting
-
-The function `redisConnect` is used to create a so-called `redisContext`. The
-context is where Hiredis holds state for a connection. The `redisContext`
-struct has an integer `err` field that is non-zero when the connection is in
-an error state. The field `errstr` will contain a string with a description of
-the error. More information on errors can be found in the **Errors** section.
-After trying to connect to Redis using `redisConnect` you should
-check the `err` field to see if establishing the connection was successful:
-```c
-redisContext *c = redisConnect("127.0.0.1", 6379);
-if (c == NULL || c->err) {
-    if (c) {
-        printf("Error: %s\n", c->errstr);
-        // handle error
-    } else {
-        printf("Can't allocate redis context\n");
-    }
-}
-```
-
-*Note: A `redisContext` is not thread-safe.*
-
-### Sending commands
-
-There are several ways to issue commands to Redis. The first that will be introduced is
-`redisCommand`. This function takes a format similar to printf. In the simplest form,
-it is used like this:
-```c
-reply = redisCommand(context, "SET foo bar");
-```
-
-The specifier `%s` interpolates a string in the command, and uses `strlen` to
-determine the length of the string:
-```c
-reply = redisCommand(context, "SET foo %s", value);
-```
-When you need to pass binary safe strings in a command, the `%b` specifier can be
-used. Together with a pointer to the string, it requires a `size_t` length argument
-of the string:
-```c
-reply = redisCommand(context, "SET foo %b", value, (size_t) valuelen);
-```
-Internally, Hiredis splits the command in different arguments and will
-convert it to the protocol used to communicate with Redis.
-One or more spaces separates arguments, so you can use the specifiers
-anywhere in an argument:
-```c
-reply = redisCommand(context, "SET key:%s %s", myid, value);
-```
-
-### Using replies
-
-The return value of `redisCommand` holds a reply when the command was
-successfully executed. When an error occurs, the return value is `NULL` and
-the `err` field in the context will be set (see section on **Errors**).
-Once an error is returned the context cannot be reused and you should set up
-a new connection.
-
-The standard replies that `redisCommand` are of the type `redisReply`. The
-`type` field in the `redisReply` should be used to test what kind of reply
-was received:
-
-* **`REDIS_REPLY_STATUS`**:
-    * The command replied with a status reply. The status string can be accessed using `reply->str`.
-      The length of this string can be accessed using `reply->len`.
-
-* **`REDIS_REPLY_ERROR`**:
-    *  The command replied with an error. The error string can be accessed identical to `REDIS_REPLY_STATUS`.
-
-* **`REDIS_REPLY_INTEGER`**:
-    * The command replied with an integer. The integer value can be accessed using the
-      `reply->integer` field of type `long long`.
-
-* **`REDIS_REPLY_NIL`**:
-    * The command replied with a **nil** object. There is no data to access.
-
-* **`REDIS_REPLY_STRING`**:
-    * A bulk (string) reply. The value of the reply can be accessed using `reply->str`.
-      The length of this string can be accessed using `reply->len`.
-
-* **`REDIS_REPLY_ARRAY`**:
-    * A multi bulk reply. The number of elements in the multi bulk reply is stored in
-      `reply->elements`. Every element in the multi bulk reply is a `redisReply` object as well
-      and can be accessed via `reply->element[..index..]`.
-      Redis may reply with nested arrays but this is fully supported.
-
-Replies should be freed using the `freeReplyObject()` function.
-Note that this function will take care of freeing sub-reply objects
-contained in arrays and nested arrays, so there is no need for the user to
-free the sub replies (it is actually harmful and will corrupt the memory).
-
-**Important:** the current version of hiredis (0.10.0) frees replies when the
-asynchronous API is used. This means you should not call `freeReplyObject` when
-you use this API. The reply is cleaned up by hiredis _after_ the callback
-returns. This behavior will probably change in future releases, so make sure to
-keep an eye on the changelog when upgrading (see issue #39).
-
-### Cleaning up
-
-To disconnect and free the context the following function can be used:
-```c
-void redisFree(redisContext *c);
-```
-This function immediately closes the socket and then frees the allocations done in
-creating the context.
-
-### Sending commands (cont'd)
-
-Together with `redisCommand`, the function `redisCommandArgv` can be used to issue commands.
-It has the following prototype:
-```c
-void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
-```
-It takes the number of arguments `argc`, an array of strings `argv` and the lengths of the
-arguments `argvlen`. For convenience, `argvlen` may be set to `NULL` and the function will
-use `strlen(3)` on every argument to determine its length. Obviously, when any of the arguments
-need to be binary safe, the entire array of lengths `argvlen` should be provided.
-
-The return value has the same semantic as `redisCommand`.
-
-### Pipelining
-
-To explain how Hiredis supports pipelining in a blocking connection, there needs to be
-understanding of the internal execution flow.
-
-When any of the functions in the `redisCommand` family is called, Hiredis first formats the
-command according to the Redis protocol. The formatted command is then put in the output buffer
-of the context. This output buffer is dynamic, so it can hold any number of commands.
-After the command is put in the output buffer, `redisGetReply` is called. This function has the
-following two execution paths:
-
-1. The input buffer is non-empty:
-    * Try to parse a single reply from the input buffer and return it
-    * If no reply could be parsed, continue at *2*
-2. The input buffer is empty:
-    * Write the **entire** output buffer to the socket
-    * Read from the socket until a single reply could be parsed
-
-The function `redisGetReply` is exported as part of the Hiredis API and can be used when a reply
-is expected on the socket. To pipeline commands, the only things that needs to be done is
-filling up the output buffer. For this cause, two commands can be used that are identical
-to the `redisCommand` family, apart from not returning a reply:
-```c
-void redisAppendCommand(redisContext *c, const char *format, ...);
-void redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
-```
-After calling either function one or more times, `redisGetReply` can be used to receive the
-subsequent replies. The return value for this function is either `REDIS_OK` or `REDIS_ERR`, where
-the latter means an error occurred while reading a reply. Just as with the other commands,
-the `err` field in the context can be used to find out what the cause of this error is.
-
-The following examples shows a simple pipeline (resulting in only a single call to `write(2)` and
-a single call to `read(2)`):
-```c
-redisReply *reply;
-redisAppendCommand(context,"SET foo bar");
-redisAppendCommand(context,"GET foo");
-redisGetReply(context,&reply); // reply for SET
-freeReplyObject(reply);
-redisGetReply(context,&reply); // reply for GET
-freeReplyObject(reply);
-```
-This API can also be used to implement a blocking subscriber:
-```c
-reply = redisCommand(context,"SUBSCRIBE foo");
-freeReplyObject(reply);
-while(redisGetReply(context,&reply) == REDIS_OK) {
-    // consume message
-    freeReplyObject(reply);
-}
-```
-### Errors
-
-When a function call is not successful, depending on the function either `NULL` or `REDIS_ERR` is
-returned. The `err` field inside the context will be non-zero and set to one of the
-following constants:
-
-* **`REDIS_ERR_IO`**:
-    There was an I/O error while creating the connection, trying to write
-    to the socket or read from the socket. If you included `errno.h` in your
-    application, you can use the global `errno` variable to find out what is
-    wrong.
-
-* **`REDIS_ERR_EOF`**:
-    The server closed the connection which resulted in an empty read.
-
-* **`REDIS_ERR_PROTOCOL`**:
-    There was an error while parsing the protocol.
-
-* **`REDIS_ERR_OTHER`**:
-    Any other error. Currently, it is only used when a specified hostname to connect
-    to cannot be resolved.
-
-In every case, the `errstr` field in the context will be set to hold a string representation
-of the error.
-
-## Asynchronous API
-
-Hiredis comes with an asynchronous API that works easily with any event library.
-Examples are bundled that show using Hiredis with [libev](http://software.schmorp.de/pkg/libev.html)
-and [libevent](http://monkey.org/~provos/libevent/).
-
-### Connecting
-
-The function `redisAsyncConnect` can be used to establish a non-blocking connection to
-Redis. It returns a pointer to the newly created `redisAsyncContext` struct. The `err` field
-should be checked after creation to see if there were errors creating the connection.
-Because the connection that will be created is non-blocking, the kernel is not able to
-instantly return if the specified host and port is able to accept a connection.
-
-*Note: A `redisAsyncContext` is not thread-safe.*
-
-```c
-redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
-if (c->err) {
-    printf("Error: %s\n", c->errstr);
-    // handle error
-}
-```
-
-The asynchronous context can hold a disconnect callback function that is called when the
-connection is disconnected (either because of an error or per user request). This function should
-have the following prototype:
-```c
-void(const redisAsyncContext *c, int status);
-```
-On a disconnect, the `status` argument is set to `REDIS_OK` when disconnection was initiated by the
-user, or `REDIS_ERR` when the disconnection was caused by an error. When it is `REDIS_ERR`, the `err`
-field in the context can be accessed to find out the cause of the error.
-
-The context object is always freed after the disconnect callback fired. When a reconnect is needed,
-the disconnect callback is a good point to do so.
-
-Setting the disconnect callback can only be done once per context. For subsequent calls it will
-return `REDIS_ERR`. The function to set the disconnect callback has the following prototype:
-```c
-int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn);
-```
-`ac->data` may be used to pass user data to this callback, the same can be done for redisConnectCallback.
-### Sending commands and their callbacks
-
-In an asynchronous context, commands are automatically pipelined due to the nature of an event loop.
-Therefore, unlike the synchronous API, there is only a single way to send commands.
-Because commands are sent to Redis asynchronously, issuing a command requires a callback function
-that is called when the reply is received. Reply callbacks should have the following prototype:
-```c
-void(redisAsyncContext *c, void *reply, void *privdata);
-```
-The `privdata` argument can be used to curry arbitrary data to the callback from the point where
-the command is initially queued for execution.
-
-The functions that can be used to issue commands in an asynchronous context are:
-```c
-int redisAsyncCommand(
-  redisAsyncContext *ac, redisCallbackFn *fn, void *privdata,
-  const char *format, ...);
-int redisAsyncCommandArgv(
-  redisAsyncContext *ac, redisCallbackFn *fn, void *privdata,
-  int argc, const char **argv, const size_t *argvlen);
-```
-Both functions work like their blocking counterparts. The return value is `REDIS_OK` when the command
-was successfully added to the output buffer and `REDIS_ERR` otherwise. Example: when the connection
-is being disconnected per user-request, no new commands may be added to the output buffer and `REDIS_ERR` is
-returned on calls to the `redisAsyncCommand` family.
-
-If the reply for a command with a `NULL` callback is read, it is immediately freed. When the callback
-for a command is non-`NULL`, the memory is freed immediately following the callback: the reply is only
-valid for the duration of the callback.
-
-All pending callbacks are called with a `NULL` reply when the context encountered an error.
-
-### Disconnecting
-
-An asynchronous connection can be terminated using:
-```c
-void redisAsyncDisconnect(redisAsyncContext *ac);
-```
-When this function is called, the connection is **not** immediately terminated. Instead, new
-commands are no longer accepted and the connection is only terminated when all pending commands
-have been written to the socket, their respective replies have been read and their respective
-callbacks have been executed. After this, the disconnection callback is executed with the
-`REDIS_OK` status and the context object is freed.
-
-### Hooking it up to event library *X*
-
-There are a few hooks that need to be set on the context object after it is created.
-See the `adapters/` directory for bindings to *libev* and *libevent*.
-
-## Reply parsing API
-
-Hiredis comes with a reply parsing API that makes it easy for writing higher
-level language bindings.
-
-The reply parsing API consists of the following functions:
-```c
-redisReader *redisReaderCreate(void);
-void redisReaderFree(redisReader *reader);
-int redisReaderFeed(redisReader *reader, const char *buf, size_t len);
-int redisReaderGetReply(redisReader *reader, void **reply);
-```
-The same set of functions are used internally by hiredis when creating a
-normal Redis context, the above API just exposes it to the user for a direct
-usage.
-
-### Usage
-
-The function `redisReaderCreate` creates a `redisReader` structure that holds a
-buffer with unparsed data and state for the protocol parser.
-
-Incoming data -- most likely from a socket -- can be placed in the internal
-buffer of the `redisReader` using `redisReaderFeed`. This function will make a
-copy of the buffer pointed to by `buf` for `len` bytes. This data is parsed
-when `redisReaderGetReply` is called. This function returns an integer status
-and a reply object (as described above) via `void **reply`. The returned status
-can be either `REDIS_OK` or `REDIS_ERR`, where the latter means something went
-wrong (either a protocol error, or an out of memory error).
-
-The parser limits the level of nesting for multi bulk payloads to 7. If the
-multi bulk nesting level is higher than this, the parser returns an error.
-
-### Customizing replies
-
-The function `redisReaderGetReply` creates `redisReply` and makes the function
-argument `reply` point to the created `redisReply` variable. For instance, if
-the response of type `REDIS_REPLY_STATUS` then the `str` field of `redisReply`
-will hold the status as a vanilla C string. However, the functions that are
-responsible for creating instances of the `redisReply` can be customized by
-setting the `fn` field on the `redisReader` struct. This should be done
-immediately after creating the `redisReader`.
-
-For example, [hiredis-rb](https://github.com/pietern/hiredis-rb/blob/master/ext/hiredis_ext/reader.c)
-uses customized reply object functions to create Ruby objects.
-
-### Reader max buffer
-
-Both when using the Reader API directly or when using it indirectly via a
-normal Redis context, the redisReader structure uses a buffer in order to
-accumulate data from the server.
-Usually this buffer is destroyed when it is empty and is larger than 16
-KiB in order to avoid wasting memory in unused buffers
-
-However when working with very big payloads destroying the buffer may slow
-down performances considerably, so it is possible to modify the max size of
-an idle buffer changing the value of the `maxbuf` field of the reader structure
-to the desired value. The special value of 0 means that there is no maximum
-value for an idle buffer, so the buffer will never get freed.
-
-For instance if you have a normal Redis context you can set the maximum idle
-buffer to zero (unlimited) just with:
-```c
-context->reader->maxbuf = 0;
-```
-This should be done only in order to maximize performances when working with
-large payloads. The context should be set back to `REDIS_READER_MAX_BUF` again
-as soon as possible in order to prevent allocation of useless memory.
-
-## AUTHORS
-
-Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and
-Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license.
-Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and
-Jan-Erik Rediger (janerik at fnordig dot com)
diff --git a/deps/hiredis/adapters/ae.h b/deps/hiredis/adapters/ae.h
deleted file mode 100644
index 5c551c2ed..000000000
--- a/deps/hiredis/adapters/ae.h
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HIREDIS_AE_H__
-#define __HIREDIS_AE_H__
-#include <sys/types.h>
-#include <ae.h>
-#include "../hiredis.h"
-#include "../async.h"
-
-typedef struct redisAeEvents {
-    redisAsyncContext *context;
-    aeEventLoop *loop;
-    int fd;
-    int reading, writing;
-} redisAeEvents;
-
-static void redisAeReadEvent(aeEventLoop *el, int fd, void *privdata, int mask) {
-    ((void)el); ((void)fd); ((void)mask);
-
-    redisAeEvents *e = (redisAeEvents*)privdata;
-    redisAsyncHandleRead(e->context);
-}
-
-static void redisAeWriteEvent(aeEventLoop *el, int fd, void *privdata, int mask) {
-    ((void)el); ((void)fd); ((void)mask);
-
-    redisAeEvents *e = (redisAeEvents*)privdata;
-    redisAsyncHandleWrite(e->context);
-}
-
-static void redisAeAddRead(void *privdata) {
-    redisAeEvents *e = (redisAeEvents*)privdata;
-    aeEventLoop *loop = e->loop;
-    if (!e->reading) {
-        e->reading = 1;
-        aeCreateFileEvent(loop,e->fd,AE_READABLE,redisAeReadEvent,e);
-    }
-}
-
-static void redisAeDelRead(void *privdata) {
-    redisAeEvents *e = (redisAeEvents*)privdata;
-    aeEventLoop *loop = e->loop;
-    if (e->reading) {
-        e->reading = 0;
-        aeDeleteFileEvent(loop,e->fd,AE_READABLE);
-    }
-}
-
-static void redisAeAddWrite(void *privdata) {
-    redisAeEvents *e = (redisAeEvents*)privdata;
-    aeEventLoop *loop = e->loop;
-    if (!e->writing) {
-        e->writing = 1;
-        aeCreateFileEvent(loop,e->fd,AE_WRITABLE,redisAeWriteEvent,e);
-    }
-}
-
-static void redisAeDelWrite(void *privdata) {
-    redisAeEvents *e = (redisAeEvents*)privdata;
-    aeEventLoop *loop = e->loop;
-    if (e->writing) {
-        e->writing = 0;
-        aeDeleteFileEvent(loop,e->fd,AE_WRITABLE);
-    }
-}
-
-static void redisAeCleanup(void *privdata) {
-    redisAeEvents *e = (redisAeEvents*)privdata;
-    redisAeDelRead(privdata);
-    redisAeDelWrite(privdata);
-    free(e);
-}
-
-static int redisAeAttach(aeEventLoop *loop, redisAsyncContext *ac) {
-    redisContext *c = &(ac->c);
-    redisAeEvents *e;
-
-    /* Nothing should be attached when something is already attached */
-    if (ac->ev.data != NULL)
-        return REDIS_ERR;
-
-    /* Create container for context and r/w events */
-    e = (redisAeEvents*)malloc(sizeof(*e));
-    e->context = ac;
-    e->loop = loop;
-    e->fd = c->fd;
-    e->reading = e->writing = 0;
-
-    /* Register functions to start/stop listening for events */
-    ac->ev.addRead = redisAeAddRead;
-    ac->ev.delRead = redisAeDelRead;
-    ac->ev.addWrite = redisAeAddWrite;
-    ac->ev.delWrite = redisAeDelWrite;
-    ac->ev.cleanup = redisAeCleanup;
-    ac->ev.data = e;
-
-    return REDIS_OK;
-}
-#endif
diff --git a/deps/hiredis/adapters/glib.h b/deps/hiredis/adapters/glib.h
deleted file mode 100644
index e0a6411d3..000000000
--- a/deps/hiredis/adapters/glib.h
+++ /dev/null
@@ -1,153 +0,0 @@
-#ifndef __HIREDIS_GLIB_H__
-#define __HIREDIS_GLIB_H__
-
-#include <glib.h>
-
-#include "../hiredis.h"
-#include "../async.h"
-
-typedef struct
-{
-    GSource source;
-    redisAsyncContext *ac;
-    GPollFD poll_fd;
-} RedisSource;
-
-static void
-redis_source_add_read (gpointer data)
-{
-    RedisSource *source = (RedisSource *)data;
-    g_return_if_fail(source);
-    source->poll_fd.events |= G_IO_IN;
-    g_main_context_wakeup(g_source_get_context((GSource *)data));
-}
-
-static void
-redis_source_del_read (gpointer data)
-{
-    RedisSource *source = (RedisSource *)data;
-    g_return_if_fail(source);
-    source->poll_fd.events &= ~G_IO_IN;
-    g_main_context_wakeup(g_source_get_context((GSource *)data));
-}
-
-static void
-redis_source_add_write (gpointer data)
-{
-    RedisSource *source = (RedisSource *)data;
-    g_return_if_fail(source);
-    source->poll_fd.events |= G_IO_OUT;
-    g_main_context_wakeup(g_source_get_context((GSource *)data));
-}
-
-static void
-redis_source_del_write (gpointer data)
-{
-    RedisSource *source = (RedisSource *)data;
-    g_return_if_fail(source);
-    source->poll_fd.events &= ~G_IO_OUT;
-    g_main_context_wakeup(g_source_get_context((GSource *)data));
-}
-
-static void
-redis_source_cleanup (gpointer data)
-{
-    RedisSource *source = (RedisSource *)data;
-
-    g_return_if_fail(source);
-
-    redis_source_del_read(source);
-    redis_source_del_write(source);
-    /*
-     * It is not our responsibility to remove ourself from the
-     * current main loop. However, we will remove the GPollFD.
-     */
-    if (source->poll_fd.fd >= 0) {
-        g_source_remove_poll((GSource *)data, &source->poll_fd);
-        source->poll_fd.fd = -1;
-    }
-}
-
-static gboolean
-redis_source_prepare (GSource *source,
-                      gint    *timeout_)
-{
-    RedisSource *redis = (RedisSource *)source;
-    *timeout_ = -1;
-    return !!(redis->poll_fd.events & redis->poll_fd.revents);
-}
-
-static gboolean
-redis_source_check (GSource *source)
-{
-    RedisSource *redis = (RedisSource *)source;
-    return !!(redis->poll_fd.events & redis->poll_fd.revents);
-}
-
-static gboolean
-redis_source_dispatch (GSource      *source,
-                       GSourceFunc   callback,
-                       gpointer      user_data)
-{
-    RedisSource *redis = (RedisSource *)source;
-
-    if ((redis->poll_fd.revents & G_IO_OUT)) {
-        redisAsyncHandleWrite(redis->ac);
-        redis->poll_fd.revents &= ~G_IO_OUT;
-    }
-
-    if ((redis->poll_fd.revents & G_IO_IN)) {
-        redisAsyncHandleRead(redis->ac);
-        redis->poll_fd.revents &= ~G_IO_IN;
-    }
-
-    if (callback) {
-        return callback(user_data);
-    }
-
-    return TRUE;
-}
-
-static void
-redis_source_finalize (GSource *source)
-{
-    RedisSource *redis = (RedisSource *)source;
-
-    if (redis->poll_fd.fd >= 0) {
-        g_source_remove_poll(source, &redis->poll_fd);
-        redis->poll_fd.fd = -1;
-    }
-}
-
-static GSource *
-redis_source_new (redisAsyncContext *ac)
-{
-    static GSourceFuncs source_funcs = {
-        .prepare  = redis_source_prepare,
-        .check     = redis_source_check,
-        .dispatch = redis_source_dispatch,
-        .finalize = redis_source_finalize,
-    };
-    redisContext *c = &ac->c;
-    RedisSource *source;
-
-    g_return_val_if_fail(ac != NULL, NULL);
-
-    source = (RedisSource *)g_source_new(&source_funcs, sizeof *source);
-    source->ac = ac;
-    source->poll_fd.fd = c->fd;
-    source->poll_fd.events = 0;
-    source->poll_fd.revents = 0;
-    g_source_add_poll((GSource *)source, &source->poll_fd);
-
-    ac->ev.addRead = redis_source_add_read;
-    ac->ev.delRead = redis_source_del_read;
-    ac->ev.addWrite = redis_source_add_write;
-    ac->ev.delWrite = redis_source_del_write;
-    ac->ev.cleanup = redis_source_cleanup;
-    ac->ev.data = source;
-
-    return (GSource *)source;
-}
-
-#endif /* __HIREDIS_GLIB_H__ */
diff --git a/deps/hiredis/adapters/ivykis.h b/deps/hiredis/adapters/ivykis.h
deleted file mode 100644
index 6a12a868a..000000000
--- a/deps/hiredis/adapters/ivykis.h
+++ /dev/null
@@ -1,81 +0,0 @@
-#ifndef __HIREDIS_IVYKIS_H__
-#define __HIREDIS_IVYKIS_H__
-#include <iv.h>
-#include "../hiredis.h"
-#include "../async.h"
-
-typedef struct redisIvykisEvents {
-    redisAsyncContext *context;
-    struct iv_fd fd;
-} redisIvykisEvents;
-
-static void redisIvykisReadEvent(void *arg) {
-    redisAsyncContext *context = (redisAsyncContext *)arg;
-    redisAsyncHandleRead(context);
-}
-
-static void redisIvykisWriteEvent(void *arg) {
-    redisAsyncContext *context = (redisAsyncContext *)arg;
-    redisAsyncHandleWrite(context);
-}
-
-static void redisIvykisAddRead(void *privdata) {
-    redisIvykisEvents *e = (redisIvykisEvents*)privdata;
-    iv_fd_set_handler_in(&e->fd, redisIvykisReadEvent);
-}
-
-static void redisIvykisDelRead(void *privdata) {
-    redisIvykisEvents *e = (redisIvykisEvents*)privdata;
-    iv_fd_set_handler_in(&e->fd, NULL);
-}
-
-static void redisIvykisAddWrite(void *privdata) {
-    redisIvykisEvents *e = (redisIvykisEvents*)privdata;
-    iv_fd_set_handler_out(&e->fd, redisIvykisWriteEvent);
-}
-
-static void redisIvykisDelWrite(void *privdata) {
-    redisIvykisEvents *e = (redisIvykisEvents*)privdata;
-    iv_fd_set_handler_out(&e->fd, NULL);
-}
-
-static void redisIvykisCleanup(void *privdata) {
-    redisIvykisEvents *e = (redisIvykisEvents*)privdata;
-
-    iv_fd_unregister(&e->fd);
-    free(e);
-}
-
-static int redisIvykisAttach(redisAsyncContext *ac) {
-    redisContext *c = &(ac->c);
-    redisIvykisEvents *e;
-
-    /* Nothing should be attached when something is already attached */
-    if (ac->ev.data != NULL)
-        return REDIS_ERR;
-
-    /* Create container for context and r/w events */
-    e = (redisIvykisEvents*)malloc(sizeof(*e));
-    e->context = ac;
-
-    /* Register functions to start/stop listening for events */
-    ac->ev.addRead = redisIvykisAddRead;
-    ac->ev.delRead = redisIvykisDelRead;
-    ac->ev.addWrite = redisIvykisAddWrite;
-    ac->ev.delWrite = redisIvykisDelWrite;
-    ac->ev.cleanup = redisIvykisCleanup;
-    ac->ev.data = e;
-
-    /* Initialize and install read/write events */
-    IV_FD_INIT(&e->fd);
-    e->fd.fd = c->fd;
-    e->fd.handler_in = redisIvykisReadEvent;
-    e->fd.handler_out = redisIvykisWriteEvent;
-    e->fd.handler_err = NULL;
-    e->fd.cookie = e->context;
-
-    iv_fd_register(&e->fd);
-
-    return REDIS_OK;
-}
-#endif
diff --git a/deps/hiredis/adapters/libev.h b/deps/hiredis/adapters/libev.h
deleted file mode 100644
index 2bf8d521f..000000000
--- a/deps/hiredis/adapters/libev.h
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HIREDIS_LIBEV_H__
-#define __HIREDIS_LIBEV_H__
-#include <stdlib.h>
-#include <sys/types.h>
-#include <ev.h>
-#include "../hiredis.h"
-#include "../async.h"
-
-typedef struct redisLibevEvents {
-    redisAsyncContext *context;
-    struct ev_loop *loop;
-    int reading, writing;
-    ev_io rev, wev;
-} redisLibevEvents;
-
-static void redisLibevReadEvent(EV_P_ ev_io *watcher, int revents) {
-#if EV_MULTIPLICITY
-    ((void)loop);
-#endif
-    ((void)revents);
-
-    redisLibevEvents *e = (redisLibevEvents*)watcher->data;
-    redisAsyncHandleRead(e->context);
-}
-
-static void redisLibevWriteEvent(EV_P_ ev_io *watcher, int revents) {
-#if EV_MULTIPLICITY
-    ((void)loop);
-#endif
-    ((void)revents);
-
-    redisLibevEvents *e = (redisLibevEvents*)watcher->data;
-    redisAsyncHandleWrite(e->context);
-}
-
-static void redisLibevAddRead(void *privdata) {
-    redisLibevEvents *e = (redisLibevEvents*)privdata;
-    struct ev_loop *loop = e->loop;
-    ((void)loop);
-    if (!e->reading) {
-        e->reading = 1;
-        ev_io_start(EV_A_ &e->rev);
-    }
-}
-
-static void redisLibevDelRead(void *privdata) {
-    redisLibevEvents *e = (redisLibevEvents*)privdata;
-    struct ev_loop *loop = e->loop;
-    ((void)loop);
-    if (e->reading) {
-        e->reading = 0;
-        ev_io_stop(EV_A_ &e->rev);
-    }
-}
-
-static void redisLibevAddWrite(void *privdata) {
-    redisLibevEvents *e = (redisLibevEvents*)privdata;
-    struct ev_loop *loop = e->loop;
-    ((void)loop);
-    if (!e->writing) {
-        e->writing = 1;
-        ev_io_start(EV_A_ &e->wev);
-    }
-}
-
-static void redisLibevDelWrite(void *privdata) {
-    redisLibevEvents *e = (redisLibevEvents*)privdata;
-    struct ev_loop *loop = e->loop;
-    ((void)loop);
-    if (e->writing) {
-        e->writing = 0;
-        ev_io_stop(EV_A_ &e->wev);
-    }
-}
-
-static void redisLibevCleanup(void *privdata) {
-    redisLibevEvents *e = (redisLibevEvents*)privdata;
-    redisLibevDelRead(privdata);
-    redisLibevDelWrite(privdata);
-    free(e);
-}
-
-static int redisLibevAttach(EV_P_ redisAsyncContext *ac) {
-    redisContext *c = &(ac->c);
-    redisLibevEvents *e;
-
-    /* Nothing should be attached when something is already attached */
-    if (ac->ev.data != NULL)
-        return REDIS_ERR;
-
-    /* Create container for context and r/w events */
-    e = (redisLibevEvents*)malloc(sizeof(*e));
-    e->context = ac;
-#if EV_MULTIPLICITY
-    e->loop = loop;
-#else
-    e->loop = NULL;
-#endif
-    e->reading = e->writing = 0;
-    e->rev.data = e;
-    e->wev.data = e;
-
-    /* Register functions to start/stop listening for events */
-    ac->ev.addRead = redisLibevAddRead;
-    ac->ev.delRead = redisLibevDelRead;
-    ac->ev.addWrite = redisLibevAddWrite;
-    ac->ev.delWrite = redisLibevDelWrite;
-    ac->ev.cleanup = redisLibevCleanup;
-    ac->ev.data = e;
-
-    /* Initialize read/write events */
-    ev_io_init(&e->rev,redisLibevReadEvent,c->fd,EV_READ);
-    ev_io_init(&e->wev,redisLibevWriteEvent,c->fd,EV_WRITE);
-    return REDIS_OK;
-}
-
-#endif
diff --git a/deps/hiredis/adapters/libevent.h b/deps/hiredis/adapters/libevent.h
deleted file mode 100644
index a4952776c..000000000
--- a/deps/hiredis/adapters/libevent.h
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HIREDIS_LIBEVENT_H__
-#define __HIREDIS_LIBEVENT_H__
-#include <event2/event.h>
-#include "../hiredis.h"
-#include "../async.h"
-
-#define REDIS_LIBEVENT_DELETED 0x01
-#define REDIS_LIBEVENT_ENTERED 0x02
-
-typedef struct redisLibeventEvents {
-    redisAsyncContext *context;
-    struct event *ev;
-    struct event_base *base;
-    struct timeval tv;
-    short flags;
-    short state;
-} redisLibeventEvents;
-
-static void redisLibeventDestroy(redisLibeventEvents *e) {
-    free(e);
-}
-
-static void redisLibeventHandler(int fd, short event, void *arg) {
-    ((void)fd);
-    redisLibeventEvents *e = (redisLibeventEvents*)arg;
-    e->state |= REDIS_LIBEVENT_ENTERED;
-
-    #define CHECK_DELETED() if (e->state & REDIS_LIBEVENT_DELETED) {\
-        redisLibeventDestroy(e);\
-        return; \
-    }
-
-    if ((event & EV_TIMEOUT) && (e->state & REDIS_LIBEVENT_DELETED) == 0) {
-        redisAsyncHandleTimeout(e->context);
-        CHECK_DELETED();
-    }
-
-    if ((event & EV_READ) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) {
-        redisAsyncHandleRead(e->context);
-        CHECK_DELETED();
-    }
-
-    if ((event & EV_WRITE) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) {
-        redisAsyncHandleWrite(e->context);
-        CHECK_DELETED();
-    }
-
-    e->state &= ~REDIS_LIBEVENT_ENTERED;
-    #undef CHECK_DELETED
-}
-
-static void redisLibeventUpdate(void *privdata, short flag, int isRemove) {
-    redisLibeventEvents *e = (redisLibeventEvents *)privdata;
-    const struct timeval *tv = e->tv.tv_sec || e->tv.tv_usec ? &e->tv : NULL;
-
-    if (isRemove) {
-        if ((e->flags & flag) == 0) {
-            return;
-        } else {
-            e->flags &= ~flag;
-        }
-    } else {
-        if (e->flags & flag) {
-            return;
-        } else {
-            e->flags |= flag;
-        }
-    }
-
-    event_del(e->ev);
-    event_assign(e->ev, e->base, e->context->c.fd, e->flags | EV_PERSIST,
-                 redisLibeventHandler, privdata);
-    event_add(e->ev, tv);
-}
-
-static void redisLibeventAddRead(void *privdata) {
-    redisLibeventUpdate(privdata, EV_READ, 0);
-}
-
-static void redisLibeventDelRead(void *privdata) {
-    redisLibeventUpdate(privdata, EV_READ, 1);
-}
-
-static void redisLibeventAddWrite(void *privdata) {
-    redisLibeventUpdate(privdata, EV_WRITE, 0);
-}
-
-static void redisLibeventDelWrite(void *privdata) {
-    redisLibeventUpdate(privdata, EV_WRITE, 1);
-}
-
-static void redisLibeventCleanup(void *privdata) {
-    redisLibeventEvents *e = (redisLibeventEvents*)privdata;
-    if (!e) {
-        return;
-    }
-    event_del(e->ev);
-    event_free(e->ev);
-    e->ev = NULL;
-
-    if (e->state & REDIS_LIBEVENT_ENTERED) {
-        e->state |= REDIS_LIBEVENT_DELETED;
-    } else {
-        redisLibeventDestroy(e);
-    }
-}
-
-static void redisLibeventSetTimeout(void *privdata, struct timeval tv) {
-    redisLibeventEvents *e = (redisLibeventEvents *)privdata;
-    short flags = e->flags;
-    e->flags = 0;
-    e->tv = tv;
-    redisLibeventUpdate(e, flags, 0);
-}
-
-static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) {
-    redisContext *c = &(ac->c);
-    redisLibeventEvents *e;
-
-    /* Nothing should be attached when something is already attached */
-    if (ac->ev.data != NULL)
-        return REDIS_ERR;
-
-    /* Create container for context and r/w events */
-    e = (redisLibeventEvents*)calloc(1, sizeof(*e));
-    e->context = ac;
-
-    /* Register functions to start/stop listening for events */
-    ac->ev.addRead = redisLibeventAddRead;
-    ac->ev.delRead = redisLibeventDelRead;
-    ac->ev.addWrite = redisLibeventAddWrite;
-    ac->ev.delWrite = redisLibeventDelWrite;
-    ac->ev.cleanup = redisLibeventCleanup;
-    ac->ev.scheduleTimer = redisLibeventSetTimeout;
-    ac->ev.data = e;
-
-    /* Initialize and install read/write events */
-    e->ev = event_new(base, c->fd, EV_READ | EV_WRITE, redisLibeventHandler, e);
-    e->base = base;
-    return REDIS_OK;
-}
-#endif
diff --git a/deps/hiredis/adapters/libuv.h b/deps/hiredis/adapters/libuv.h
deleted file mode 100644
index 39ef7cf5e..000000000
--- a/deps/hiredis/adapters/libuv.h
+++ /dev/null
@@ -1,119 +0,0 @@
-#ifndef __HIREDIS_LIBUV_H__
-#define __HIREDIS_LIBUV_H__
-#include <stdlib.h>
-#include <uv.h>
-#include "../hiredis.h"
-#include "../async.h"
-#include <string.h>
-
-typedef struct redisLibuvEvents {
-  redisAsyncContext* context;
-  uv_poll_t          handle;
-  int                events;
-} redisLibuvEvents;
-
-
-static void redisLibuvPoll(uv_poll_t* handle, int status, int events) {
-  redisLibuvEvents* p = (redisLibuvEvents*)handle->data;
-  int ev = (status ? p->events : events);
-
-  if (p->context != NULL && (ev & UV_READABLE)) {
-    redisAsyncHandleRead(p->context);
-  }
-  if (p->context != NULL && (ev & UV_WRITABLE)) {
-    redisAsyncHandleWrite(p->context);
-  }
-}
-
-
-static void redisLibuvAddRead(void *privdata) {
-  redisLibuvEvents* p = (redisLibuvEvents*)privdata;
-
-  p->events |= UV_READABLE;
-
-  uv_poll_start(&p->handle, p->events, redisLibuvPoll);
-}
-
-
-static void redisLibuvDelRead(void *privdata) {
-  redisLibuvEvents* p = (redisLibuvEvents*)privdata;
-
-  p->events &= ~UV_READABLE;
-
-  if (p->events) {
-    uv_poll_start(&p->handle, p->events, redisLibuvPoll);
-  } else {
-    uv_poll_stop(&p->handle);
-  }
-}
-
-
-static void redisLibuvAddWrite(void *privdata) {
-  redisLibuvEvents* p = (redisLibuvEvents*)privdata;
-
-  p->events |= UV_WRITABLE;
-
-  uv_poll_start(&p->handle, p->events, redisLibuvPoll);
-}
-
-
-static void redisLibuvDelWrite(void *privdata) {
-  redisLibuvEvents* p = (redisLibuvEvents*)privdata;
-
-  p->events &= ~UV_WRITABLE;
-
-  if (p->events) {
-    uv_poll_start(&p->handle, p->events, redisLibuvPoll);
-  } else {
-    uv_poll_stop(&p->handle);
-  }
-}
-
-
-static void on_close(uv_handle_t* handle) {
-  redisLibuvEvents* p = (redisLibuvEvents*)handle->data;
-
-  free(p);
-}
-
-
-static void redisLibuvCleanup(void *privdata) {
-  redisLibuvEvents* p = (redisLibuvEvents*)privdata;
-
-  p->context = NULL; // indicate that context might no longer exist
-  uv_close((uv_handle_t*)&p->handle, on_close);
-}
-
-
-static int redisLibuvAttach(redisAsyncContext* ac, uv_loop_t* loop) {
-  redisContext *c = &(ac->c);
-
-  if (ac->ev.data != NULL) {
-    return REDIS_ERR;
-  }
-
-  ac->ev.addRead  = redisLibuvAddRead;
-  ac->ev.delRead  = redisLibuvDelRead;
-  ac->ev.addWrite = redisLibuvAddWrite;
-  ac->ev.delWrite = redisLibuvDelWrite;
-  ac->ev.cleanup  = redisLibuvCleanup;
-
-  redisLibuvEvents* p = (redisLibuvEvents*)malloc(sizeof(*p));
-
-  if (!p) {
-    return REDIS_ERR;
-  }
-
-  memset(p, 0, sizeof(*p));
-
-  if (uv_poll_init(loop, &p->handle, c->fd) != 0) {
-    return REDIS_ERR;
-  }
-
-  ac->ev.data    = p;
-  p->handle.data = p;
-  p->context     = ac;
-
-  return REDIS_OK;
-}
-#endif
diff --git a/deps/hiredis/adapters/macosx.h b/deps/hiredis/adapters/macosx.h
deleted file mode 100644
index 72121f606..000000000
--- a/deps/hiredis/adapters/macosx.h
+++ /dev/null
@@ -1,114 +0,0 @@
-//
-//  Created by Дмитрий Бахвалов on 13.07.15.
-//  Copyright (c) 2015 Dmitry Bakhvalov. All rights reserved.
-//
-
-#ifndef __HIREDIS_MACOSX_H__
-#define __HIREDIS_MACOSX_H__
-
-#include <CoreFoundation/CoreFoundation.h>
-
-#include "../hiredis.h"
-#include "../async.h"
-
-typedef struct {
-    redisAsyncContext *context;
-    CFSocketRef socketRef;
-    CFRunLoopSourceRef sourceRef;
-} RedisRunLoop;
-
-static int freeRedisRunLoop(RedisRunLoop* redisRunLoop) {
-    if( redisRunLoop != NULL ) {
-        if( redisRunLoop->sourceRef != NULL ) {
-            CFRunLoopSourceInvalidate(redisRunLoop->sourceRef);
-            CFRelease(redisRunLoop->sourceRef);
-        }
-        if( redisRunLoop->socketRef != NULL ) {
-            CFSocketInvalidate(redisRunLoop->socketRef);
-            CFRelease(redisRunLoop->socketRef);
-        }
-        free(redisRunLoop);
-    }
-    return REDIS_ERR;
-}
-
-static void redisMacOSAddRead(void *privdata) {
-    RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
-    CFSocketEnableCallBacks(redisRunLoop->socketRef, kCFSocketReadCallBack);
-}
-
-static void redisMacOSDelRead(void *privdata) {
-    RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
-    CFSocketDisableCallBacks(redisRunLoop->socketRef, kCFSocketReadCallBack);
-}
-
-static void redisMacOSAddWrite(void *privdata) {
-    RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
-    CFSocketEnableCallBacks(redisRunLoop->socketRef, kCFSocketWriteCallBack);
-}
-
-static void redisMacOSDelWrite(void *privdata) {
-    RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
-    CFSocketDisableCallBacks(redisRunLoop->socketRef, kCFSocketWriteCallBack);
-}
-
-static void redisMacOSCleanup(void *privdata) {
-    RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
-    freeRedisRunLoop(redisRunLoop);
-}
-
-static void redisMacOSAsyncCallback(CFSocketRef __unused s, CFSocketCallBackType callbackType, CFDataRef __unused address, const void __unused *data, void *info) {
-    redisAsyncContext* context = (redisAsyncContext*) info;
-
-    switch (callbackType) {
-        case kCFSocketReadCallBack:
-            redisAsyncHandleRead(context);
-            break;
-
-        case kCFSocketWriteCallBack:
-            redisAsyncHandleWrite(context);
-            break;
-
-        default:
-            break;
-    }
-}
-
-static int redisMacOSAttach(redisAsyncContext *redisAsyncCtx, CFRunLoopRef runLoop) {
-    redisContext *redisCtx = &(redisAsyncCtx->c);
-
-    /* Nothing should be attached when something is already attached */
-    if( redisAsyncCtx->ev.data != NULL ) return REDIS_ERR;
-
-    RedisRunLoop* redisRunLoop = (RedisRunLoop*) calloc(1, sizeof(RedisRunLoop));
-    if( !redisRunLoop ) return REDIS_ERR;
-
-    /* Setup redis stuff */
-    redisRunLoop->context = redisAsyncCtx;
-
-    redisAsyncCtx->ev.addRead  = redisMacOSAddRead;
-    redisAsyncCtx->ev.delRead  = redisMacOSDelRead;
-    redisAsyncCtx->ev.addWrite = redisMacOSAddWrite;
-    redisAsyncCtx->ev.delWrite = redisMacOSDelWrite;
-    redisAsyncCtx->ev.cleanup  = redisMacOSCleanup;
-    redisAsyncCtx->ev.data     = redisRunLoop;
-
-    /* Initialize and install read/write events */
-    CFSocketContext socketCtx = { 0, redisAsyncCtx, NULL, NULL, NULL };
-
-    redisRunLoop->socketRef = CFSocketCreateWithNative(NULL, redisCtx->fd,
-                                                       kCFSocketReadCallBack | kCFSocketWriteCallBack,
-                                                       redisMacOSAsyncCallback,
-                                                       &socketCtx);
-    if( !redisRunLoop->socketRef ) return freeRedisRunLoop(redisRunLoop);
-
-    redisRunLoop->sourceRef = CFSocketCreateRunLoopSource(NULL, redisRunLoop->socketRef, 0);
-    if( !redisRunLoop->sourceRef ) return freeRedisRunLoop(redisRunLoop);
-
-    CFRunLoopAddSource(runLoop, redisRunLoop->sourceRef, kCFRunLoopDefaultMode);
-
-    return REDIS_OK;
-}
-
-#endif
-
diff --git a/deps/hiredis/adapters/qt.h b/deps/hiredis/adapters/qt.h
deleted file mode 100644
index 5cc02e6ce..000000000
--- a/deps/hiredis/adapters/qt.h
+++ /dev/null
@@ -1,135 +0,0 @@
-/*-
- * Copyright (C) 2014 Pietro Cerutti <gahr@gahr.ch>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#ifndef __HIREDIS_QT_H__
-#define __HIREDIS_QT_H__
-#include <QSocketNotifier>
-#include "../async.h"
-
-static void RedisQtAddRead(void *);
-static void RedisQtDelRead(void *);
-static void RedisQtAddWrite(void *);
-static void RedisQtDelWrite(void *);
-static void RedisQtCleanup(void *);
-
-class RedisQtAdapter : public QObject {
-
-    Q_OBJECT
-
-    friend
-    void RedisQtAddRead(void * adapter) {
-        RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
-        a->addRead();
-    }
-
-    friend
-    void RedisQtDelRead(void * adapter) {
-        RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
-        a->delRead();
-    }
-
-    friend
-    void RedisQtAddWrite(void * adapter) {
-        RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
-        a->addWrite();
-    }
-
-    friend
-    void RedisQtDelWrite(void * adapter) {
-        RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
-        a->delWrite();
-    }
-
-    friend
-    void RedisQtCleanup(void * adapter) {
-        RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
-        a->cleanup();
-    }
-
-    public:
-        RedisQtAdapter(QObject * parent = 0)
-            : QObject(parent), m_ctx(0), m_read(0), m_write(0) { }
-
-        ~RedisQtAdapter() {
-            if (m_ctx != 0) {
-                m_ctx->ev.data = NULL;
-            }
-        }
-
-        int setContext(redisAsyncContext * ac) {
-            if (ac->ev.data != NULL) {
-                return REDIS_ERR;
-            }
-            m_ctx = ac;
-            m_ctx->ev.data = this;
-            m_ctx->ev.addRead = RedisQtAddRead;
-            m_ctx->ev.delRead = RedisQtDelRead;
-            m_ctx->ev.addWrite = RedisQtAddWrite;
-            m_ctx->ev.delWrite = RedisQtDelWrite;
-            m_ctx->ev.cleanup = RedisQtCleanup;
-            return REDIS_OK;
-        }
-
-    private:
-        void addRead() {
-            if (m_read) return;
-            m_read = new QSocketNotifier(m_ctx->c.fd, QSocketNotifier::Read, 0);
-            connect(m_read, SIGNAL(activated(int)), this, SLOT(read()));
-        }
-
-        void delRead() {
-            if (!m_read) return;
-            delete m_read;
-            m_read = 0;
-        }
-
-        void addWrite() {
-            if (m_write) return;
-            m_write = new QSocketNotifier(m_ctx->c.fd, QSocketNotifier::Write, 0);
-            connect(m_write, SIGNAL(activated(int)), this, SLOT(write()));
-        }
-
-        void delWrite() {
-            if (!m_write) return;
-            delete m_write;
-            m_write = 0;
-        }
-
-        void cleanup() {
-            delRead();
-            delWrite();
-        }
-
-    private slots:
-        void read() { redisAsyncHandleRead(m_ctx); }
-        void write() { redisAsyncHandleWrite(m_ctx); }
-
-    private:
-        redisAsyncContext * m_ctx;
-        QSocketNotifier * m_read;
-        QSocketNotifier * m_write;
-};
-
-#endif /* !__HIREDIS_QT_H__ */
diff --git a/deps/hiredis/appveyor.yml b/deps/hiredis/appveyor.yml
deleted file mode 100644
index 5b43fdbeb..000000000
--- a/deps/hiredis/appveyor.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-# Appveyor configuration file for CI build of hiredis on Windows (under Cygwin)
-environment:
-    matrix:
-        - CYG_BASH: C:\cygwin64\bin\bash
-          CC: gcc
-        - CYG_BASH: C:\cygwin\bin\bash
-          CC: gcc
-          CFLAGS: -m32
-          CXXFLAGS: -m32
-          LDFLAGS: -m32
-
-clone_depth: 1
-
-# Attempt to ensure we don't try to convert line endings to Win32 CRLF as this will cause build to fail
-init:
-    - git config --global core.autocrlf input
-
-# Install needed build dependencies
-install:
-    - '%CYG_BASH% -lc "cygcheck -dc cygwin"'
-
-build_script:
-    - 'echo building...'
-    - '%CYG_BASH% -lc "cd $APPVEYOR_BUILD_FOLDER; exec 0</dev/null; mkdir build && cd build && cmake .. -G \"Unix Makefiles\" && make VERBOSE=1"'
diff --git a/deps/hiredis/async.c b/deps/hiredis/async.c
deleted file mode 100644
index 4f422d566..000000000
--- a/deps/hiredis/async.c
+++ /dev/null
@@ -1,766 +0,0 @@
-/*
- * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "fmacros.h"
-#include <stdlib.h>
-#include <string.h>
-#ifndef _MSC_VER
-#include <strings.h>
-#endif
-#include <assert.h>
-#include <ctype.h>
-#include <errno.h>
-#include "async.h"
-#include "net.h"
-#include "dict.c"
-#include "sds.h"
-#include "win32.h"
-
-#include "async_private.h"
-
-/* Forward declaration of function in hiredis.c */
-int __redisAppendCommand(redisContext *c, const char *cmd, size_t len);
-
-/* Functions managing dictionary of callbacks for pub/sub. */
-static unsigned int callbackHash(const void *key) {
-    return dictGenHashFunction((const unsigned char *)key,
-                               sdslen((const sds)key));
-}
-
-static void *callbackValDup(void *privdata, const void *src) {
-    ((void) privdata);
-    redisCallback *dup = malloc(sizeof(*dup));
-    memcpy(dup,src,sizeof(*dup));
-    return dup;
-}
-
-static int callbackKeyCompare(void *privdata, const void *key1, const void *key2) {
-    int l1, l2;
-    ((void) privdata);
-
-    l1 = sdslen((const sds)key1);
-    l2 = sdslen((const sds)key2);
-    if (l1 != l2) return 0;
-    return memcmp(key1,key2,l1) == 0;
-}
-
-static void callbackKeyDestructor(void *privdata, void *key) {
-    ((void) privdata);
-    sdsfree((sds)key);
-}
-
-static void callbackValDestructor(void *privdata, void *val) {
-    ((void) privdata);
-    free(val);
-}
-
-static dictType callbackDict = {
-    callbackHash,
-    NULL,
-    callbackValDup,
-    callbackKeyCompare,
-    callbackKeyDestructor,
-    callbackValDestructor
-};
-
-static redisAsyncContext *redisAsyncInitialize(redisContext *c) {
-    redisAsyncContext *ac;
-
-    ac = realloc(c,sizeof(redisAsyncContext));
-    if (ac == NULL)
-        return NULL;
-
-    c = &(ac->c);
-
-    /* The regular connect functions will always set the flag REDIS_CONNECTED.
-     * For the async API, we want to wait until the first write event is
-     * received up before setting this flag, so reset it here. */
-    c->flags &= ~REDIS_CONNECTED;
-
-    ac->err = 0;
-    ac->errstr = NULL;
-    ac->data = NULL;
-
-    ac->ev.data = NULL;
-    ac->ev.addRead = NULL;
-    ac->ev.delRead = NULL;
-    ac->ev.addWrite = NULL;
-    ac->ev.delWrite = NULL;
-    ac->ev.cleanup = NULL;
-    ac->ev.scheduleTimer = NULL;
-
-    ac->onConnect = NULL;
-    ac->onDisconnect = NULL;
-
-    ac->replies.head = NULL;
-    ac->replies.tail = NULL;
-    ac->sub.invalid.head = NULL;
-    ac->sub.invalid.tail = NULL;
-    ac->sub.channels = dictCreate(&callbackDict,NULL);
-    ac->sub.patterns = dictCreate(&callbackDict,NULL);
-    return ac;
-}
-
-/* We want the error field to be accessible directly instead of requiring
- * an indirection to the redisContext struct. */
-static void __redisAsyncCopyError(redisAsyncContext *ac) {
-    if (!ac)
-        return;
-
-    redisContext *c = &(ac->c);
-    ac->err = c->err;
-    ac->errstr = c->errstr;
-}
-
-redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options) {
-    redisOptions myOptions = *options;
-    redisContext *c;
-    redisAsyncContext *ac;
-
-    myOptions.options |= REDIS_OPT_NONBLOCK;
-    c = redisConnectWithOptions(&myOptions);
-    if (c == NULL) {
-        return NULL;
-    }
-    ac = redisAsyncInitialize(c);
-    if (ac == NULL) {
-        redisFree(c);
-        return NULL;
-    }
-    __redisAsyncCopyError(ac);
-    return ac;
-}
-
-redisAsyncContext *redisAsyncConnect(const char *ip, int port) {
-    redisOptions options = {0};
-    REDIS_OPTIONS_SET_TCP(&options, ip, port);
-    return redisAsyncConnectWithOptions(&options);
-}
-
-redisAsyncContext *redisAsyncConnectBind(const char *ip, int port,
-                                         const char *source_addr) {
-    redisOptions options = {0};
-    REDIS_OPTIONS_SET_TCP(&options, ip, port);
-    options.endpoint.tcp.source_addr = source_addr;
-    return redisAsyncConnectWithOptions(&options);
-}
-
-redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port,
-                                                  const char *source_addr) {
-    redisOptions options = {0};
-    REDIS_OPTIONS_SET_TCP(&options, ip, port);
-    options.options |= REDIS_OPT_REUSEADDR;
-    options.endpoint.tcp.source_addr = source_addr;
-    return redisAsyncConnectWithOptions(&options);
-}
-
-redisAsyncContext *redisAsyncConnectUnix(const char *path) {
-    redisOptions options = {0};
-    REDIS_OPTIONS_SET_UNIX(&options, path);
-    return redisAsyncConnectWithOptions(&options);
-}
-
-int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn) {
-    if (ac->onConnect == NULL) {
-        ac->onConnect = fn;
-
-        /* The common way to detect an established connection is to wait for
-         * the first write event to be fired. This assumes the related event
-         * library functions are already set. */
-        _EL_ADD_WRITE(ac);
-        return REDIS_OK;
-    }
-    return REDIS_ERR;
-}
-
-int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn) {
-    if (ac->onDisconnect == NULL) {
-        ac->onDisconnect = fn;
-        return REDIS_OK;
-    }
-    return REDIS_ERR;
-}
-
-/* Helper functions to push/shift callbacks */
-static int __redisPushCallback(redisCallbackList *list, redisCallback *source) {
-    redisCallback *cb;
-
-    /* Copy callback from stack to heap */
-    cb = malloc(sizeof(*cb));
-    if (cb == NULL)
-        return REDIS_ERR_OOM;
-
-    if (source != NULL) {
-        memcpy(cb,source,sizeof(*cb));
-        cb->next = NULL;
-    }
-
-    /* Store callback in list */
-    if (list->head == NULL)
-        list->head = cb;
-    if (list->tail != NULL)
-        list->tail->next = cb;
-    list->tail = cb;
-    return REDIS_OK;
-}
-
-static int __redisShiftCallback(redisCallbackList *list, redisCallback *target) {
-    redisCallback *cb = list->head;
-    if (cb != NULL) {
-        list->head = cb->next;
-        if (cb == list->tail)
-            list->tail = NULL;
-
-        /* Copy callback from heap to stack */
-        if (target != NULL)
-            memcpy(target,cb,sizeof(*cb));
-        free(cb);
-        return REDIS_OK;
-    }
-    return REDIS_ERR;
-}
-
-static void __redisRunCallback(redisAsyncContext *ac, redisCallback *cb, redisReply *reply) {
-    redisContext *c = &(ac->c);
-    if (cb->fn != NULL) {
-        c->flags |= REDIS_IN_CALLBACK;
-        cb->fn(ac,reply,cb->privdata);
-        c->flags &= ~REDIS_IN_CALLBACK;
-    }
-}
-
-/* Helper function to free the context. */
-static void __redisAsyncFree(redisAsyncContext *ac) {
-    redisContext *c = &(ac->c);
-    redisCallback cb;
-    dictIterator *it;
-    dictEntry *de;
-
-    /* Execute pending callbacks with NULL reply. */
-    while (__redisShiftCallback(&ac->replies,&cb) == REDIS_OK)
-        __redisRunCallback(ac,&cb,NULL);
-
-    /* Execute callbacks for invalid commands */
-    while (__redisShiftCallback(&ac->sub.invalid,&cb) == REDIS_OK)
-        __redisRunCallback(ac,&cb,NULL);
-
-    /* Run subscription callbacks callbacks with NULL reply */
-    it = dictGetIterator(ac->sub.channels);
-    while ((de = dictNext(it)) != NULL)
-        __redisRunCallback(ac,dictGetEntryVal(de),NULL);
-    dictReleaseIterator(it);
-    dictRelease(ac->sub.channels);
-
-    it = dictGetIterator(ac->sub.patterns);
-    while ((de = dictNext(it)) != NULL)
-        __redisRunCallback(ac,dictGetEntryVal(de),NULL);
-    dictReleaseIterator(it);
-    dictRelease(ac->sub.patterns);
-
-    /* Signal event lib to clean up */
-    _EL_CLEANUP(ac);
-
-    /* Execute disconnect callback. When redisAsyncFree() initiated destroying
-     * this context, the status will always be REDIS_OK. */
-    if (ac->onDisconnect && (c->flags & REDIS_CONNECTED)) {
-        if (c->flags & REDIS_FREEING) {
-            ac->onDisconnect(ac,REDIS_OK);
-        } else {
-            ac->onDisconnect(ac,(ac->err == 0) ? REDIS_OK : REDIS_ERR);
-        }
-    }
-
-    /* Cleanup self */
-    redisFree(c);
-}
-
-/* Free the async context. When this function is called from a callback,
- * control needs to be returned to redisProcessCallbacks() before actual
- * free'ing. To do so, a flag is set on the context which is picked up by
- * redisProcessCallbacks(). Otherwise, the context is immediately free'd. */
-void redisAsyncFree(redisAsyncContext *ac) {
-    redisContext *c = &(ac->c);
-    c->flags |= REDIS_FREEING;
-    if (!(c->flags & REDIS_IN_CALLBACK))
-        __redisAsyncFree(ac);
-}
-
-/* Helper function to make the disconnect happen and clean up. */
-void __redisAsyncDisconnect(redisAsyncContext *ac) {
-    redisContext *c = &(ac->c);
-
-    /* Make sure error is accessible if there is any */
-    __redisAsyncCopyError(ac);
-
-    if (ac->err == 0) {
-        /* For clean disconnects, there should be no pending callbacks. */
-        int ret = __redisShiftCallback(&ac->replies,NULL);
-        assert(ret == REDIS_ERR);
-    } else {
-        /* Disconnection is caused by an error, make sure that pending
-         * callbacks cannot call new commands. */
-        c->flags |= REDIS_DISCONNECTING;
-    }
-
-    /* cleanup event library on disconnect.
-     * this is safe to call multiple times */
-    _EL_CLEANUP(ac);
-
-    /* For non-clean disconnects, __redisAsyncFree() will execute pending
-     * callbacks with a NULL-reply. */
-    if (!(c->flags & REDIS_NO_AUTO_FREE)) {
-      __redisAsyncFree(ac);
-    }
-}
-
-/* Tries to do a clean disconnect from Redis, meaning it stops new commands
- * from being issued, but tries to flush the output buffer and execute
- * callbacks for all remaining replies. When this function is called from a
- * callback, there might be more replies and we can safely defer disconnecting
- * to redisProcessCallbacks(). Otherwise, we can only disconnect immediately
- * when there are no pending callbacks. */
-void redisAsyncDisconnect(redisAsyncContext *ac) {
-    redisContext *c = &(ac->c);
-    c->flags |= REDIS_DISCONNECTING;
-
-    /** unset the auto-free flag here, because disconnect undoes this */
-    c->flags &= ~REDIS_NO_AUTO_FREE;
-    if (!(c->flags & REDIS_IN_CALLBACK) && ac->replies.head == NULL)
-        __redisAsyncDisconnect(ac);
-}
-
-static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, redisCallback *dstcb) {
-    redisContext *c = &(ac->c);
-    dict *callbacks;
-    redisCallback *cb;
-    dictEntry *de;
-    int pvariant;
-    char *stype;
-    sds sname;
-
-    /* Custom reply functions are not supported for pub/sub. This will fail
-     * very hard when they are used... */
-    if (reply->type == REDIS_REPLY_ARRAY) {
-        assert(reply->elements >= 2);
-        assert(reply->element[0]->type == REDIS_REPLY_STRING);
-        stype = reply->element[0]->str;
-        pvariant = (tolower(stype[0]) == 'p') ? 1 : 0;
-
-        if (pvariant)
-            callbacks = ac->sub.patterns;
-        else
-            callbacks = ac->sub.channels;
-
-        /* Locate the right callback */
-        assert(reply->element[1]->type == REDIS_REPLY_STRING);
-        sname = sdsnewlen(reply->element[1]->str,reply->element[1]->len);
-        de = dictFind(callbacks,sname);
-        if (de != NULL) {
-            cb = dictGetEntryVal(de);
-
-            /* If this is an subscribe reply decrease pending counter. */
-            if (strcasecmp(stype+pvariant,"subscribe") == 0) {
-                cb->pending_subs -= 1;
-            }
-
-            memcpy(dstcb,cb,sizeof(*dstcb));
-
-            /* If this is an unsubscribe message, remove it. */
-            if (strcasecmp(stype+pvariant,"unsubscribe") == 0) {
-                if (cb->pending_subs == 0)
-                    dictDelete(callbacks,sname);
-
-                /* If this was the last unsubscribe message, revert to
-                 * non-subscribe mode. */
-                assert(reply->element[2]->type == REDIS_REPLY_INTEGER);
-
-                /* Unset subscribed flag only when no pipelined pending subscribe. */
-                if (reply->element[2]->integer == 0
-                    && dictSize(ac->sub.channels) == 0
-                    && dictSize(ac->sub.patterns) == 0)
-                    c->flags &= ~REDIS_SUBSCRIBED;
-            }
-        }
-        sdsfree(sname);
-    } else {
-        /* Shift callback for invalid commands. */
-        __redisShiftCallback(&ac->sub.invalid,dstcb);
-    }
-    return REDIS_OK;
-}
-
-void redisProcessCallbacks(redisAsyncContext *ac) {
-    redisContext *c = &(ac->c);
-    redisCallback cb = {NULL, NULL, 0, NULL};
-    void *reply = NULL;
-    int status;
-
-    while((status = redisGetReply(c,&reply)) == REDIS_OK) {
-        if (reply == NULL) {
-            /* When the connection is being disconnected and there are
-             * no more replies, this is the cue to really disconnect. */
-            if (c->flags & REDIS_DISCONNECTING && sdslen(c->obuf) == 0
-                && ac->replies.head == NULL) {
-                __redisAsyncDisconnect(ac);
-                return;
-            }
-
-            /* If monitor mode, repush callback */
-            if(c->flags & REDIS_MONITORING) {
-                __redisPushCallback(&ac->replies,&cb);
-            }
-
-            /* When the connection is not being disconnected, simply stop
-             * trying to get replies and wait for the next loop tick. */
-            break;
-        }
-
-        /* Even if the context is subscribed, pending regular callbacks will
-         * get a reply before pub/sub messages arrive. */
-        if (__redisShiftCallback(&ac->replies,&cb) != REDIS_OK) {
-            /*
-             * A spontaneous reply in a not-subscribed context can be the error
-             * reply that is sent when a new connection exceeds the maximum
-             * number of allowed connections on the server side.
-             *
-             * This is seen as an error instead of a regular reply because the
-             * server closes the connection after sending it.
-             *
-             * To prevent the error from being overwritten by an EOF error the
-             * connection is closed here. See issue #43.
-             *
-             * Another possibility is that the server is loading its dataset.
-             * In this case we also want to close the connection, and have the
-             * user wait until the server is ready to take our request.
-             */
-            if (((redisReply*)reply)->type == REDIS_REPLY_ERROR) {
-                c->err = REDIS_ERR_OTHER;
-                snprintf(c->errstr,sizeof(c->errstr),"%s",((redisReply*)reply)->str);
-                c->reader->fn->freeObject(reply);
-                __redisAsyncDisconnect(ac);
-                return;
-            }
-            /* No more regular callbacks and no errors, the context *must* be subscribed or monitoring. */
-            assert((c->flags & REDIS_SUBSCRIBED || c->flags & REDIS_MONITORING));
-            if(c->flags & REDIS_SUBSCRIBED)
-                __redisGetSubscribeCallback(ac,reply,&cb);
-        }
-
-        if (cb.fn != NULL) {
-            __redisRunCallback(ac,&cb,reply);
-            c->reader->fn->freeObject(reply);
-
-            /* Proceed with free'ing when redisAsyncFree() was called. */
-            if (c->flags & REDIS_FREEING) {
-                __redisAsyncFree(ac);
-                return;
-            }
-        } else {
-            /* No callback for this reply. This can either be a NULL callback,
-             * or there were no callbacks to begin with. Either way, don't
-             * abort with an error, but simply ignore it because the client
-             * doesn't know what the server will spit out over the wire. */
-            c->reader->fn->freeObject(reply);
-        }
-    }
-
-    /* Disconnect when there was an error reading the reply */
-    if (status != REDIS_OK)
-        __redisAsyncDisconnect(ac);
-}
-
-/* Internal helper function to detect socket status the first time a read or
- * write event fires. When connecting was not successful, the connect callback
- * is called with a REDIS_ERR status and the context is free'd. */
-static int __redisAsyncHandleConnect(redisAsyncContext *ac) {
-    int completed = 0;
-    redisContext *c = &(ac->c);
-    if (redisCheckConnectDone(c, &completed) == REDIS_ERR) {
-        /* Error! */
-        redisCheckSocketError(c);
-        if (ac->onConnect) ac->onConnect(ac, REDIS_ERR);
-        __redisAsyncDisconnect(ac);
-        return REDIS_ERR;
-    } else if (completed == 1) {
-        /* connected! */
-        if (ac->onConnect) ac->onConnect(ac, REDIS_OK);
-        c->flags |= REDIS_CONNECTED;
-        return REDIS_OK;
-    } else {
-        return REDIS_OK;
-    }
-}
-
-void redisAsyncRead(redisAsyncContext *ac) {
-    redisContext *c = &(ac->c);
-
-    if (redisBufferRead(c) == REDIS_ERR) {
-        __redisAsyncDisconnect(ac);
-    } else {
-        /* Always re-schedule reads */
-        _EL_ADD_READ(ac);
-        redisProcessCallbacks(ac);
-    }
-}
-
-/* This function should be called when the socket is readable.
- * It processes all replies that can be read and executes their callbacks.
- */
-void redisAsyncHandleRead(redisAsyncContext *ac) {
-    redisContext *c = &(ac->c);
-
-    if (!(c->flags & REDIS_CONNECTED)) {
-        /* Abort connect was not successful. */
-        if (__redisAsyncHandleConnect(ac) != REDIS_OK)
-            return;
-        /* Try again later when the context is still not connected. */
-        if (!(c->flags & REDIS_CONNECTED))
-            return;
-    }
-
-    c->funcs->async_read(ac);
-}
-
-void redisAsyncWrite(redisAsyncContext *ac) {
-    redisContext *c = &(ac->c);
-    int done = 0;
-
-    if (redisBufferWrite(c,&done) == REDIS_ERR) {
-        __redisAsyncDisconnect(ac);
-    } else {
-        /* Continue writing when not done, stop writing otherwise */
-        if (!done)
-            _EL_ADD_WRITE(ac);
-        else
-            _EL_DEL_WRITE(ac);
-
-        /* Always schedule reads after writes */
-        _EL_ADD_READ(ac);
-    }
-}
-
-void redisAsyncHandleWrite(redisAsyncContext *ac) {
-    redisContext *c = &(ac->c);
-
-    if (!(c->flags & REDIS_CONNECTED)) {
-        /* Abort connect was not successful. */
-        if (__redisAsyncHandleConnect(ac) != REDIS_OK)
-            return;
-        /* Try again later when the context is still not connected. */
-        if (!(c->flags & REDIS_CONNECTED))
-            return;
-    }
-
-    c->funcs->async_write(ac);
-}
-
-void __redisSetError(redisContext *c, int type, const char *str);
-
-void redisAsyncHandleTimeout(redisAsyncContext *ac) {
-    redisContext *c = &(ac->c);
-    redisCallback cb;
-
-    if ((c->flags & REDIS_CONNECTED) && ac->replies.head == NULL) {
-        /* Nothing to do - just an idle timeout */
-        return;
-    }
-
-    if (!c->err) {
-        __redisSetError(c, REDIS_ERR_TIMEOUT, "Timeout");
-    }
-
-    if (!(c->flags & REDIS_CONNECTED) && ac->onConnect) {
-        ac->onConnect(ac, REDIS_ERR);
-    }
-
-    while (__redisShiftCallback(&ac->replies, &cb) == REDIS_OK) {
-        __redisRunCallback(ac, &cb, NULL);
-    }
-
-    /**
-     * TODO: Don't automatically sever the connection,
-     * rather, allow to ignore <x> responses before the queue is clear
-     */
-    __redisAsyncDisconnect(ac);
-}
-
-/* Sets a pointer to the first argument and its length starting at p. Returns
- * the number of bytes to skip to get to the following argument. */
-static const char *nextArgument(const char *start, const char **str, size_t *len) {
-    const char *p = start;
-    if (p[0] != '$') {
-        p = strchr(p,'$');
-        if (p == NULL) return NULL;
-    }
-
-    *len = (int)strtol(p+1,NULL,10);
-    p = strchr(p,'\r');
-    assert(p);
-    *str = p+2;
-    return p+2+(*len)+2;
-}
-
-/* Helper function for the redisAsyncCommand* family of functions. Writes a
- * formatted command to the output buffer and registers the provided callback
- * function with the context. */
-static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) {
-    redisContext *c = &(ac->c);
-    redisCallback cb;
-    struct dict *cbdict;
-    dictEntry *de;
-    redisCallback *existcb;
-    int pvariant, hasnext;
-    const char *cstr, *astr;
-    size_t clen, alen;
-    const char *p;
-    sds sname;
-    int ret;
-
-    /* Don't accept new commands when the connection is about to be closed. */
-    if (c->flags & (REDIS_DISCONNECTING | REDIS_FREEING)) return REDIS_ERR;
-
-    /* Setup callback */
-    cb.fn = fn;
-    cb.privdata = privdata;
-    cb.pending_subs = 1;
-
-    /* Find out which command will be appended. */
-    p = nextArgument(cmd,&cstr,&clen);
-    assert(p != NULL);
-    hasnext = (p[0] == '$');
-    pvariant = (tolower(cstr[0]) == 'p') ? 1 : 0;
-    cstr += pvariant;
-    clen -= pvariant;
-
-    if (hasnext && strncasecmp(cstr,"subscribe\r\n",11) == 0) {
-        c->flags |= REDIS_SUBSCRIBED;
-
-        /* Add every channel/pattern to the list of subscription callbacks. */
-        while ((p = nextArgument(p,&astr,&alen)) != NULL) {
-            sname = sdsnewlen(astr,alen);
-            if (pvariant)
-                cbdict = ac->sub.patterns;
-            else
-                cbdict = ac->sub.channels;
-
-            de = dictFind(cbdict,sname);
-
-            if (de != NULL) {
-                existcb = dictGetEntryVal(de);
-                cb.pending_subs = existcb->pending_subs + 1;
-            }
-
-            ret = dictReplace(cbdict,sname,&cb);
-
-            if (ret == 0) sdsfree(sname);
-        }
-    } else if (strncasecmp(cstr,"unsubscribe\r\n",13) == 0) {
-        /* It is only useful to call (P)UNSUBSCRIBE when the context is
-         * subscribed to one or more channels or patterns. */
-        if (!(c->flags & REDIS_SUBSCRIBED)) return REDIS_ERR;
-
-        /* (P)UNSUBSCRIBE does not have its own response: every channel or
-         * pattern that is unsubscribed will receive a message. This means we
-         * should not append a callback function for this command. */
-     } else if(strncasecmp(cstr,"monitor\r\n",9) == 0) {
-         /* Set monitor flag and push callback */
-         c->flags |= REDIS_MONITORING;
-         __redisPushCallback(&ac->replies,&cb);
-    } else {
-        if (c->flags & REDIS_SUBSCRIBED)
-            /* This will likely result in an error reply, but it needs to be
-             * received and passed to the callback. */
-            __redisPushCallback(&ac->sub.invalid,&cb);
-        else
-            __redisPushCallback(&ac->replies,&cb);
-    }
-
-    __redisAppendCommand(c,cmd,len);
-
-    /* Always schedule a write when the write buffer is non-empty */
-    _EL_ADD_WRITE(ac);
-
-    return REDIS_OK;
-}
-
-int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap) {
-    char *cmd;
-    int len;
-    int status;
-    len = redisvFormatCommand(&cmd,format,ap);
-
-    /* We don't want to pass -1 or -2 to future functions as a length. */
-    if (len < 0)
-        return REDIS_ERR;
-
-    status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
-    free(cmd);
-    return status;
-}
-
-int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...) {
-    va_list ap;
-    int status;
-    va_start(ap,format);
-    status = redisvAsyncCommand(ac,fn,privdata,format,ap);
-    va_end(ap);
-    return status;
-}
-
-int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen) {
-    sds cmd;
-    int len;
-    int status;
-    len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen);
-    if (len < 0)
-        return REDIS_ERR;
-    status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
-    sdsfree(cmd);
-    return status;
-}
-
-int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) {
-    int status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
-    return status;
-}
-
-void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv) {
-    if (!ac->c.timeout) {
-        ac->c.timeout = calloc(1, sizeof(tv));
-    }
-
-    if (tv.tv_sec == ac->c.timeout->tv_sec &&
-        tv.tv_usec == ac->c.timeout->tv_usec) {
-        return;
-    }
-
-    *ac->c.timeout = tv;
-}
diff --git a/deps/hiredis/async.h b/deps/hiredis/async.h
deleted file mode 100644
index 4f6b3b783..000000000
--- a/deps/hiredis/async.h
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HIREDIS_ASYNC_H
-#define __HIREDIS_ASYNC_H
-#include "hiredis.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct redisAsyncContext; /* need forward declaration of redisAsyncContext */
-struct dict; /* dictionary header is included in async.c */
-
-/* Reply callback prototype and container */
-typedef void (redisCallbackFn)(struct redisAsyncContext*, void*, void*);
-typedef struct redisCallback {
-    struct redisCallback *next; /* simple singly linked list */
-    redisCallbackFn *fn;
-    int pending_subs;
-    void *privdata;
-} redisCallback;
-
-/* List of callbacks for either regular replies or pub/sub */
-typedef struct redisCallbackList {
-    redisCallback *head, *tail;
-} redisCallbackList;
-
-/* Connection callback prototypes */
-typedef void (redisDisconnectCallback)(const struct redisAsyncContext*, int status);
-typedef void (redisConnectCallback)(const struct redisAsyncContext*, int status);
-typedef void(redisTimerCallback)(void *timer, void *privdata);
-
-/* Context for an async connection to Redis */
-typedef struct redisAsyncContext {
-    /* Hold the regular context, so it can be realloc'ed. */
-    redisContext c;
-
-    /* Setup error flags so they can be used directly. */
-    int err;
-    char *errstr;
-
-    /* Not used by hiredis */
-    void *data;
-
-    /* Event library data and hooks */
-    struct {
-        void *data;
-
-        /* Hooks that are called when the library expects to start
-         * reading/writing. These functions should be idempotent. */
-        void (*addRead)(void *privdata);
-        void (*delRead)(void *privdata);
-        void (*addWrite)(void *privdata);
-        void (*delWrite)(void *privdata);
-        void (*cleanup)(void *privdata);
-        void (*scheduleTimer)(void *privdata, struct timeval tv);
-    } ev;
-
-    /* Called when either the connection is terminated due to an error or per
-     * user request. The status is set accordingly (REDIS_OK, REDIS_ERR). */
-    redisDisconnectCallback *onDisconnect;
-
-    /* Called when the first write event was received. */
-    redisConnectCallback *onConnect;
-
-    /* Regular command callbacks */
-    redisCallbackList replies;
-
-    /* Address used for connect() */
-    struct sockaddr *saddr;
-    size_t addrlen;
-
-    /* Subscription callbacks */
-    struct {
-        redisCallbackList invalid;
-        struct dict *channels;
-        struct dict *patterns;
-    } sub;
-} redisAsyncContext;
-
-/* Functions that proxy to hiredis */
-redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options);
-redisAsyncContext *redisAsyncConnect(const char *ip, int port);
-redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr);
-redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port,
-                                                  const char *source_addr);
-redisAsyncContext *redisAsyncConnectUnix(const char *path);
-int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn);
-int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn);
-
-void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv);
-void redisAsyncDisconnect(redisAsyncContext *ac);
-void redisAsyncFree(redisAsyncContext *ac);
-
-/* Handle read/write events */
-void redisAsyncHandleRead(redisAsyncContext *ac);
-void redisAsyncHandleWrite(redisAsyncContext *ac);
-void redisAsyncHandleTimeout(redisAsyncContext *ac);
-void redisAsyncRead(redisAsyncContext *ac);
-void redisAsyncWrite(redisAsyncContext *ac);
-
-/* Command functions for an async context. Write the command to the
- * output buffer and register the provided callback. */
-int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap);
-int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...);
-int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen);
-int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/deps/hiredis/async_private.h b/deps/hiredis/async_private.h
deleted file mode 100644
index d0133ae18..000000000
--- a/deps/hiredis/async_private.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HIREDIS_ASYNC_PRIVATE_H
-#define __HIREDIS_ASYNC_PRIVATE_H
-
-#define _EL_ADD_READ(ctx)                                         \
-    do {                                                          \
-        refreshTimeout(ctx);                                      \
-        if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \
-    } while (0)
-#define _EL_DEL_READ(ctx) do { \
-        if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \
-    } while(0)
-#define _EL_ADD_WRITE(ctx)                                          \
-    do {                                                            \
-        refreshTimeout(ctx);                                        \
-        if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \
-    } while (0)
-#define _EL_DEL_WRITE(ctx) do { \
-        if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \
-    } while(0)
-#define _EL_CLEANUP(ctx) do { \
-        if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \
-        ctx->ev.cleanup = NULL; \
-    } while(0);
-
-static inline void refreshTimeout(redisAsyncContext *ctx) {
-    if (ctx->c.timeout && ctx->ev.scheduleTimer &&
-        (ctx->c.timeout->tv_sec || ctx->c.timeout->tv_usec)) {
-        ctx->ev.scheduleTimer(ctx->ev.data, *ctx->c.timeout);
-    // } else {
-    //     printf("Not scheduling timer.. (tmo=%p)\n", ctx->c.timeout);
-    //     if (ctx->c.timeout){
-    //         printf("tv_sec: %u. tv_usec: %u\n", ctx->c.timeout->tv_sec,
-    //                ctx->c.timeout->tv_usec);
-    //     }
-    }
-}
-
-void __redisAsyncDisconnect(redisAsyncContext *ac);
-void redisProcessCallbacks(redisAsyncContext *ac);
-
-#endif  /* __HIREDIS_ASYNC_PRIVATE_H */
diff --git a/deps/hiredis/dict.c b/deps/hiredis/dict.c
deleted file mode 100644
index e17a62546..000000000
--- a/deps/hiredis/dict.c
+++ /dev/null
@@ -1,338 +0,0 @@
-/* Hash table implementation.
- *
- * This file implements in memory hash tables with insert/del/replace/find/
- * get-random-element operations. Hash tables will auto resize if needed
- * tables of power of two in size are used, collisions are handled by
- * chaining. See the source code for more information... :)
- *
- * Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "fmacros.h"
-#include <stdlib.h>
-#include <assert.h>
-#include <limits.h>
-#include "dict.h"
-
-/* -------------------------- private prototypes ---------------------------- */
-
-static int _dictExpandIfNeeded(dict *ht);
-static unsigned long _dictNextPower(unsigned long size);
-static int _dictKeyIndex(dict *ht, const void *key);
-static int _dictInit(dict *ht, dictType *type, void *privDataPtr);
-
-/* -------------------------- hash functions -------------------------------- */
-
-/* Generic hash function (a popular one from Bernstein).
- * I tested a few and this was the best. */
-static unsigned int dictGenHashFunction(const unsigned char *buf, int len) {
-    unsigned int hash = 5381;
-
-    while (len--)
-        hash = ((hash << 5) + hash) + (*buf++); /* hash * 33 + c */
-    return hash;
-}
-
-/* ----------------------------- API implementation ------------------------- */
-
-/* Reset an hashtable already initialized with ht_init().
- * NOTE: This function should only called by ht_destroy(). */
-static void _dictReset(dict *ht) {
-    ht->table = NULL;
-    ht->size = 0;
-    ht->sizemask = 0;
-    ht->used = 0;
-}
-
-/* Create a new hash table */
-static dict *dictCreate(dictType *type, void *privDataPtr) {
-    dict *ht = malloc(sizeof(*ht));
-    _dictInit(ht,type,privDataPtr);
-    return ht;
-}
-
-/* Initialize the hash table */
-static int _dictInit(dict *ht, dictType *type, void *privDataPtr) {
-    _dictReset(ht);
-    ht->type = type;
-    ht->privdata = privDataPtr;
-    return DICT_OK;
-}
-
-/* Expand or create the hashtable */
-static int dictExpand(dict *ht, unsigned long size) {
-    dict n; /* the new hashtable */
-    unsigned long realsize = _dictNextPower(size), i;
-
-    /* the size is invalid if it is smaller than the number of
-     * elements already inside the hashtable */
-    if (ht->used > size)
-        return DICT_ERR;
-
-    _dictInit(&n, ht->type, ht->privdata);
-    n.size = realsize;
-    n.sizemask = realsize-1;
-    n.table = calloc(realsize,sizeof(dictEntry*));
-
-    /* Copy all the elements from the old to the new table:
-     * note that if the old hash table is empty ht->size is zero,
-     * so dictExpand just creates an hash table. */
-    n.used = ht->used;
-    for (i = 0; i < ht->size && ht->used > 0; i++) {
-        dictEntry *he, *nextHe;
-
-        if (ht->table[i] == NULL) continue;
-
-        /* For each hash entry on this slot... */
-        he = ht->table[i];
-        while(he) {
-            unsigned int h;
-
-            nextHe = he->next;
-            /* Get the new element index */
-            h = dictHashKey(ht, he->key) & n.sizemask;
-            he->next = n.table[h];
-            n.table[h] = he;
-            ht->used--;
-            /* Pass to the next element */
-            he = nextHe;
-        }
-    }
-    assert(ht->used == 0);
-    free(ht->table);
-
-    /* Remap the new hashtable in the old */
-    *ht = n;
-    return DICT_OK;
-}
-
-/* Add an element to the target hash table */
-static int dictAdd(dict *ht, void *key, void *val) {
-    int index;
-    dictEntry *entry;
-
-    /* Get the index of the new element, or -1 if
-     * the element already exists. */
-    if ((index = _dictKeyIndex(ht, key)) == -1)
-        return DICT_ERR;
-
-    /* Allocates the memory and stores key */
-    entry = malloc(sizeof(*entry));
-    entry->next = ht->table[index];
-    ht->table[index] = entry;
-
-    /* Set the hash entry fields. */
-    dictSetHashKey(ht, entry, key);
-    dictSetHashVal(ht, entry, val);
-    ht->used++;
-    return DICT_OK;
-}
-
-/* Add an element, discarding the old if the key already exists.
- * Return 1 if the key was added from scratch, 0 if there was already an
- * element with such key and dictReplace() just performed a value update
- * operation. */
-static int dictReplace(dict *ht, void *key, void *val) {
-    dictEntry *entry, auxentry;
-
-    /* Try to add the element. If the key
-     * does not exists dictAdd will succeed. */
-    if (dictAdd(ht, key, val) == DICT_OK)
-        return 1;
-    /* It already exists, get the entry */
-    entry = dictFind(ht, key);
-    /* Free the old value and set the new one */
-    /* Set the new value and free the old one. Note that it is important
-     * to do that in this order, as the value may just be exactly the same
-     * as the previous one. In this context, think to reference counting,
-     * you want to increment (set), and then decrement (free), and not the
-     * reverse. */
-    auxentry = *entry;
-    dictSetHashVal(ht, entry, val);
-    dictFreeEntryVal(ht, &auxentry);
-    return 0;
-}
-
-/* Search and remove an element */
-static int dictDelete(dict *ht, const void *key) {
-    unsigned int h;
-    dictEntry *de, *prevde;
-
-    if (ht->size == 0)
-        return DICT_ERR;
-    h = dictHashKey(ht, key) & ht->sizemask;
-    de = ht->table[h];
-
-    prevde = NULL;
-    while(de) {
-        if (dictCompareHashKeys(ht,key,de->key)) {
-            /* Unlink the element from the list */
-            if (prevde)
-                prevde->next = de->next;
-            else
-                ht->table[h] = de->next;
-
-            dictFreeEntryKey(ht,de);
-            dictFreeEntryVal(ht,de);
-            free(de);
-            ht->used--;
-            return DICT_OK;
-        }
-        prevde = de;
-        de = de->next;
-    }
-    return DICT_ERR; /* not found */
-}
-
-/* Destroy an entire hash table */
-static int _dictClear(dict *ht) {
-    unsigned long i;
-
-    /* Free all the elements */
-    for (i = 0; i < ht->size && ht->used > 0; i++) {
-        dictEntry *he, *nextHe;
-
-        if ((he = ht->table[i]) == NULL) continue;
-        while(he) {
-            nextHe = he->next;
-            dictFreeEntryKey(ht, he);
-            dictFreeEntryVal(ht, he);
-            free(he);
-            ht->used--;
-            he = nextHe;
-        }
-    }
-    /* Free the table and the allocated cache structure */
-    free(ht->table);
-    /* Re-initialize the table */
-    _dictReset(ht);
-    return DICT_OK; /* never fails */
-}
-
-/* Clear & Release the hash table */
-static void dictRelease(dict *ht) {
-    _dictClear(ht);
-    free(ht);
-}
-
-static dictEntry *dictFind(dict *ht, const void *key) {
-    dictEntry *he;
-    unsigned int h;
-
-    if (ht->size == 0) return NULL;
-    h = dictHashKey(ht, key) & ht->sizemask;
-    he = ht->table[h];
-    while(he) {
-        if (dictCompareHashKeys(ht, key, he->key))
-            return he;
-        he = he->next;
-    }
-    return NULL;
-}
-
-static dictIterator *dictGetIterator(dict *ht) {
-    dictIterator *iter = malloc(sizeof(*iter));
-
-    iter->ht = ht;
-    iter->index = -1;
-    iter->entry = NULL;
-    iter->nextEntry = NULL;
-    return iter;
-}
-
-static dictEntry *dictNext(dictIterator *iter) {
-    while (1) {
-        if (iter->entry == NULL) {
-            iter->index++;
-            if (iter->index >=
-                    (signed)iter->ht->size) break;
-            iter->entry = iter->ht->table[iter->index];
-        } else {
-            iter->entry = iter->nextEntry;
-        }
-        if (iter->entry) {
-            /* We need to save the 'next' here, the iterator user
-             * may delete the entry we are returning. */
-            iter->nextEntry = iter->entry->next;
-            return iter->entry;
-        }
-    }
-    return NULL;
-}
-
-static void dictReleaseIterator(dictIterator *iter) {
-    free(iter);
-}
-
-/* ------------------------- private functions ------------------------------ */
-
-/* Expand the hash table if needed */
-static int _dictExpandIfNeeded(dict *ht) {
-    /* If the hash table is empty expand it to the initial size,
-     * if the table is "full" dobule its size. */
-    if (ht->size == 0)
-        return dictExpand(ht, DICT_HT_INITIAL_SIZE);
-    if (ht->used == ht->size)
-        return dictExpand(ht, ht->size*2);
-    return DICT_OK;
-}
-
-/* Our hash table capability is a power of two */
-static unsigned long _dictNextPower(unsigned long size) {
-    unsigned long i = DICT_HT_INITIAL_SIZE;
-
-    if (size >= LONG_MAX) return LONG_MAX;
-    while(1) {
-        if (i >= size)
-            return i;
-        i *= 2;
-    }
-}
-
-/* Returns the index of a free slot that can be populated with
- * an hash entry for the given 'key'.
- * If the key already exists, -1 is returned. */
-static int _dictKeyIndex(dict *ht, const void *key) {
-    unsigned int h;
-    dictEntry *he;
-
-    /* Expand the hashtable if needed */
-    if (_dictExpandIfNeeded(ht) == DICT_ERR)
-        return -1;
-    /* Compute the key hash value */
-    h = dictHashKey(ht, key) & ht->sizemask;
-    /* Search if this slot does not already contain the given key */
-    he = ht->table[h];
-    while(he) {
-        if (dictCompareHashKeys(ht, key, he->key))
-            return -1;
-        he = he->next;
-    }
-    return h;
-}
-
diff --git a/deps/hiredis/dict.h b/deps/hiredis/dict.h
deleted file mode 100644
index 95fcd280e..000000000
--- a/deps/hiredis/dict.h
+++ /dev/null
@@ -1,126 +0,0 @@
-/* Hash table implementation.
- *
- * This file implements in memory hash tables with insert/del/replace/find/
- * get-random-element operations. Hash tables will auto resize if needed
- * tables of power of two in size are used, collisions are handled by
- * chaining. See the source code for more information... :)
- *
- * Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __DICT_H
-#define __DICT_H
-
-#define DICT_OK 0
-#define DICT_ERR 1
-
-/* Unused arguments generate annoying warnings... */
-#define DICT_NOTUSED(V) ((void) V)
-
-typedef struct dictEntry {
-    void *key;
-    void *val;
-    struct dictEntry *next;
-} dictEntry;
-
-typedef struct dictType {
-    unsigned int (*hashFunction)(const void *key);
-    void *(*keyDup)(void *privdata, const void *key);
-    void *(*valDup)(void *privdata, const void *obj);
-    int (*keyCompare)(void *privdata, const void *key1, const void *key2);
-    void (*keyDestructor)(void *privdata, void *key);
-    void (*valDestructor)(void *privdata, void *obj);
-} dictType;
-
-typedef struct dict {
-    dictEntry **table;
-    dictType *type;
-    unsigned long size;
-    unsigned long sizemask;
-    unsigned long used;
-    void *privdata;
-} dict;
-
-typedef struct dictIterator {
-    dict *ht;
-    int index;
-    dictEntry *entry, *nextEntry;
-} dictIterator;
-
-/* This is the initial size of every hash table */
-#define DICT_HT_INITIAL_SIZE     4
-
-/* ------------------------------- Macros ------------------------------------*/
-#define dictFreeEntryVal(ht, entry) \
-    if ((ht)->type->valDestructor) \
-        (ht)->type->valDestructor((ht)->privdata, (entry)->val)
-
-#define dictSetHashVal(ht, entry, _val_) do { \
-    if ((ht)->type->valDup) \
-        entry->val = (ht)->type->valDup((ht)->privdata, _val_); \
-    else \
-        entry->val = (_val_); \
-} while(0)
-
-#define dictFreeEntryKey(ht, entry) \
-    if ((ht)->type->keyDestructor) \
-        (ht)->type->keyDestructor((ht)->privdata, (entry)->key)
-
-#define dictSetHashKey(ht, entry, _key_) do { \
-    if ((ht)->type->keyDup) \
-        entry->key = (ht)->type->keyDup((ht)->privdata, _key_); \
-    else \
-        entry->key = (_key_); \
-} while(0)
-
-#define dictCompareHashKeys(ht, key1, key2) \
-    (((ht)->type->keyCompare) ? \
-        (ht)->type->keyCompare((ht)->privdata, key1, key2) : \
-        (key1) == (key2))
-
-#define dictHashKey(ht, key) (ht)->type->hashFunction(key)
-
-#define dictGetEntryKey(he) ((he)->key)
-#define dictGetEntryVal(he) ((he)->val)
-#define dictSlots(ht) ((ht)->size)
-#define dictSize(ht) ((ht)->used)
-
-/* API */
-static unsigned int dictGenHashFunction(const unsigned char *buf, int len);
-static dict *dictCreate(dictType *type, void *privDataPtr);
-static int dictExpand(dict *ht, unsigned long size);
-static int dictAdd(dict *ht, void *key, void *val);
-static int dictReplace(dict *ht, void *key, void *val);
-static int dictDelete(dict *ht, const void *key);
-static void dictRelease(dict *ht);
-static dictEntry * dictFind(dict *ht, const void *key);
-static dictIterator *dictGetIterator(dict *ht);
-static dictEntry *dictNext(dictIterator *iter);
-static void dictReleaseIterator(dictIterator *iter);
-
-#endif /* __DICT_H */
diff --git a/deps/hiredis/examples/CMakeLists.txt b/deps/hiredis/examples/CMakeLists.txt
deleted file mode 100644
index dd3a313ac..000000000
--- a/deps/hiredis/examples/CMakeLists.txt
+++ /dev/null
@@ -1,46 +0,0 @@
-INCLUDE(FindPkgConfig)
-# Check for GLib
-
-PKG_CHECK_MODULES(GLIB2 glib-2.0)
-if (GLIB2_FOUND)
-    INCLUDE_DIRECTORIES(${GLIB2_INCLUDE_DIRS})
-    LINK_DIRECTORIES(${GLIB2_LIBRARY_DIRS})
-    ADD_EXECUTABLE(example-glib example-glib.c)
-    TARGET_LINK_LIBRARIES(example-glib hiredis ${GLIB2_LIBRARIES})
-ENDIF(GLIB2_FOUND)
-
-FIND_PATH(LIBEV ev.h
-    HINTS /usr/local /usr/opt/local
-    ENV LIBEV_INCLUDE_DIR)
-
-if (LIBEV)
-    # Just compile and link with libev
-    ADD_EXECUTABLE(example-libev example-libev.c)
-    TARGET_LINK_LIBRARIES(example-libev hiredis ev)
-ENDIF()
-
-FIND_PATH(LIBEVENT event.h)
-if (LIBEVENT)
-    ADD_EXECUTABLE(example-libevent example-libevent)
-    TARGET_LINK_LIBRARIES(example-libevent hiredis event)
-ENDIF()
-
-FIND_PATH(LIBUV uv.h)
-IF (LIBUV)
-    ADD_EXECUTABLE(example-libuv example-libuv.c)
-    TARGET_LINK_LIBRARIES(example-libuv hiredis uv)
-ENDIF()
-
-IF (APPLE)
-    FIND_LIBRARY(CF CoreFoundation)
-    ADD_EXECUTABLE(example-macosx example-macosx.c)
-    TARGET_LINK_LIBRARIES(example-macosx hiredis ${CF})
-ENDIF()
-
-IF (ENABLE_SSL)
-    ADD_EXECUTABLE(example-ssl example-ssl.c)
-    TARGET_LINK_LIBRARIES(example-ssl hiredis hiredis_ssl)
-ENDIF()
-
-ADD_EXECUTABLE(example example.c)
-TARGET_LINK_LIBRARIES(example hiredis)
diff --git a/deps/hiredis/examples/example-ae.c b/deps/hiredis/examples/example-ae.c
deleted file mode 100644
index 8efa7306a..000000000
--- a/deps/hiredis/examples/example-ae.c
+++ /dev/null
@@ -1,62 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <signal.h>
-
-#include <hiredis.h>
-#include <async.h>
-#include <adapters/ae.h>
-
-/* Put event loop in the global scope, so it can be explicitly stopped */
-static aeEventLoop *loop;
-
-void getCallback(redisAsyncContext *c, void *r, void *privdata) {
-    redisReply *reply = r;
-    if (reply == NULL) return;
-    printf("argv[%s]: %s\n", (char*)privdata, reply->str);
-
-    /* Disconnect after receiving the reply to GET */
-    redisAsyncDisconnect(c);
-}
-
-void connectCallback(const redisAsyncContext *c, int status) {
-    if (status != REDIS_OK) {
-        printf("Error: %s\n", c->errstr);
-        aeStop(loop);
-        return;
-    }
-
-    printf("Connected...\n");
-}
-
-void disconnectCallback(const redisAsyncContext *c, int status) {
-    if (status != REDIS_OK) {
-        printf("Error: %s\n", c->errstr);
-        aeStop(loop);
-        return;
-    }
-
-    printf("Disconnected...\n");
-    aeStop(loop);
-}
-
-int main (int argc, char **argv) {
-    signal(SIGPIPE, SIG_IGN);
-
-    redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
-    if (c->err) {
-        /* Let *c leak for now... */
-        printf("Error: %s\n", c->errstr);
-        return 1;
-    }
-
-    loop = aeCreateEventLoop(64);
-    redisAeAttach(loop, c);
-    redisAsyncSetConnectCallback(c,connectCallback);
-    redisAsyncSetDisconnectCallback(c,disconnectCallback);
-    redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
-    redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
-    aeMain(loop);
-    return 0;
-}
-
diff --git a/deps/hiredis/examples/example-glib.c b/deps/hiredis/examples/example-glib.c
deleted file mode 100644
index d6e10f8e8..000000000
--- a/deps/hiredis/examples/example-glib.c
+++ /dev/null
@@ -1,73 +0,0 @@
-#include <stdlib.h>
-
-#include <hiredis.h>
-#include <async.h>
-#include <adapters/glib.h>
-
-static GMainLoop *mainloop;
-
-static void
-connect_cb (const redisAsyncContext *ac G_GNUC_UNUSED,
-            int status)
-{
-    if (status != REDIS_OK) {
-        g_printerr("Failed to connect: %s\n", ac->errstr);
-        g_main_loop_quit(mainloop);
-    } else {
-        g_printerr("Connected...\n");
-    }
-}
-
-static void
-disconnect_cb (const redisAsyncContext *ac G_GNUC_UNUSED,
-               int status)
-{
-    if (status != REDIS_OK) {
-        g_error("Failed to disconnect: %s", ac->errstr);
-    } else {
-        g_printerr("Disconnected...\n");
-        g_main_loop_quit(mainloop);
-    }
-}
-
-static void
-command_cb(redisAsyncContext *ac,
-           gpointer r,
-           gpointer user_data G_GNUC_UNUSED)
-{
-    redisReply *reply = r;
-
-    if (reply) {
-        g_print("REPLY: %s\n", reply->str);
-    }
-
-    redisAsyncDisconnect(ac);
-}
-
-gint
-main (gint argc     G_GNUC_UNUSED,
-      gchar *argv[] G_GNUC_UNUSED)
-{
-    redisAsyncContext *ac;
-    GMainContext *context = NULL;
-    GSource *source;
-
-    ac = redisAsyncConnect("127.0.0.1", 6379);
-    if (ac->err) {
-        g_printerr("%s\n", ac->errstr);
-        exit(EXIT_FAILURE);
-    }
-
-    source = redis_source_new(ac);
-    mainloop = g_main_loop_new(context, FALSE);
-    g_source_attach(source, context);
-
-    redisAsyncSetConnectCallback(ac, connect_cb);
-    redisAsyncSetDisconnectCallback(ac, disconnect_cb);
-    redisAsyncCommand(ac, command_cb, NULL, "SET key 1234");
-    redisAsyncCommand(ac, command_cb, NULL, "GET key");
-
-    g_main_loop_run(mainloop);
-
-    return EXIT_SUCCESS;
-}
diff --git a/deps/hiredis/examples/example-ivykis.c b/deps/hiredis/examples/example-ivykis.c
deleted file mode 100644
index 67affcef3..000000000
--- a/deps/hiredis/examples/example-ivykis.c
+++ /dev/null
@@ -1,58 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <signal.h>
-
-#include <hiredis.h>
-#include <async.h>
-#include <adapters/ivykis.h>
-
-void getCallback(redisAsyncContext *c, void *r, void *privdata) {
-    redisReply *reply = r;
-    if (reply == NULL) return;
-    printf("argv[%s]: %s\n", (char*)privdata, reply->str);
-
-    /* Disconnect after receiving the reply to GET */
-    redisAsyncDisconnect(c);
-}
-
-void connectCallback(const redisAsyncContext *c, int status) {
-    if (status != REDIS_OK) {
-        printf("Error: %s\n", c->errstr);
-        return;
-    }
-    printf("Connected...\n");
-}
-
-void disconnectCallback(const redisAsyncContext *c, int status) {
-    if (status != REDIS_OK) {
-        printf("Error: %s\n", c->errstr);
-        return;
-    }
-    printf("Disconnected...\n");
-}
-
-int main (int argc, char **argv) {
-    signal(SIGPIPE, SIG_IGN);
-
-    iv_init();
-
-    redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
-    if (c->err) {
-        /* Let *c leak for now... */
-        printf("Error: %s\n", c->errstr);
-        return 1;
-    }
-
-    redisIvykisAttach(c);
-    redisAsyncSetConnectCallback(c,connectCallback);
-    redisAsyncSetDisconnectCallback(c,disconnectCallback);
-    redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
-    redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
-
-    iv_main();
-
-    iv_deinit();
-
-    return 0;
-}
diff --git a/deps/hiredis/examples/example-libev.c b/deps/hiredis/examples/example-libev.c
deleted file mode 100644
index cc8b166ec..000000000
--- a/deps/hiredis/examples/example-libev.c
+++ /dev/null
@@ -1,52 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <signal.h>
-
-#include <hiredis.h>
-#include <async.h>
-#include <adapters/libev.h>
-
-void getCallback(redisAsyncContext *c, void *r, void *privdata) {
-    redisReply *reply = r;
-    if (reply == NULL) return;
-    printf("argv[%s]: %s\n", (char*)privdata, reply->str);
-
-    /* Disconnect after receiving the reply to GET */
-    redisAsyncDisconnect(c);
-}
-
-void connectCallback(const redisAsyncContext *c, int status) {
-    if (status != REDIS_OK) {
-        printf("Error: %s\n", c->errstr);
-        return;
-    }
-    printf("Connected...\n");
-}
-
-void disconnectCallback(const redisAsyncContext *c, int status) {
-    if (status != REDIS_OK) {
-        printf("Error: %s\n", c->errstr);
-        return;
-    }
-    printf("Disconnected...\n");
-}
-
-int main (int argc, char **argv) {
-    signal(SIGPIPE, SIG_IGN);
-
-    redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
-    if (c->err) {
-        /* Let *c leak for now... */
-        printf("Error: %s\n", c->errstr);
-        return 1;
-    }
-
-    redisLibevAttach(EV_DEFAULT_ c);
-    redisAsyncSetConnectCallback(c,connectCallback);
-    redisAsyncSetDisconnectCallback(c,disconnectCallback);
-    redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
-    redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
-    ev_loop(EV_DEFAULT_ 0);
-    return 0;
-}
diff --git a/deps/hiredis/examples/example-libevent-ssl.c b/deps/hiredis/examples/example-libevent-ssl.c
deleted file mode 100644
index 1021113b9..000000000
--- a/deps/hiredis/examples/example-libevent-ssl.c
+++ /dev/null
@@ -1,73 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <signal.h>
-
-#include <hiredis.h>
-#include <hiredis_ssl.h>
-#include <async.h>
-#include <adapters/libevent.h>
-
-void getCallback(redisAsyncContext *c, void *r, void *privdata) {
-    redisReply *reply = r;
-    if (reply == NULL) return;
-    printf("argv[%s]: %s\n", (char*)privdata, reply->str);
-
-    /* Disconnect after receiving the reply to GET */
-    redisAsyncDisconnect(c);
-}
-
-void connectCallback(const redisAsyncContext *c, int status) {
-    if (status != REDIS_OK) {
-        printf("Error: %s\n", c->errstr);
-        return;
-    }
-    printf("Connected...\n");
-}
-
-void disconnectCallback(const redisAsyncContext *c, int status) {
-    if (status != REDIS_OK) {
-        printf("Error: %s\n", c->errstr);
-        return;
-    }
-    printf("Disconnected...\n");
-}
-
-int main (int argc, char **argv) {
-    signal(SIGPIPE, SIG_IGN);
-    struct event_base *base = event_base_new();
-    if (argc < 5) {
-        fprintf(stderr,
-                "Usage: %s <key> <host> <port> <cert> <certKey> [ca]\n", argv[0]);
-        exit(1);
-    }
-
-    const char *value = argv[1];
-    size_t nvalue = strlen(value);
-
-    const char *hostname = argv[2];
-    int port = atoi(argv[3]);
-
-    const char *cert = argv[4];
-    const char *certKey = argv[5];
-    const char *caCert = argc > 5 ? argv[6] : NULL;
-
-    redisAsyncContext *c = redisAsyncConnect(hostname, port);
-    if (c->err) {
-        /* Let *c leak for now... */
-        printf("Error: %s\n", c->errstr);
-        return 1;
-    }
-    if (redisSecureConnection(&c->c, caCert, cert, certKey, "sni") != REDIS_OK) {
-        printf("SSL Error!\n");
-        exit(1);
-    }
-
-    redisLibeventAttach(c,base);
-    redisAsyncSetConnectCallback(c,connectCallback);
-    redisAsyncSetDisconnectCallback(c,disconnectCallback);
-    redisAsyncCommand(c, NULL, NULL, "SET key %b", value, nvalue);
-    redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
-    event_base_dispatch(base);
-    return 0;
-}
diff --git a/deps/hiredis/examples/example-libevent.c b/deps/hiredis/examples/example-libevent.c
deleted file mode 100644
index 1fe71ae4e..000000000
--- a/deps/hiredis/examples/example-libevent.c
+++ /dev/null
@@ -1,64 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <signal.h>
-
-#include <hiredis.h>
-#include <async.h>
-#include <adapters/libevent.h>
-
-void getCallback(redisAsyncContext *c, void *r, void *privdata) {
-    redisReply *reply = r;
-    if (reply == NULL) {
-        if (c->errstr) {
-            printf("errstr: %s\n", c->errstr);
-        }
-        return;
-    }
-    printf("argv[%s]: %s\n", (char*)privdata, reply->str);
-
-    /* Disconnect after receiving the reply to GET */
-    redisAsyncDisconnect(c);
-}
-
-void connectCallback(const redisAsyncContext *c, int status) {
-    if (status != REDIS_OK) {
-        printf("Error: %s\n", c->errstr);
-        return;
-    }
-    printf("Connected...\n");
-}
-
-void disconnectCallback(const redisAsyncContext *c, int status) {
-    if (status != REDIS_OK) {
-        printf("Error: %s\n", c->errstr);
-        return;
-    }
-    printf("Disconnected...\n");
-}
-
-int main (int argc, char **argv) {
-    signal(SIGPIPE, SIG_IGN);
-    struct event_base *base = event_base_new();
-    redisOptions options = {0};
-    REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379);
-    struct timeval tv = {0};
-    tv.tv_sec = 1;
-    options.timeout = &tv;
-
-
-    redisAsyncContext *c = redisAsyncConnectWithOptions(&options);
-    if (c->err) {
-        /* Let *c leak for now... */
-        printf("Error: %s\n", c->errstr);
-        return 1;
-    }
-
-    redisLibeventAttach(c,base);
-    redisAsyncSetConnectCallback(c,connectCallback);
-    redisAsyncSetDisconnectCallback(c,disconnectCallback);
-    redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
-    redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
-    event_base_dispatch(base);
-    return 0;
-}
diff --git a/deps/hiredis/examples/example-libuv.c b/deps/hiredis/examples/example-libuv.c
deleted file mode 100644
index a5462d410..000000000
--- a/deps/hiredis/examples/example-libuv.c
+++ /dev/null
@@ -1,53 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <signal.h>
-
-#include <hiredis.h>
-#include <async.h>
-#include <adapters/libuv.h>
-
-void getCallback(redisAsyncContext *c, void *r, void *privdata) {
-    redisReply *reply = r;
-    if (reply == NULL) return;
-    printf("argv[%s]: %s\n", (char*)privdata, reply->str);
-
-    /* Disconnect after receiving the reply to GET */
-    redisAsyncDisconnect(c);
-}
-
-void connectCallback(const redisAsyncContext *c, int status) {
-    if (status != REDIS_OK) {
-        printf("Error: %s\n", c->errstr);
-        return;
-    }
-    printf("Connected...\n");
-}
-
-void disconnectCallback(const redisAsyncContext *c, int status) {
-    if (status != REDIS_OK) {
-        printf("Error: %s\n", c->errstr);
-        return;
-    }
-    printf("Disconnected...\n");
-}
-
-int main (int argc, char **argv) {
-    signal(SIGPIPE, SIG_IGN);
-    uv_loop_t* loop = uv_default_loop();
-
-    redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
-    if (c->err) {
-        /* Let *c leak for now... */
-        printf("Error: %s\n", c->errstr);
-        return 1;
-    }
-
-    redisLibuvAttach(c,loop);
-    redisAsyncSetConnectCallback(c,connectCallback);
-    redisAsyncSetDisconnectCallback(c,disconnectCallback);
-    redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
-    redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
-    uv_run(loop, UV_RUN_DEFAULT);
-    return 0;
-}
diff --git a/deps/hiredis/examples/example-macosx.c b/deps/hiredis/examples/example-macosx.c
deleted file mode 100644
index bc84ed5ba..000000000
--- a/deps/hiredis/examples/example-macosx.c
+++ /dev/null
@@ -1,66 +0,0 @@
-//
-//  Created by Дмитрий Бахвалов on 13.07.15.
-//  Copyright (c) 2015 Dmitry Bakhvalov. All rights reserved.
-//
-
-#include <stdio.h>
-
-#include <hiredis.h>
-#include <async.h>
-#include <adapters/macosx.h>
-
-void getCallback(redisAsyncContext *c, void *r, void *privdata) {
-    redisReply *reply = r;
-    if (reply == NULL) return;
-    printf("argv[%s]: %s\n", (char*)privdata, reply->str);
-
-    /* Disconnect after receiving the reply to GET */
-    redisAsyncDisconnect(c);
-}
-
-void connectCallback(const redisAsyncContext *c, int status) {
-    if (status != REDIS_OK) {
-        printf("Error: %s\n", c->errstr);
-        return;
-    }
-    printf("Connected...\n");
-}
-
-void disconnectCallback(const redisAsyncContext *c, int status) {
-    if (status != REDIS_OK) {
-        printf("Error: %s\n", c->errstr);
-        return;
-    }
-    CFRunLoopStop(CFRunLoopGetCurrent());
-    printf("Disconnected...\n");
-}
-
-int main (int argc, char **argv) {
-    signal(SIGPIPE, SIG_IGN);
-
-    CFRunLoopRef loop = CFRunLoopGetCurrent();
-    if( !loop ) {
-        printf("Error: Cannot get current run loop\n");
-        return 1;
-    }
-
-    redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
-    if (c->err) {
-        /* Let *c leak for now... */
-        printf("Error: %s\n", c->errstr);
-        return 1;
-    }
-
-    redisMacOSAttach(c, loop);
-
-    redisAsyncSetConnectCallback(c,connectCallback);
-    redisAsyncSetDisconnectCallback(c,disconnectCallback);
-
-    redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
-    redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
-
-    CFRunLoopRun();
-
-    return 0;
-}
-
diff --git a/deps/hiredis/examples/example-qt.cpp b/deps/hiredis/examples/example-qt.cpp
deleted file mode 100644
index f524c3f3d..000000000
--- a/deps/hiredis/examples/example-qt.cpp
+++ /dev/null
@@ -1,46 +0,0 @@
-#include <iostream>
-using namespace std;
-
-#include <QCoreApplication>
-#include <QTimer>
-
-#include "example-qt.h"
-
-void getCallback(redisAsyncContext *, void * r, void * privdata) {
-
-    redisReply * reply = static_cast<redisReply *>(r);
-    ExampleQt * ex = static_cast<ExampleQt *>(privdata);
-    if (reply == nullptr || ex == nullptr) return;
-
-    cout << "key: " << reply->str << endl;
-
-    ex->finish();
-}
-
-void ExampleQt::run() {
-
-    m_ctx = redisAsyncConnect("localhost", 6379);
-
-    if (m_ctx->err) {
-        cerr << "Error: " << m_ctx->errstr << endl;
-        redisAsyncFree(m_ctx);
-        emit finished();
-    }
-
-    m_adapter.setContext(m_ctx);
-
-    redisAsyncCommand(m_ctx, NULL, NULL, "SET key %s", m_value);
-    redisAsyncCommand(m_ctx, getCallback, this, "GET key");
-}
-
-int main (int argc, char **argv) {
-
-    QCoreApplication app(argc, argv);
-
-    ExampleQt example(argv[argc-1]);
-
-    QObject::connect(&example, SIGNAL(finished()), &app, SLOT(quit()));
-    QTimer::singleShot(0, &example, SLOT(run()));
-
-    return app.exec();
-}
diff --git a/deps/hiredis/examples/example-qt.h b/deps/hiredis/examples/example-qt.h
deleted file mode 100644
index 374f47666..000000000
--- a/deps/hiredis/examples/example-qt.h
+++ /dev/null
@@ -1,32 +0,0 @@
-#ifndef __HIREDIS_EXAMPLE_QT_H
-#define __HIREDIS_EXAMPLE_QT_H
-
-#include <adapters/qt.h>
-
-class ExampleQt : public QObject {
-
-    Q_OBJECT
-
-    public:
-        ExampleQt(const char * value, QObject * parent = 0)
-            : QObject(parent), m_value(value) {}
-
-    signals:
-        void finished();
-
-    public slots:
-        void run();
-
-    private:
-        void finish() { emit finished(); }
-
-    private:
-        const char * m_value;
-        redisAsyncContext * m_ctx;
-        RedisQtAdapter m_adapter;
-
-    friend
-    void getCallback(redisAsyncContext *, void *, void *);
-};
-
-#endif /* !__HIREDIS_EXAMPLE_QT_H */
diff --git a/deps/hiredis/examples/example-ssl.c b/deps/hiredis/examples/example-ssl.c
deleted file mode 100644
index 81f4648c6..000000000
--- a/deps/hiredis/examples/example-ssl.c
+++ /dev/null
@@ -1,97 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include <hiredis.h>
-#include <hiredis_ssl.h>
-
-int main(int argc, char **argv) {
-    unsigned int j;
-    redisContext *c;
-    redisReply *reply;
-    if (argc < 4) {
-        printf("Usage: %s <host> <port> <cert> <key> [ca]\n", argv[0]);
-        exit(1);
-    }
-    const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1";
-    int port = atoi(argv[2]);
-    const char *cert = argv[3];
-    const char *key = argv[4];
-    const char *ca = argc > 4 ? argv[5] : NULL;
-
-    struct timeval tv = { 1, 500000 }; // 1.5 seconds
-    redisOptions options = {0};
-    REDIS_OPTIONS_SET_TCP(&options, hostname, port);
-    options.timeout = &tv;
-    c = redisConnectWithOptions(&options);
-
-    if (c == NULL || c->err) {
-        if (c) {
-            printf("Connection error: %s\n", c->errstr);
-            redisFree(c);
-        } else {
-            printf("Connection error: can't allocate redis context\n");
-        }
-        exit(1);
-    }
-
-    if (redisSecureConnection(c, ca, cert, key, "sni") != REDIS_OK) {
-        printf("Couldn't initialize SSL!\n");
-        printf("Error: %s\n", c->errstr);
-        redisFree(c);
-        exit(1);
-    }
-
-    /* PING server */
-    reply = redisCommand(c,"PING");
-    printf("PING: %s\n", reply->str);
-    freeReplyObject(reply);
-
-    /* Set a key */
-    reply = redisCommand(c,"SET %s %s", "foo", "hello world");
-    printf("SET: %s\n", reply->str);
-    freeReplyObject(reply);
-
-    /* Set a key using binary safe API */
-    reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5);
-    printf("SET (binary API): %s\n", reply->str);
-    freeReplyObject(reply);
-
-    /* Try a GET and two INCR */
-    reply = redisCommand(c,"GET foo");
-    printf("GET foo: %s\n", reply->str);
-    freeReplyObject(reply);
-
-    reply = redisCommand(c,"INCR counter");
-    printf("INCR counter: %lld\n", reply->integer);
-    freeReplyObject(reply);
-    /* again ... */
-    reply = redisCommand(c,"INCR counter");
-    printf("INCR counter: %lld\n", reply->integer);
-    freeReplyObject(reply);
-
-    /* Create a list of numbers, from 0 to 9 */
-    reply = redisCommand(c,"DEL mylist");
-    freeReplyObject(reply);
-    for (j = 0; j < 10; j++) {
-        char buf[64];
-
-        snprintf(buf,64,"%u",j);
-        reply = redisCommand(c,"LPUSH mylist element-%s", buf);
-        freeReplyObject(reply);
-    }
-
-    /* Let's check what we have inside the list */
-    reply = redisCommand(c,"LRANGE mylist 0 -1");
-    if (reply->type == REDIS_REPLY_ARRAY) {
-        for (j = 0; j < reply->elements; j++) {
-            printf("%u) %s\n", j, reply->element[j]->str);
-        }
-    }
-    freeReplyObject(reply);
-
-    /* Disconnects and frees the context */
-    redisFree(c);
-
-    return 0;
-}
diff --git a/deps/hiredis/examples/example.c b/deps/hiredis/examples/example.c
deleted file mode 100644
index 0e93fc8b3..000000000
--- a/deps/hiredis/examples/example.c
+++ /dev/null
@@ -1,91 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include <hiredis.h>
-
-int main(int argc, char **argv) {
-    unsigned int j, isunix = 0;
-    redisContext *c;
-    redisReply *reply;
-    const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1";
-
-    if (argc > 2) {
-        if (*argv[2] == 'u' || *argv[2] == 'U') {
-            isunix = 1;
-            /* in this case, host is the path to the unix socket */
-            printf("Will connect to unix socket @%s\n", hostname);
-        }
-    }
-
-    int port = (argc > 2) ? atoi(argv[2]) : 6379;
-
-    struct timeval timeout = { 1, 500000 }; // 1.5 seconds
-    if (isunix) {
-        c = redisConnectUnixWithTimeout(hostname, timeout);
-    } else {
-        c = redisConnectWithTimeout(hostname, port, timeout);
-    }
-    if (c == NULL || c->err) {
-        if (c) {
-            printf("Connection error: %s\n", c->errstr);
-            redisFree(c);
-        } else {
-            printf("Connection error: can't allocate redis context\n");
-        }
-        exit(1);
-    }
-
-    /* PING server */
-    reply = redisCommand(c,"PING");
-    printf("PING: %s\n", reply->str);
-    freeReplyObject(reply);
-
-    /* Set a key */
-    reply = redisCommand(c,"SET %s %s", "foo", "hello world");
-    printf("SET: %s\n", reply->str);
-    freeReplyObject(reply);
-
-    /* Set a key using binary safe API */
-    reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5);
-    printf("SET (binary API): %s\n", reply->str);
-    freeReplyObject(reply);
-
-    /* Try a GET and two INCR */
-    reply = redisCommand(c,"GET foo");
-    printf("GET foo: %s\n", reply->str);
-    freeReplyObject(reply);
-
-    reply = redisCommand(c,"INCR counter");
-    printf("INCR counter: %lld\n", reply->integer);
-    freeReplyObject(reply);
-    /* again ... */
-    reply = redisCommand(c,"INCR counter");
-    printf("INCR counter: %lld\n", reply->integer);
-    freeReplyObject(reply);
-
-    /* Create a list of numbers, from 0 to 9 */
-    reply = redisCommand(c,"DEL mylist");
-    freeReplyObject(reply);
-    for (j = 0; j < 10; j++) {
-        char buf[64];
-
-        snprintf(buf,64,"%u",j);
-        reply = redisCommand(c,"LPUSH mylist element-%s", buf);
-        freeReplyObject(reply);
-    }
-
-    /* Let's check what we have inside the list */
-    reply = redisCommand(c,"LRANGE mylist 0 -1");
-    if (reply->type == REDIS_REPLY_ARRAY) {
-        for (j = 0; j < reply->elements; j++) {
-            printf("%u) %s\n", j, reply->element[j]->str);
-        }
-    }
-    freeReplyObject(reply);
-
-    /* Disconnects and frees the context */
-    redisFree(c);
-
-    return 0;
-}
diff --git a/deps/hiredis/fmacros.h b/deps/hiredis/fmacros.h
deleted file mode 100644
index 3227faafd..000000000
--- a/deps/hiredis/fmacros.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef __HIREDIS_FMACRO_H
-#define __HIREDIS_FMACRO_H
-
-#define _XOPEN_SOURCE 600
-#define _POSIX_C_SOURCE 200112L
-
-#if defined(__APPLE__) && defined(__MACH__)
-/* Enable TCP_KEEPALIVE */
-#define _DARWIN_C_SOURCE
-#endif
-
-#endif
diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c
deleted file mode 100644
index abd94c01d..000000000
--- a/deps/hiredis/hiredis.c
+++ /dev/null
@@ -1,1086 +0,0 @@
-/*
- * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
- *                     Jan-Erik Rediger <janerik at fnordig dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "fmacros.h"
-#include <string.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <errno.h>
-#include <ctype.h>
-
-#include "hiredis.h"
-#include "net.h"
-#include "sds.h"
-#include "async.h"
-#include "win32.h"
-
-static redisContextFuncs redisContextDefaultFuncs = {
-    .free_privdata = NULL,
-    .async_read = redisAsyncRead,
-    .async_write = redisAsyncWrite,
-    .read = redisNetRead,
-    .write = redisNetWrite
-};
-
-static redisReply *createReplyObject(int type);
-static void *createStringObject(const redisReadTask *task, char *str, size_t len);
-static void *createArrayObject(const redisReadTask *task, size_t elements);
-static void *createIntegerObject(const redisReadTask *task, long long value);
-static void *createDoubleObject(const redisReadTask *task, double value, char *str, size_t len);
-static void *createNilObject(const redisReadTask *task);
-static void *createBoolObject(const redisReadTask *task, int bval);
-
-/* Default set of functions to build the reply. Keep in mind that such a
- * function returning NULL is interpreted as OOM. */
-static redisReplyObjectFunctions defaultFunctions = {
-    createStringObject,
-    createArrayObject,
-    createIntegerObject,
-    createDoubleObject,
-    createNilObject,
-    createBoolObject,
-    freeReplyObject
-};
-
-/* Create a reply object */
-static redisReply *createReplyObject(int type) {
-    redisReply *r = calloc(1,sizeof(*r));
-
-    if (r == NULL)
-        return NULL;
-
-    r->type = type;
-    return r;
-}
-
-/* Free a reply object */
-void freeReplyObject(void *reply) {
-    redisReply *r = reply;
-    size_t j;
-
-    if (r == NULL)
-        return;
-
-    switch(r->type) {
-    case REDIS_REPLY_INTEGER:
-        break; /* Nothing to free */
-    case REDIS_REPLY_ARRAY:
-    case REDIS_REPLY_MAP:
-    case REDIS_REPLY_SET:
-        if (r->element != NULL) {
-            for (j = 0; j < r->elements; j++)
-                freeReplyObject(r->element[j]);
-            free(r->element);
-        }
-        break;
-    case REDIS_REPLY_ERROR:
-    case REDIS_REPLY_STATUS:
-    case REDIS_REPLY_STRING:
-    case REDIS_REPLY_DOUBLE:
-        free(r->str);
-        break;
-    }
-    free(r);
-}
-
-static void *createStringObject(const redisReadTask *task, char *str, size_t len) {
-    redisReply *r, *parent;
-    char *buf;
-
-    r = createReplyObject(task->type);
-    if (r == NULL)
-        return NULL;
-
-    assert(task->type == REDIS_REPLY_ERROR  ||
-           task->type == REDIS_REPLY_STATUS ||
-           task->type == REDIS_REPLY_STRING ||
-           task->type == REDIS_REPLY_VERB);
-
-    /* Copy string value */
-    if (task->type == REDIS_REPLY_VERB) {
-        buf = malloc(len-4+1); /* Skip 4 bytes of verbatim type header. */
-        if (buf == NULL) {
-            freeReplyObject(r);
-            return NULL;
-        }
-        memcpy(r->vtype,str,3);
-        r->vtype[3] = '\0';
-        memcpy(buf,str+4,len-4);
-        buf[len-4] = '\0';
-        r->len = len-4;
-    } else {
-        buf = malloc(len+1);
-        if (buf == NULL) {
-            freeReplyObject(r);
-            return NULL;
-        }
-        memcpy(buf,str,len);
-        buf[len] = '\0';
-        r->len = len;
-    }
-    r->str = buf;
-
-    if (task->parent) {
-        parent = task->parent->obj;
-        assert(parent->type == REDIS_REPLY_ARRAY ||
-               parent->type == REDIS_REPLY_MAP ||
-               parent->type == REDIS_REPLY_SET);
-        parent->element[task->idx] = r;
-    }
-    return r;
-}
-
-static void *createArrayObject(const redisReadTask *task, size_t elements) {
-    redisReply *r, *parent;
-
-    r = createReplyObject(task->type);
-    if (r == NULL)
-        return NULL;
-
-    if (elements > 0) {
-        r->element = calloc(elements,sizeof(redisReply*));
-        if (r->element == NULL) {
-            freeReplyObject(r);
-            return NULL;
-        }
-    }
-
-    r->elements = elements;
-
-    if (task->parent) {
-        parent = task->parent->obj;
-        assert(parent->type == REDIS_REPLY_ARRAY ||
-               parent->type == REDIS_REPLY_MAP ||
-               parent->type == REDIS_REPLY_SET);
-        parent->element[task->idx] = r;
-    }
-    return r;
-}
-
-static void *createIntegerObject(const redisReadTask *task, long long value) {
-    redisReply *r, *parent;
-
-    r = createReplyObject(REDIS_REPLY_INTEGER);
-    if (r == NULL)
-        return NULL;
-
-    r->integer = value;
-
-    if (task->parent) {
-        parent = task->parent->obj;
-        assert(parent->type == REDIS_REPLY_ARRAY ||
-               parent->type == REDIS_REPLY_MAP ||
-               parent->type == REDIS_REPLY_SET);
-        parent->element[task->idx] = r;
-    }
-    return r;
-}
-
-static void *createDoubleObject(const redisReadTask *task, double value, char *str, size_t len) {
-    redisReply *r, *parent;
-
-    r = createReplyObject(REDIS_REPLY_DOUBLE);
-    if (r == NULL)
-        return NULL;
-
-    r->dval = value;
-    r->str = malloc(len+1);
-    if (r->str == NULL) {
-        freeReplyObject(r);
-        return NULL;
-    }
-
-    /* The double reply also has the original protocol string representing a
-     * double as a null terminated string. This way the caller does not need
-     * to format back for string conversion, especially since Redis does efforts
-     * to make the string more human readable avoiding the calssical double
-     * decimal string conversion artifacts. */
-    memcpy(r->str, str, len);
-    r->str[len] = '\0';
-
-    if (task->parent) {
-        parent = task->parent->obj;
-        assert(parent->type == REDIS_REPLY_ARRAY ||
-               parent->type == REDIS_REPLY_MAP ||
-               parent->type == REDIS_REPLY_SET);
-        parent->element[task->idx] = r;
-    }
-    return r;
-}
-
-static void *createNilObject(const redisReadTask *task) {
-    redisReply *r, *parent;
-
-    r = createReplyObject(REDIS_REPLY_NIL);
-    if (r == NULL)
-        return NULL;
-
-    if (task->parent) {
-        parent = task->parent->obj;
-        assert(parent->type == REDIS_REPLY_ARRAY ||
-               parent->type == REDIS_REPLY_MAP ||
-               parent->type == REDIS_REPLY_SET);
-        parent->element[task->idx] = r;
-    }
-    return r;
-}
-
-static void *createBoolObject(const redisReadTask *task, int bval) {
-    redisReply *r, *parent;
-
-    r = createReplyObject(REDIS_REPLY_BOOL);
-    if (r == NULL)
-        return NULL;
-
-    r->integer = bval != 0;
-
-    if (task->parent) {
-        parent = task->parent->obj;
-        assert(parent->type == REDIS_REPLY_ARRAY ||
-               parent->type == REDIS_REPLY_MAP ||
-               parent->type == REDIS_REPLY_SET);
-        parent->element[task->idx] = r;
-    }
-    return r;
-}
-
-/* Return the number of digits of 'v' when converted to string in radix 10.
- * Implementation borrowed from link in redis/src/util.c:string2ll(). */
-static uint32_t countDigits(uint64_t v) {
-  uint32_t result = 1;
-  for (;;) {
-    if (v < 10) return result;
-    if (v < 100) return result + 1;
-    if (v < 1000) return result + 2;
-    if (v < 10000) return result + 3;
-    v /= 10000U;
-    result += 4;
-  }
-}
-
-/* Helper that calculates the bulk length given a certain string length. */
-static size_t bulklen(size_t len) {
-    return 1+countDigits(len)+2+len+2;
-}
-
-int redisvFormatCommand(char **target, const char *format, va_list ap) {
-    const char *c = format;
-    char *cmd = NULL; /* final command */
-    int pos; /* position in final command */
-    sds curarg, newarg; /* current argument */
-    int touched = 0; /* was the current argument touched? */
-    char **curargv = NULL, **newargv = NULL;
-    int argc = 0;
-    int totlen = 0;
-    int error_type = 0; /* 0 = no error; -1 = memory error; -2 = format error */
-    int j;
-
-    /* Abort if there is not target to set */
-    if (target == NULL)
-        return -1;
-
-    /* Build the command string accordingly to protocol */
-    curarg = sdsempty();
-    if (curarg == NULL)
-        return -1;
-
-    while(*c != '\0') {
-        if (*c != '%' || c[1] == '\0') {
-            if (*c == ' ') {
-                if (touched) {
-                    newargv = realloc(curargv,sizeof(char*)*(argc+1));
-                    if (newargv == NULL) goto memory_err;
-                    curargv = newargv;
-                    curargv[argc++] = curarg;
-                    totlen += bulklen(sdslen(curarg));
-
-                    /* curarg is put in argv so it can be overwritten. */
-                    curarg = sdsempty();
-                    if (curarg == NULL) goto memory_err;
-                    touched = 0;
-                }
-            } else {
-                newarg = sdscatlen(curarg,c,1);
-                if (newarg == NULL) goto memory_err;
-                curarg = newarg;
-                touched = 1;
-            }
-        } else {
-            char *arg;
-            size_t size;
-
-            /* Set newarg so it can be checked even if it is not touched. */
-            newarg = curarg;
-
-            switch(c[1]) {
-            case 's':
-                arg = va_arg(ap,char*);
-                size = strlen(arg);
-                if (size > 0)
-                    newarg = sdscatlen(curarg,arg,size);
-                break;
-            case 'b':
-                arg = va_arg(ap,char*);
-                size = va_arg(ap,size_t);
-                if (size > 0)
-                    newarg = sdscatlen(curarg,arg,size);
-                break;
-            case '%':
-                newarg = sdscat(curarg,"%");
-                break;
-            default:
-                /* Try to detect printf format */
-                {
-                    static const char intfmts[] = "diouxX";
-                    static const char flags[] = "#0-+ ";
-                    char _format[16];
-                    const char *_p = c+1;
-                    size_t _l = 0;
-                    va_list _cpy;
-
-                    /* Flags */
-                    while (*_p != '\0' && strchr(flags,*_p) != NULL) _p++;
-
-                    /* Field width */
-                    while (*_p != '\0' && isdigit(*_p)) _p++;
-
-                    /* Precision */
-                    if (*_p == '.') {
-                        _p++;
-                        while (*_p != '\0' && isdigit(*_p)) _p++;
-                    }
-
-                    /* Copy va_list before consuming with va_arg */
-                    va_copy(_cpy,ap);
-
-                    /* Integer conversion (without modifiers) */
-                    if (strchr(intfmts,*_p) != NULL) {
-                        va_arg(ap,int);
-                        goto fmt_valid;
-                    }
-
-                    /* Double conversion (without modifiers) */
-                    if (strchr("eEfFgGaA",*_p) != NULL) {
-                        va_arg(ap,double);
-                        goto fmt_valid;
-                    }
-
-                    /* Size: char */
-                    if (_p[0] == 'h' && _p[1] == 'h') {
-                        _p += 2;
-                        if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
-                            va_arg(ap,int); /* char gets promoted to int */
-                            goto fmt_valid;
-                        }
-                        goto fmt_invalid;
-                    }
-
-                    /* Size: short */
-                    if (_p[0] == 'h') {
-                        _p += 1;
-                        if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
-                            va_arg(ap,int); /* short gets promoted to int */
-                            goto fmt_valid;
-                        }
-                        goto fmt_invalid;
-                    }
-
-                    /* Size: long long */
-                    if (_p[0] == 'l' && _p[1] == 'l') {
-                        _p += 2;
-                        if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
-                            va_arg(ap,long long);
-                            goto fmt_valid;
-                        }
-                        goto fmt_invalid;
-                    }
-
-                    /* Size: long */
-                    if (_p[0] == 'l') {
-                        _p += 1;
-                        if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
-                            va_arg(ap,long);
-                            goto fmt_valid;
-                        }
-                        goto fmt_invalid;
-                    }
-
-                fmt_invalid:
-                    va_end(_cpy);
-                    goto format_err;
-
-                fmt_valid:
-                    _l = (_p+1)-c;
-                    if (_l < sizeof(_format)-2) {
-                        memcpy(_format,c,_l);
-                        _format[_l] = '\0';
-                        newarg = sdscatvprintf(curarg,_format,_cpy);
-
-                        /* Update current position (note: outer blocks
-                         * increment c twice so compensate here) */
-                        c = _p-1;
-                    }
-
-                    va_end(_cpy);
-                    break;
-                }
-            }
-
-            if (newarg == NULL) goto memory_err;
-            curarg = newarg;
-
-            touched = 1;
-            c++;
-        }
-        c++;
-    }
-
-    /* Add the last argument if needed */
-    if (touched) {
-        newargv = realloc(curargv,sizeof(char*)*(argc+1));
-        if (newargv == NULL) goto memory_err;
-        curargv = newargv;
-        curargv[argc++] = curarg;
-        totlen += bulklen(sdslen(curarg));
-    } else {
-        sdsfree(curarg);
-    }
-
-    /* Clear curarg because it was put in curargv or was free'd. */
-    curarg = NULL;
-
-    /* Add bytes needed to hold multi bulk count */
-    totlen += 1+countDigits(argc)+2;
-
-    /* Build the command at protocol level */
-    cmd = malloc(totlen+1);
-    if (cmd == NULL) goto memory_err;
-
-    pos = sprintf(cmd,"*%d\r\n",argc);
-    for (j = 0; j < argc; j++) {
-        pos += sprintf(cmd+pos,"$%zu\r\n",sdslen(curargv[j]));
-        memcpy(cmd+pos,curargv[j],sdslen(curargv[j]));
-        pos += sdslen(curargv[j]);
-        sdsfree(curargv[j]);
-        cmd[pos++] = '\r';
-        cmd[pos++] = '\n';
-    }
-    assert(pos == totlen);
-    cmd[pos] = '\0';
-
-    free(curargv);
-    *target = cmd;
-    return totlen;
-
-format_err:
-    error_type = -2;
-    goto cleanup;
-
-memory_err:
-    error_type = -1;
-    goto cleanup;
-
-cleanup:
-    if (curargv) {
-        while(argc--)
-            sdsfree(curargv[argc]);
-        free(curargv);
-    }
-
-    sdsfree(curarg);
-    free(cmd);
-
-    return error_type;
-}
-
-/* Format a command according to the Redis protocol. This function
- * takes a format similar to printf:
- *
- * %s represents a C null terminated string you want to interpolate
- * %b represents a binary safe string
- *
- * When using %b you need to provide both the pointer to the string
- * and the length in bytes as a size_t. Examples:
- *
- * len = redisFormatCommand(target, "GET %s", mykey);
- * len = redisFormatCommand(target, "SET %s %b", mykey, myval, myvallen);
- */
-int redisFormatCommand(char **target, const char *format, ...) {
-    va_list ap;
-    int len;
-    va_start(ap,format);
-    len = redisvFormatCommand(target,format,ap);
-    va_end(ap);
-
-    /* The API says "-1" means bad result, but we now also return "-2" in some
-     * cases.  Force the return value to always be -1. */
-    if (len < 0)
-        len = -1;
-
-    return len;
-}
-
-/* Format a command according to the Redis protocol using an sds string and
- * sdscatfmt for the processing of arguments. This function takes the
- * number of arguments, an array with arguments and an array with their
- * lengths. If the latter is set to NULL, strlen will be used to compute the
- * argument lengths.
- */
-int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv,
-                              const size_t *argvlen)
-{
-    sds cmd;
-    unsigned long long totlen;
-    int j;
-    size_t len;
-
-    /* Abort on a NULL target */
-    if (target == NULL)
-        return -1;
-
-    /* Calculate our total size */
-    totlen = 1+countDigits(argc)+2;
-    for (j = 0; j < argc; j++) {
-        len = argvlen ? argvlen[j] : strlen(argv[j]);
-        totlen += bulklen(len);
-    }
-
-    /* Use an SDS string for command construction */
-    cmd = sdsempty();
-    if (cmd == NULL)
-        return -1;
-
-    /* We already know how much storage we need */
-    cmd = sdsMakeRoomFor(cmd, totlen);
-    if (cmd == NULL)
-        return -1;
-
-    /* Construct command */
-    cmd = sdscatfmt(cmd, "*%i\r\n", argc);
-    for (j=0; j < argc; j++) {
-        len = argvlen ? argvlen[j] : strlen(argv[j]);
-        cmd = sdscatfmt(cmd, "$%u\r\n", len);
-        cmd = sdscatlen(cmd, argv[j], len);
-        cmd = sdscatlen(cmd, "\r\n", sizeof("\r\n")-1);
-    }
-
-    assert(sdslen(cmd)==totlen);
-
-    *target = cmd;
-    return totlen;
-}
-
-void redisFreeSdsCommand(sds cmd) {
-    sdsfree(cmd);
-}
-
-/* Format a command according to the Redis protocol. This function takes the
- * number of arguments, an array with arguments and an array with their
- * lengths. If the latter is set to NULL, strlen will be used to compute the
- * argument lengths.
- */
-int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen) {
-    char *cmd = NULL; /* final command */
-    int pos; /* position in final command */
-    size_t len;
-    int totlen, j;
-
-    /* Abort on a NULL target */
-    if (target == NULL)
-        return -1;
-
-    /* Calculate number of bytes needed for the command */
-    totlen = 1+countDigits(argc)+2;
-    for (j = 0; j < argc; j++) {
-        len = argvlen ? argvlen[j] : strlen(argv[j]);
-        totlen += bulklen(len);
-    }
-
-    /* Build the command at protocol level */
-    cmd = malloc(totlen+1);
-    if (cmd == NULL)
-        return -1;
-
-    pos = sprintf(cmd,"*%d\r\n",argc);
-    for (j = 0; j < argc; j++) {
-        len = argvlen ? argvlen[j] : strlen(argv[j]);
-        pos += sprintf(cmd+pos,"$%zu\r\n",len);
-        memcpy(cmd+pos,argv[j],len);
-        pos += len;
-        cmd[pos++] = '\r';
-        cmd[pos++] = '\n';
-    }
-    assert(pos == totlen);
-    cmd[pos] = '\0';
-
-    *target = cmd;
-    return totlen;
-}
-
-void redisFreeCommand(char *cmd) {
-    free(cmd);
-}
-
-void __redisSetError(redisContext *c, int type, const char *str) {
-    size_t len;
-
-    c->err = type;
-    if (str != NULL) {
-        len = strlen(str);
-        len = len < (sizeof(c->errstr)-1) ? len : (sizeof(c->errstr)-1);
-        memcpy(c->errstr,str,len);
-        c->errstr[len] = '\0';
-    } else {
-        /* Only REDIS_ERR_IO may lack a description! */
-        assert(type == REDIS_ERR_IO);
-        strerror_r(errno, c->errstr, sizeof(c->errstr));
-    }
-}
-
-redisReader *redisReaderCreate(void) {
-    return redisReaderCreateWithFunctions(&defaultFunctions);
-}
-
-static redisContext *redisContextInit(const redisOptions *options) {
-    redisContext *c;
-
-    c = calloc(1, sizeof(*c));
-    if (c == NULL)
-        return NULL;
-
-    c->funcs = &redisContextDefaultFuncs;
-    c->obuf = sdsempty();
-    c->reader = redisReaderCreate();
-    c->fd = REDIS_INVALID_FD;
-
-    if (c->obuf == NULL || c->reader == NULL) {
-        redisFree(c);
-        return NULL;
-    }
-    (void)options; /* options are used in other functions */
-    return c;
-}
-
-void redisFree(redisContext *c) {
-    if (c == NULL)
-        return;
-    redisNetClose(c);
-
-    sdsfree(c->obuf);
-    redisReaderFree(c->reader);
-    free(c->tcp.host);
-    free(c->tcp.source_addr);
-    free(c->unix_sock.path);
-    free(c->timeout);
-    free(c->saddr);
-    if (c->funcs->free_privdata) {
-        c->funcs->free_privdata(c->privdata);
-    }
-    memset(c, 0xff, sizeof(*c));
-    free(c);
-}
-
-redisFD redisFreeKeepFd(redisContext *c) {
-    redisFD fd = c->fd;
-    c->fd = REDIS_INVALID_FD;
-    redisFree(c);
-    return fd;
-}
-
-int redisReconnect(redisContext *c) {
-    c->err = 0;
-    memset(c->errstr, '\0', strlen(c->errstr));
-
-    if (c->privdata && c->funcs->free_privdata) {
-        c->funcs->free_privdata(c->privdata);
-        c->privdata = NULL;
-    }
-
-    redisNetClose(c);
-
-    sdsfree(c->obuf);
-    redisReaderFree(c->reader);
-
-    c->obuf = sdsempty();
-    c->reader = redisReaderCreate();
-
-    if (c->connection_type == REDIS_CONN_TCP) {
-        return redisContextConnectBindTcp(c, c->tcp.host, c->tcp.port,
-                c->timeout, c->tcp.source_addr);
-    } else if (c->connection_type == REDIS_CONN_UNIX) {
-        return redisContextConnectUnix(c, c->unix_sock.path, c->timeout);
-    } else {
-        /* Something bad happened here and shouldn't have. There isn't
-           enough information in the context to reconnect. */
-        __redisSetError(c,REDIS_ERR_OTHER,"Not enough information to reconnect");
-    }
-
-    return REDIS_ERR;
-}
-
-redisContext *redisConnectWithOptions(const redisOptions *options) {
-    redisContext *c = redisContextInit(options);
-    if (c == NULL) {
-        return NULL;
-    }
-    if (!(options->options & REDIS_OPT_NONBLOCK)) {
-        c->flags |= REDIS_BLOCK;
-    }
-    if (options->options & REDIS_OPT_REUSEADDR) {
-        c->flags |= REDIS_REUSEADDR;
-    }
-    if (options->options & REDIS_OPT_NOAUTOFREE) {
-      c->flags |= REDIS_NO_AUTO_FREE;
-    }
-
-    if (options->type == REDIS_CONN_TCP) {
-        redisContextConnectBindTcp(c, options->endpoint.tcp.ip,
-                                   options->endpoint.tcp.port, options->timeout,
-                                   options->endpoint.tcp.source_addr);
-    } else if (options->type == REDIS_CONN_UNIX) {
-        redisContextConnectUnix(c, options->endpoint.unix_socket,
-                                options->timeout);
-    } else if (options->type == REDIS_CONN_USERFD) {
-        c->fd = options->endpoint.fd;
-        c->flags |= REDIS_CONNECTED;
-    } else {
-        // Unknown type - FIXME - FREE
-        return NULL;
-    }
-    if (options->timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) {
-        redisContextSetTimeout(c, *options->timeout);
-    }
-    return c;
-}
-
-/* Connect to a Redis instance. On error the field error in the returned
- * context will be set to the return value of the error function.
- * When no set of reply functions is given, the default set will be used. */
-redisContext *redisConnect(const char *ip, int port) {
-    redisOptions options = {0};
-    REDIS_OPTIONS_SET_TCP(&options, ip, port);
-    return redisConnectWithOptions(&options);
-}
-
-redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) {
-    redisOptions options = {0};
-    REDIS_OPTIONS_SET_TCP(&options, ip, port);
-    options.timeout = &tv;
-    return redisConnectWithOptions(&options);
-}
-
-redisContext *redisConnectNonBlock(const char *ip, int port) {
-    redisOptions options = {0};
-    REDIS_OPTIONS_SET_TCP(&options, ip, port);
-    options.options |= REDIS_OPT_NONBLOCK;
-    return redisConnectWithOptions(&options);
-}
-
-redisContext *redisConnectBindNonBlock(const char *ip, int port,
-                                       const char *source_addr) {
-    redisOptions options = {0};
-    REDIS_OPTIONS_SET_TCP(&options, ip, port);
-    options.endpoint.tcp.source_addr = source_addr;
-    options.options |= REDIS_OPT_NONBLOCK;
-    return redisConnectWithOptions(&options);
-}
-
-redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port,
-                                                const char *source_addr) {
-    redisOptions options = {0};
-    REDIS_OPTIONS_SET_TCP(&options, ip, port);
-    options.endpoint.tcp.source_addr = source_addr;
-    options.options |= REDIS_OPT_NONBLOCK|REDIS_OPT_REUSEADDR;
-    return redisConnectWithOptions(&options);
-}
-
-redisContext *redisConnectUnix(const char *path) {
-    redisOptions options = {0};
-    REDIS_OPTIONS_SET_UNIX(&options, path);
-    return redisConnectWithOptions(&options);
-}
-
-redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) {
-    redisOptions options = {0};
-    REDIS_OPTIONS_SET_UNIX(&options, path);
-    options.timeout = &tv;
-    return redisConnectWithOptions(&options);
-}
-
-redisContext *redisConnectUnixNonBlock(const char *path) {
-    redisOptions options = {0};
-    REDIS_OPTIONS_SET_UNIX(&options, path);
-    options.options |= REDIS_OPT_NONBLOCK;
-    return redisConnectWithOptions(&options);
-}
-
-redisContext *redisConnectFd(redisFD fd) {
-    redisOptions options = {0};
-    options.type = REDIS_CONN_USERFD;
-    options.endpoint.fd = fd;
-    return redisConnectWithOptions(&options);
-}
-
-/* Set read/write timeout on a blocking socket. */
-int redisSetTimeout(redisContext *c, const struct timeval tv) {
-    if (c->flags & REDIS_BLOCK)
-        return redisContextSetTimeout(c,tv);
-    return REDIS_ERR;
-}
-
-/* Enable connection KeepAlive. */
-int redisEnableKeepAlive(redisContext *c) {
-    if (redisKeepAlive(c, REDIS_KEEPALIVE_INTERVAL) != REDIS_OK)
-        return REDIS_ERR;
-    return REDIS_OK;
-}
-
-/* Use this function to handle a read event on the descriptor. It will try
- * and read some bytes from the socket and feed them to the reply parser.
- *
- * After this function is called, you may use redisGetReplyFromReader to
- * see if there is a reply available. */
-int redisBufferRead(redisContext *c) {
-    char buf[1024*16];
-    int nread;
-
-    /* Return early when the context has seen an error. */
-    if (c->err)
-        return REDIS_ERR;
-
-    nread = c->funcs->read(c, buf, sizeof(buf));
-    if (nread > 0) {
-        if (redisReaderFeed(c->reader, buf, nread) != REDIS_OK) {
-            __redisSetError(c, c->reader->err, c->reader->errstr);
-            return REDIS_ERR;
-        } else {
-        }
-    } else if (nread < 0) {
-        return REDIS_ERR;
-    }
-    return REDIS_OK;
-}
-
-/* Write the output buffer to the socket.
- *
- * Returns REDIS_OK when the buffer is empty, or (a part of) the buffer was
- * successfully written to the socket. When the buffer is empty after the
- * write operation, "done" is set to 1 (if given).
- *
- * Returns REDIS_ERR if an error occurred trying to write and sets
- * c->errstr to hold the appropriate error string.
- */
-int redisBufferWrite(redisContext *c, int *done) {
-
-    /* Return early when the context has seen an error. */
-    if (c->err)
-        return REDIS_ERR;
-
-    if (sdslen(c->obuf) > 0) {
-        int nwritten = c->funcs->write(c);
-        if (nwritten < 0) {
-            return REDIS_ERR;
-        } else if (nwritten > 0) {
-            if (nwritten == (signed)sdslen(c->obuf)) {
-                sdsfree(c->obuf);
-                c->obuf = sdsempty();
-            } else {
-                sdsrange(c->obuf,nwritten,-1);
-            }
-        }
-    }
-    if (done != NULL) *done = (sdslen(c->obuf) == 0);
-    return REDIS_OK;
-}
-
-/* Internal helper function to try and get a reply from the reader,
- * or set an error in the context otherwise. */
-int redisGetReplyFromReader(redisContext *c, void **reply) {
-    if (redisReaderGetReply(c->reader,reply) == REDIS_ERR) {
-        __redisSetError(c,c->reader->err,c->reader->errstr);
-        return REDIS_ERR;
-    }
-    return REDIS_OK;
-}
-
-int redisGetReply(redisContext *c, void **reply) {
-    int wdone = 0;
-    void *aux = NULL;
-
-    /* Try to read pending replies */
-    if (redisGetReplyFromReader(c,&aux) == REDIS_ERR)
-        return REDIS_ERR;
-
-    /* For the blocking context, flush output buffer and read reply */
-    if (aux == NULL && c->flags & REDIS_BLOCK) {
-        /* Write until done */
-        do {
-            if (redisBufferWrite(c,&wdone) == REDIS_ERR)
-                return REDIS_ERR;
-        } while (!wdone);
-
-        /* Read until there is a reply */
-        do {
-            if (redisBufferRead(c) == REDIS_ERR)
-                return REDIS_ERR;
-            if (redisGetReplyFromReader(c,&aux) == REDIS_ERR)
-                return REDIS_ERR;
-        } while (aux == NULL);
-    }
-
-    /* Set reply object */
-    if (reply != NULL) *reply = aux;
-    return REDIS_OK;
-}
-
-
-/* Helper function for the redisAppendCommand* family of functions.
- *
- * Write a formatted command to the output buffer. When this family
- * is used, you need to call redisGetReply yourself to retrieve
- * the reply (or replies in pub/sub).
- */
-int __redisAppendCommand(redisContext *c, const char *cmd, size_t len) {
-    sds newbuf;
-
-    newbuf = sdscatlen(c->obuf,cmd,len);
-    if (newbuf == NULL) {
-        __redisSetError(c,REDIS_ERR_OOM,"Out of memory");
-        return REDIS_ERR;
-    }
-
-    c->obuf = newbuf;
-    return REDIS_OK;
-}
-
-int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len) {
-
-    if (__redisAppendCommand(c, cmd, len) != REDIS_OK) {
-        return REDIS_ERR;
-    }
-
-    return REDIS_OK;
-}
-
-int redisvAppendCommand(redisContext *c, const char *format, va_list ap) {
-    char *cmd;
-    int len;
-
-    len = redisvFormatCommand(&cmd,format,ap);
-    if (len == -1) {
-        __redisSetError(c,REDIS_ERR_OOM,"Out of memory");
-        return REDIS_ERR;
-    } else if (len == -2) {
-        __redisSetError(c,REDIS_ERR_OTHER,"Invalid format string");
-        return REDIS_ERR;
-    }
-
-    if (__redisAppendCommand(c,cmd,len) != REDIS_OK) {
-        free(cmd);
-        return REDIS_ERR;
-    }
-
-    free(cmd);
-    return REDIS_OK;
-}
-
-int redisAppendCommand(redisContext *c, const char *format, ...) {
-    va_list ap;
-    int ret;
-
-    va_start(ap,format);
-    ret = redisvAppendCommand(c,format,ap);
-    va_end(ap);
-    return ret;
-}
-
-int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) {
-    sds cmd;
-    int len;
-
-    len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen);
-    if (len == -1) {
-        __redisSetError(c,REDIS_ERR_OOM,"Out of memory");
-        return REDIS_ERR;
-    }
-
-    if (__redisAppendCommand(c,cmd,len) != REDIS_OK) {
-        sdsfree(cmd);
-        return REDIS_ERR;
-    }
-
-    sdsfree(cmd);
-    return REDIS_OK;
-}
-
-/* Helper function for the redisCommand* family of functions.
- *
- * Write a formatted command to the output buffer. If the given context is
- * blocking, immediately read the reply into the "reply" pointer. When the
- * context is non-blocking, the "reply" pointer will not be used and the
- * command is simply appended to the write buffer.
- *
- * Returns the reply when a reply was successfully retrieved. Returns NULL
- * otherwise. When NULL is returned in a blocking context, the error field
- * in the context will be set.
- */
-static void *__redisBlockForReply(redisContext *c) {
-    void *reply;
-
-    if (c->flags & REDIS_BLOCK) {
-        if (redisGetReply(c,&reply) != REDIS_OK)
-            return NULL;
-        return reply;
-    }
-    return NULL;
-}
-
-void *redisvCommand(redisContext *c, const char *format, va_list ap) {
-    if (redisvAppendCommand(c,format,ap) != REDIS_OK)
-        return NULL;
-    return __redisBlockForReply(c);
-}
-
-void *redisCommand(redisContext *c, const char *format, ...) {
-    va_list ap;
-    va_start(ap,format);
-    void *reply = redisvCommand(c,format,ap);
-    va_end(ap);
-    return reply;
-}
-
-void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) {
-    if (redisAppendCommandArgv(c,argc,argv,argvlen) != REDIS_OK)
-        return NULL;
-    return __redisBlockForReply(c);
-}
diff --git a/deps/hiredis/hiredis.h b/deps/hiredis/hiredis.h
deleted file mode 100644
index 69dc39c5e..000000000
--- a/deps/hiredis/hiredis.h
+++ /dev/null
@@ -1,299 +0,0 @@
-/*
- * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
- *                     Jan-Erik Rediger <janerik at fnordig dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HIREDIS_H
-#define __HIREDIS_H
-#include "read.h"
-#include <stdarg.h> /* for va_list */
-#ifndef _MSC_VER
-#include <sys/time.h> /* for struct timeval */
-#else
-struct timeval; /* forward declaration */
-#endif
-#include <stdint.h> /* uintXX_t, etc */
-#include "sds.h" /* for sds */
-
-#define HIREDIS_MAJOR 0
-#define HIREDIS_MINOR 14
-#define HIREDIS_PATCH 0
-#define HIREDIS_SONAME 0.14
-
-/* Connection type can be blocking or non-blocking and is set in the
- * least significant bit of the flags field in redisContext. */
-#define REDIS_BLOCK 0x1
-
-/* Connection may be disconnected before being free'd. The second bit
- * in the flags field is set when the context is connected. */
-#define REDIS_CONNECTED 0x2
-
-/* The async API might try to disconnect cleanly and flush the output
- * buffer and read all subsequent replies before disconnecting.
- * This flag means no new commands can come in and the connection
- * should be terminated once all replies have been read. */
-#define REDIS_DISCONNECTING 0x4
-
-/* Flag specific to the async API which means that the context should be clean
- * up as soon as possible. */
-#define REDIS_FREEING 0x8
-
-/* Flag that is set when an async callback is executed. */
-#define REDIS_IN_CALLBACK 0x10
-
-/* Flag that is set when the async context has one or more subscriptions. */
-#define REDIS_SUBSCRIBED 0x20
-
-/* Flag that is set when monitor mode is active */
-#define REDIS_MONITORING 0x40
-
-/* Flag that is set when we should set SO_REUSEADDR before calling bind() */
-#define REDIS_REUSEADDR 0x80
-
-/**
- * Flag that indicates the user does not want the context to
- * be automatically freed upon error
- */
-#define REDIS_NO_AUTO_FREE 0x200
-
-#define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */
-
-/* number of times we retry to connect in the case of EADDRNOTAVAIL and
- * SO_REUSEADDR is being used. */
-#define REDIS_CONNECT_RETRIES  10
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* This is the reply object returned by redisCommand() */
-typedef struct redisReply {
-    int type; /* REDIS_REPLY_* */
-    long long integer; /* The integer when type is REDIS_REPLY_INTEGER */
-    double dval; /* The double when type is REDIS_REPLY_DOUBLE */
-    size_t len; /* Length of string */
-    char *str; /* Used for REDIS_REPLY_ERROR, REDIS_REPLY_STRING
-                  and REDIS_REPLY_DOUBLE (in additionl to dval). */
-    char vtype[4]; /* Used for REDIS_REPLY_VERB, contains the null
-                      terminated 3 character content type, such as "txt". */
-    size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */
-    struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */
-} redisReply;
-
-redisReader *redisReaderCreate(void);
-
-/* Function to free the reply objects hiredis returns by default. */
-void freeReplyObject(void *reply);
-
-/* Functions to format a command according to the protocol. */
-int redisvFormatCommand(char **target, const char *format, va_list ap);
-int redisFormatCommand(char **target, const char *format, ...);
-int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen);
-int redisFormatSdsCommandArgv(sds *target, int argc, const char ** argv, const size_t *argvlen);
-void redisFreeCommand(char *cmd);
-void redisFreeSdsCommand(sds cmd);
-
-enum redisConnectionType {
-    REDIS_CONN_TCP,
-    REDIS_CONN_UNIX,
-    REDIS_CONN_USERFD
-};
-
-struct redisSsl;
-
-#define REDIS_OPT_NONBLOCK 0x01
-#define REDIS_OPT_REUSEADDR 0x02
-
-/**
- * Don't automatically free the async object on a connection failure,
- * or other implicit conditions. Only free on an explicit call to disconnect() or free()
- */
-#define REDIS_OPT_NOAUTOFREE 0x04
-
-/* In Unix systems a file descriptor is a regular signed int, with -1
- * representing an invalid descriptor. In Windows it is a SOCKET
- * (32- or 64-bit unsigned integer depending on the architecture), where
- * all bits set (~0) is INVALID_SOCKET.  */
-#ifndef _WIN32
-typedef int redisFD;
-#define REDIS_INVALID_FD -1
-#else
-#ifdef _WIN64
-typedef unsigned long long redisFD; /* SOCKET = 64-bit UINT_PTR */
-#else
-typedef unsigned long redisFD;      /* SOCKET = 32-bit UINT_PTR */
-#endif
-#define REDIS_INVALID_FD ((redisFD)(~0)) /* INVALID_SOCKET */
-#endif
-
-typedef struct {
-    /*
-     * the type of connection to use. This also indicates which
-     * `endpoint` member field to use
-     */
-    int type;
-    /* bit field of REDIS_OPT_xxx */
-    int options;
-    /* timeout value. if NULL, no timeout is used */
-    const struct timeval *timeout;
-    union {
-        /** use this field for tcp/ip connections */
-        struct {
-            const char *source_addr;
-            const char *ip;
-            int port;
-        } tcp;
-        /** use this field for unix domain sockets */
-        const char *unix_socket;
-        /**
-         * use this field to have hiredis operate an already-open
-         * file descriptor */
-        redisFD fd;
-    } endpoint;
-} redisOptions;
-
-/**
- * Helper macros to initialize options to their specified fields.
- */
-#define REDIS_OPTIONS_SET_TCP(opts, ip_, port_) \
-    (opts)->type = REDIS_CONN_TCP; \
-    (opts)->endpoint.tcp.ip = ip_; \
-    (opts)->endpoint.tcp.port = port_;
-
-#define REDIS_OPTIONS_SET_UNIX(opts, path) \
-    (opts)->type = REDIS_CONN_UNIX;        \
-    (opts)->endpoint.unix_socket = path;
-
-struct redisAsyncContext;
-struct redisContext;
-
-typedef struct redisContextFuncs {
-    void (*free_privdata)(void *);
-    void (*async_read)(struct redisAsyncContext *);
-    void (*async_write)(struct redisAsyncContext *);
-    int (*read)(struct redisContext *, char *, size_t);
-    int (*write)(struct redisContext *);
-} redisContextFuncs;
-
-/* Context for a connection to Redis */
-typedef struct redisContext {
-    const redisContextFuncs *funcs;   /* Function table */
-
-    int err; /* Error flags, 0 when there is no error */
-    char errstr[128]; /* String representation of error when applicable */
-    redisFD fd;
-    int flags;
-    char *obuf; /* Write buffer */
-    redisReader *reader; /* Protocol reader */
-
-    enum redisConnectionType connection_type;
-    struct timeval *timeout;
-
-    struct {
-        char *host;
-        char *source_addr;
-        int port;
-    } tcp;
-
-    struct {
-        char *path;
-    } unix_sock;
-
-    /* For non-blocking connect */
-    struct sockadr *saddr;
-    size_t addrlen;
-
-    /* Additional private data for hiredis addons such as SSL */
-    void *privdata;
-} redisContext;
-
-redisContext *redisConnectWithOptions(const redisOptions *options);
-redisContext *redisConnect(const char *ip, int port);
-redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv);
-redisContext *redisConnectNonBlock(const char *ip, int port);
-redisContext *redisConnectBindNonBlock(const char *ip, int port,
-                                       const char *source_addr);
-redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port,
-                                                const char *source_addr);
-redisContext *redisConnectUnix(const char *path);
-redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv);
-redisContext *redisConnectUnixNonBlock(const char *path);
-redisContext *redisConnectFd(redisFD fd);
-
-/**
- * Reconnect the given context using the saved information.
- *
- * This re-uses the exact same connect options as in the initial connection.
- * host, ip (or path), timeout and bind address are reused,
- * flags are used unmodified from the existing context.
- *
- * Returns REDIS_OK on successful connect or REDIS_ERR otherwise.
- */
-int redisReconnect(redisContext *c);
-
-int redisSetTimeout(redisContext *c, const struct timeval tv);
-int redisEnableKeepAlive(redisContext *c);
-void redisFree(redisContext *c);
-redisFD redisFreeKeepFd(redisContext *c);
-int redisBufferRead(redisContext *c);
-int redisBufferWrite(redisContext *c, int *done);
-
-/* In a blocking context, this function first checks if there are unconsumed
- * replies to return and returns one if so. Otherwise, it flushes the output
- * buffer to the socket and reads until it has a reply. In a non-blocking
- * context, it will return unconsumed replies until there are no more. */
-int redisGetReply(redisContext *c, void **reply);
-int redisGetReplyFromReader(redisContext *c, void **reply);
-
-/* Write a formatted command to the output buffer. Use these functions in blocking mode
- * to get a pipeline of commands. */
-int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len);
-
-/* Write a command to the output buffer. Use these functions in blocking mode
- * to get a pipeline of commands. */
-int redisvAppendCommand(redisContext *c, const char *format, va_list ap);
-int redisAppendCommand(redisContext *c, const char *format, ...);
-int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
-
-/* Issue a command to Redis. In a blocking context, it is identical to calling
- * redisAppendCommand, followed by redisGetReply. The function will return
- * NULL if there was an error in performing the request, otherwise it will
- * return the reply. In a non-blocking context, it is identical to calling
- * only redisAppendCommand and will always return NULL. */
-void *redisvCommand(redisContext *c, const char *format, va_list ap);
-void *redisCommand(redisContext *c, const char *format, ...);
-void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/deps/hiredis/hiredis.pc.in b/deps/hiredis/hiredis.pc.in
deleted file mode 100644
index 140b040f1..000000000
--- a/deps/hiredis/hiredis.pc.in
+++ /dev/null
@@ -1,11 +0,0 @@
-prefix=@CMAKE_INSTALL_PREFIX@
-exec_prefix=${prefix}
-libdir=${exec_prefix}/lib
-includedir=${prefix}/include
-pkgincludedir=${includedir}/hiredis
-
-Name: hiredis
-Description: Minimalistic C client library for Redis.
-Version: @PROJECT_VERSION@
-Libs: -L${libdir} -lhiredis
-Cflags: -I${pkgincludedir} -D_FILE_OFFSET_BITS=64
diff --git a/deps/hiredis/hiredis_ssl.h b/deps/hiredis/hiredis_ssl.h
deleted file mode 100644
index f844f9548..000000000
--- a/deps/hiredis/hiredis_ssl.h
+++ /dev/null
@@ -1,53 +0,0 @@
-
-/*
- * Copyright (c) 2019, Redis Labs
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HIREDIS_SSL_H
-#define __HIREDIS_SSL_H
-
-/* This is the underlying struct for SSL in ssl.h, which is not included to
- * keep build dependencies short here.
- */
-struct ssl_st;
-
-/**
- * Secure the connection using SSL. This should be done before any command is
- * executed on the connection.
- */
-int redisSecureConnection(redisContext *c, const char *capath, const char *certpath,
-                          const char *keypath, const char *servername);
-
-/**
- * Initiate SSL/TLS negotiation on a provided context.
- */
-
-int redisInitiateSSL(redisContext *c, struct ssl_st *ssl);
-
-#endif  /* __HIREDIS_SSL_H */
diff --git a/deps/hiredis/hiredis_ssl.pc.in b/deps/hiredis/hiredis_ssl.pc.in
deleted file mode 100644
index 588a978a5..000000000
--- a/deps/hiredis/hiredis_ssl.pc.in
+++ /dev/null
@@ -1,12 +0,0 @@
-prefix=@CMAKE_INSTALL_PREFIX@
-exec_prefix=${prefix}
-libdir=${exec_prefix}/lib
-includedir=${prefix}/include
-pkgincludedir=${includedir}/hiredis
-
-Name: hiredis_ssl
-Description: SSL Support for hiredis.
-Version: @PROJECT_VERSION@
-Requires: hiredis
-Libs: -L${libdir} -lhiredis_ssl
-Libs.private: -lssl -lcrypto
diff --git a/deps/hiredis/net.c b/deps/hiredis/net.c
deleted file mode 100644
index e5f40b0a4..000000000
--- a/deps/hiredis/net.c
+++ /dev/null
@@ -1,571 +0,0 @@
-/* Extracted from anet.c to work properly with Hiredis error reporting.
- *
- * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
- *                     Jan-Erik Rediger <janerik at fnordig dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "fmacros.h"
-#include <sys/types.h>
-#include <fcntl.h>
-#include <string.h>
-#include <errno.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <limits.h>
-#include <stdlib.h>
-
-#include "net.h"
-#include "sds.h"
-#include "sockcompat.h"
-#include "win32.h"
-
-/* Defined in hiredis.c */
-void __redisSetError(redisContext *c, int type, const char *str);
-
-void redisNetClose(redisContext *c) {
-    if (c && c->fd != REDIS_INVALID_FD) {
-        close(c->fd);
-        c->fd = REDIS_INVALID_FD;
-    }
-}
-
-int redisNetRead(redisContext *c, char *buf, size_t bufcap) {
-    int nread = recv(c->fd, buf, bufcap, 0);
-    if (nread == -1) {
-        if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) {
-            /* Try again later */
-            return 0;
-        } else if(errno == ETIMEDOUT && (c->flags & REDIS_BLOCK)) {
-            /* especially in windows */
-            __redisSetError(c, REDIS_ERR_TIMEOUT, "recv timeout");
-            return -1;
-        } else {
-            __redisSetError(c, REDIS_ERR_IO, NULL);
-            return -1;
-        }
-    } else if (nread == 0) {
-        __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection");
-        return -1;
-    } else {
-        return nread;
-    }
-}
-
-int redisNetWrite(redisContext *c) {
-    int nwritten = send(c->fd, c->obuf, sdslen(c->obuf), 0);
-    if (nwritten < 0) {
-        if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) {
-            /* Try again later */
-        } else {
-            __redisSetError(c, REDIS_ERR_IO, NULL);
-            return -1;
-        }
-    }
-    return nwritten;
-}
-
-static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) {
-    int errorno = errno;  /* snprintf() may change errno */
-    char buf[128] = { 0 };
-    size_t len = 0;
-
-    if (prefix != NULL)
-        len = snprintf(buf,sizeof(buf),"%s: ",prefix);
-    strerror_r(errorno, (char *)(buf + len), sizeof(buf) - len);
-    __redisSetError(c,type,buf);
-}
-
-static int redisSetReuseAddr(redisContext *c) {
-    int on = 1;
-    if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) {
-        __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
-        redisNetClose(c);
-        return REDIS_ERR;
-    }
-    return REDIS_OK;
-}
-
-static int redisCreateSocket(redisContext *c, int type) {
-    redisFD s;
-    if ((s = socket(type, SOCK_STREAM, 0)) == REDIS_INVALID_FD) {
-        __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
-        return REDIS_ERR;
-    }
-    c->fd = s;
-    if (type == AF_INET) {
-        if (redisSetReuseAddr(c) == REDIS_ERR) {
-            return REDIS_ERR;
-        }
-    }
-    return REDIS_OK;
-}
-
-static int redisSetBlocking(redisContext *c, int blocking) {
-#ifndef _WIN32
-    int flags;
-
-    /* Set the socket nonblocking.
-     * Note that fcntl(2) for F_GETFL and F_SETFL can't be
-     * interrupted by a signal. */
-    if ((flags = fcntl(c->fd, F_GETFL)) == -1) {
-        __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)");
-        redisNetClose(c);
-        return REDIS_ERR;
-    }
-
-    if (blocking)
-        flags &= ~O_NONBLOCK;
-    else
-        flags |= O_NONBLOCK;
-
-    if (fcntl(c->fd, F_SETFL, flags) == -1) {
-        __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)");
-        redisNetClose(c);
-        return REDIS_ERR;
-    }
-#else
-    u_long mode = blocking ? 0 : 1;
-    if (ioctl(c->fd, FIONBIO, &mode) == -1) {
-        __redisSetErrorFromErrno(c, REDIS_ERR_IO, "ioctl(FIONBIO)");
-        redisNetClose(c);
-        return REDIS_ERR;
-    }
-#endif /* _WIN32 */
-    return REDIS_OK;
-}
-
-int redisKeepAlive(redisContext *c, int interval) {
-    int val = 1;
-    redisFD fd = c->fd;
-
-    if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){
-        __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
-        return REDIS_ERR;
-    }
-
-    val = interval;
-
-#if defined(__APPLE__) && defined(__MACH__)
-    if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &val, sizeof(val)) < 0) {
-        __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
-        return REDIS_ERR;
-    }
-#else
-#if defined(__GLIBC__) && !defined(__FreeBSD_kernel__)
-    if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) {
-        __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
-        return REDIS_ERR;
-    }
-
-    val = interval/3;
-    if (val == 0) val = 1;
-    if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &val, sizeof(val)) < 0) {
-        __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
-        return REDIS_ERR;
-    }
-
-    val = 3;
-    if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &val, sizeof(val)) < 0) {
-        __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
-        return REDIS_ERR;
-    }
-#endif
-#endif
-
-    return REDIS_OK;
-}
-
-static int redisSetTcpNoDelay(redisContext *c) {
-    int yes = 1;
-    if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) {
-        __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)");
-        redisNetClose(c);
-        return REDIS_ERR;
-    }
-    return REDIS_OK;
-}
-
-#define __MAX_MSEC (((LONG_MAX) - 999) / 1000)
-
-static int redisContextTimeoutMsec(redisContext *c, long *result)
-{
-    const struct timeval *timeout = c->timeout;
-    long msec = -1;
-
-    /* Only use timeout when not NULL. */
-    if (timeout != NULL) {
-        if (timeout->tv_usec > 1000000 || timeout->tv_sec > __MAX_MSEC) {
-            *result = msec;
-            return REDIS_ERR;
-        }
-
-        msec = (timeout->tv_sec * 1000) + ((timeout->tv_usec + 999) / 1000);
-
-        if (msec < 0 || msec > INT_MAX) {
-            msec = INT_MAX;
-        }
-    }
-
-    *result = msec;
-    return REDIS_OK;
-}
-
-static int redisContextWaitReady(redisContext *c, long msec) {
-    struct pollfd   wfd[1];
-
-    wfd[0].fd     = c->fd;
-    wfd[0].events = POLLOUT;
-
-    if (errno == EINPROGRESS) {
-        int res;
-
-        if ((res = poll(wfd, 1, msec)) == -1) {
-            __redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)");
-            redisNetClose(c);
-            return REDIS_ERR;
-        } else if (res == 0) {
-            errno = ETIMEDOUT;
-            __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
-            redisNetClose(c);
-            return REDIS_ERR;
-        }
-
-        if (redisCheckConnectDone(c, &res) != REDIS_OK || res == 0) {
-            redisCheckSocketError(c);
-            return REDIS_ERR;
-        }
-
-        return REDIS_OK;
-    }
-
-    __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
-    redisNetClose(c);
-    return REDIS_ERR;
-}
-
-int redisCheckConnectDone(redisContext *c, int *completed) {
-    int rc = connect(c->fd, (const struct sockaddr *)c->saddr, c->addrlen);
-    if (rc == 0) {
-        *completed = 1;
-        return REDIS_OK;
-    }
-    switch (errno) {
-    case EISCONN:
-        *completed = 1;
-        return REDIS_OK;
-    case EALREADY:
-    case EINPROGRESS:
-    case EWOULDBLOCK:
-        *completed = 0;
-        return REDIS_OK;
-    default:
-        return REDIS_ERR;
-    }
-}
-
-int redisCheckSocketError(redisContext *c) {
-    int err = 0, errno_saved = errno;
-    socklen_t errlen = sizeof(err);
-
-    if (getsockopt(c->fd, SOL_SOCKET, SO_ERROR, &err, &errlen) == -1) {
-        __redisSetErrorFromErrno(c,REDIS_ERR_IO,"getsockopt(SO_ERROR)");
-        return REDIS_ERR;
-    }
-
-    if (err == 0) {
-        err = errno_saved;
-    }
-
-    if (err) {
-        errno = err;
-        __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
-        return REDIS_ERR;
-    }
-
-    return REDIS_OK;
-}
-
-int redisContextSetTimeout(redisContext *c, const struct timeval tv) {
-    const void *to_ptr = &tv;
-    size_t to_sz = sizeof(tv);
-#ifdef _WIN32
-    DWORD timeout_msec = tv.tv_sec * 1000 + tv.tv_usec / 1000;
-    to_ptr = &timeout_msec;
-    to_sz = sizeof(timeout_msec);
-#endif
-    if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,to_ptr,to_sz) == -1) {
-        __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)");
-        return REDIS_ERR;
-    }
-    if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,to_ptr,to_sz) == -1) {
-        __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_SNDTIMEO)");
-        return REDIS_ERR;
-    }
-    return REDIS_OK;
-}
-
-static int _redisContextConnectTcp(redisContext *c, const char *addr, int port,
-                                   const struct timeval *timeout,
-                                   const char *source_addr) {
-    redisFD s;
-    int rv, n;
-    char _port[6];  /* strlen("65535"); */
-    struct addrinfo hints, *servinfo, *bservinfo, *p, *b;
-    int blocking = (c->flags & REDIS_BLOCK);
-    int reuseaddr = (c->flags & REDIS_REUSEADDR);
-    int reuses = 0;
-    long timeout_msec = -1;
-
-    servinfo = NULL;
-    c->connection_type = REDIS_CONN_TCP;
-    c->tcp.port = port;
-
-    /* We need to take possession of the passed parameters
-     * to make them reusable for a reconnect.
-     * We also carefully check we don't free data we already own,
-     * as in the case of the reconnect method.
-     *
-     * This is a bit ugly, but atleast it works and doesn't leak memory.
-     **/
-    if (c->tcp.host != addr) {
-        free(c->tcp.host);
-
-        c->tcp.host = strdup(addr);
-    }
-
-    if (timeout) {
-        if (c->timeout != timeout) {
-            if (c->timeout == NULL)
-                c->timeout = malloc(sizeof(struct timeval));
-
-            memcpy(c->timeout, timeout, sizeof(struct timeval));
-        }
-    } else {
-        free(c->timeout);
-        c->timeout = NULL;
-    }
-
-    if (redisContextTimeoutMsec(c, &timeout_msec) != REDIS_OK) {
-        __redisSetError(c, REDIS_ERR_IO, "Invalid timeout specified");
-        goto error;
-    }
-
-    if (source_addr == NULL) {
-        free(c->tcp.source_addr);
-        c->tcp.source_addr = NULL;
-    } else if (c->tcp.source_addr != source_addr) {
-        free(c->tcp.source_addr);
-        c->tcp.source_addr = strdup(source_addr);
-    }
-
-    snprintf(_port, 6, "%d", port);
-    memset(&hints,0,sizeof(hints));
-    hints.ai_family = AF_INET;
-    hints.ai_socktype = SOCK_STREAM;
-
-    /* Try with IPv6 if no IPv4 address was found. We do it in this order since
-     * in a Redis client you can't afford to test if you have IPv6 connectivity
-     * as this would add latency to every connect. Otherwise a more sensible
-     * route could be: Use IPv6 if both addresses are available and there is IPv6
-     * connectivity. */
-    if ((rv = getaddrinfo(c->tcp.host,_port,&hints,&servinfo)) != 0) {
-         hints.ai_family = AF_INET6;
-         if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) {
-            __redisSetError(c,REDIS_ERR_OTHER,gai_strerror(rv));
-            return REDIS_ERR;
-        }
-    }
-    for (p = servinfo; p != NULL; p = p->ai_next) {
-addrretry:
-        if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == REDIS_INVALID_FD)
-            continue;
-
-        c->fd = s;
-        if (redisSetBlocking(c,0) != REDIS_OK)
-            goto error;
-        if (c->tcp.source_addr) {
-            int bound = 0;
-            /* Using getaddrinfo saves us from self-determining IPv4 vs IPv6 */
-            if ((rv = getaddrinfo(c->tcp.source_addr, NULL, &hints, &bservinfo)) != 0) {
-                char buf[128];
-                snprintf(buf,sizeof(buf),"Can't get addr: %s",gai_strerror(rv));
-                __redisSetError(c,REDIS_ERR_OTHER,buf);
-                goto error;
-            }
-
-            if (reuseaddr) {
-                n = 1;
-                if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char*) &n,
-                               sizeof(n)) < 0) {
-                    freeaddrinfo(bservinfo);
-                    goto error;
-                }
-            }
-
-            for (b = bservinfo; b != NULL; b = b->ai_next) {
-                if (bind(s,b->ai_addr,b->ai_addrlen) != -1) {
-                    bound = 1;
-                    break;
-                }
-            }
-            freeaddrinfo(bservinfo);
-            if (!bound) {
-                char buf[128];
-                snprintf(buf,sizeof(buf),"Can't bind socket: %s",strerror(errno));
-                __redisSetError(c,REDIS_ERR_OTHER,buf);
-                goto error;
-            }
-        }
-
-        /* For repeat connection */
-        free(c->saddr);
-        c->saddr = malloc(p->ai_addrlen);
-        memcpy(c->saddr, p->ai_addr, p->ai_addrlen);
-        c->addrlen = p->ai_addrlen;
-
-        if (connect(s,p->ai_addr,p->ai_addrlen) == -1) {
-            if (errno == EHOSTUNREACH) {
-                redisNetClose(c);
-                continue;
-            } else if (errno == EINPROGRESS) {
-                if (blocking) {
-                    goto wait_for_ready;
-                }
-                /* This is ok.
-                 * Note that even when it's in blocking mode, we unset blocking
-                 * for `connect()`
-                 */
-            } else if (errno == EADDRNOTAVAIL && reuseaddr) {
-                if (++reuses >= REDIS_CONNECT_RETRIES) {
-                    goto error;
-                } else {
-                    redisNetClose(c);
-                    goto addrretry;
-                }
-            } else {
-                wait_for_ready:
-                if (redisContextWaitReady(c,timeout_msec) != REDIS_OK)
-                    goto error;
-            }
-        }
-        if (blocking && redisSetBlocking(c,1) != REDIS_OK)
-            goto error;
-        if (redisSetTcpNoDelay(c) != REDIS_OK)
-            goto error;
-
-        c->flags |= REDIS_CONNECTED;
-        rv = REDIS_OK;
-        goto end;
-    }
-    if (p == NULL) {
-        char buf[128];
-        snprintf(buf,sizeof(buf),"Can't create socket: %s",strerror(errno));
-        __redisSetError(c,REDIS_ERR_OTHER,buf);
-        goto error;
-    }
-
-error:
-    rv = REDIS_ERR;
-end:
-    if(servinfo) {
-        freeaddrinfo(servinfo);
-    }
-
-    return rv;  // Need to return REDIS_OK if alright
-}
-
-int redisContextConnectTcp(redisContext *c, const char *addr, int port,
-                           const struct timeval *timeout) {
-    return _redisContextConnectTcp(c, addr, port, timeout, NULL);
-}
-
-int redisContextConnectBindTcp(redisContext *c, const char *addr, int port,
-                               const struct timeval *timeout,
-                               const char *source_addr) {
-    return _redisContextConnectTcp(c, addr, port, timeout, source_addr);
-}
-
-int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) {
-#ifndef _WIN32
-    int blocking = (c->flags & REDIS_BLOCK);
-    struct sockaddr_un *sa;
-    long timeout_msec = -1;
-
-    if (redisCreateSocket(c,AF_UNIX) < 0)
-        return REDIS_ERR;
-    if (redisSetBlocking(c,0) != REDIS_OK)
-        return REDIS_ERR;
-
-    c->connection_type = REDIS_CONN_UNIX;
-    if (c->unix_sock.path != path)
-        c->unix_sock.path = strdup(path);
-
-    if (timeout) {
-        if (c->timeout != timeout) {
-            if (c->timeout == NULL)
-                c->timeout = malloc(sizeof(struct timeval));
-
-            memcpy(c->timeout, timeout, sizeof(struct timeval));
-        }
-    } else {
-        free(c->timeout);
-        c->timeout = NULL;
-    }
-
-    if (redisContextTimeoutMsec(c,&timeout_msec) != REDIS_OK)
-        return REDIS_ERR;
-
-    sa = (struct sockaddr_un*)(c->saddr = malloc(sizeof(struct sockaddr_un)));
-    c->addrlen = sizeof(struct sockaddr_un);
-    sa->sun_family = AF_UNIX;
-    strncpy(sa->sun_path, path, sizeof(sa->sun_path) - 1);
-    if (connect(c->fd, (struct sockaddr*)sa, sizeof(*sa)) == -1) {
-        if (errno == EINPROGRESS && !blocking) {
-            /* This is ok. */
-        } else {
-            if (redisContextWaitReady(c,timeout_msec) != REDIS_OK)
-                return REDIS_ERR;
-        }
-    }
-
-    /* Reset socket to be blocking after connect(2). */
-    if (blocking && redisSetBlocking(c,1) != REDIS_OK)
-        return REDIS_ERR;
-
-    c->flags |= REDIS_CONNECTED;
-    return REDIS_OK;
-#else
-    /* We currently do not support Unix sockets for Windows. */
-    /* TODO(m): https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/ */
-    errno = EPROTONOSUPPORT;
-    return REDIS_ERR;
-#endif /* _WIN32 */
-}
diff --git a/deps/hiredis/net.h b/deps/hiredis/net.h
deleted file mode 100644
index a4393c06b..000000000
--- a/deps/hiredis/net.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/* Extracted from anet.c to work properly with Hiredis error reporting.
- *
- * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
- *                     Jan-Erik Rediger <janerik at fnordig dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __NET_H
-#define __NET_H
-
-#include "hiredis.h"
-
-void redisNetClose(redisContext *c);
-int redisNetRead(redisContext *c, char *buf, size_t bufcap);
-int redisNetWrite(redisContext *c);
-
-int redisCheckSocketError(redisContext *c);
-int redisContextSetTimeout(redisContext *c, const struct timeval tv);
-int redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout);
-int redisContextConnectBindTcp(redisContext *c, const char *addr, int port,
-                               const struct timeval *timeout,
-                               const char *source_addr);
-int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout);
-int redisKeepAlive(redisContext *c, int interval);
-int redisCheckConnectDone(redisContext *c, int *completed);
-
-#endif
diff --git a/deps/hiredis/read.c b/deps/hiredis/read.c
deleted file mode 100644
index b9853ea9a..000000000
--- a/deps/hiredis/read.c
+++ /dev/null
@@ -1,681 +0,0 @@
-/*
- * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "fmacros.h"
-#include <string.h>
-#include <stdlib.h>
-#ifndef _MSC_VER
-#include <unistd.h>
-#include <strings.h>
-#endif
-#include <assert.h>
-#include <errno.h>
-#include <ctype.h>
-#include <limits.h>
-#include <math.h>
-
-#include "read.h"
-#include "sds.h"
-#include "win32.h"
-
-static void __redisReaderSetError(redisReader *r, int type, const char *str) {
-    size_t len;
-
-    if (r->reply != NULL && r->fn && r->fn->freeObject) {
-        r->fn->freeObject(r->reply);
-        r->reply = NULL;
-    }
-
-    /* Clear input buffer on errors. */
-    sdsfree(r->buf);
-    r->buf = NULL;
-    r->pos = r->len = 0;
-
-    /* Reset task stack. */
-    r->ridx = -1;
-
-    /* Set error. */
-    r->err = type;
-    len = strlen(str);
-    len = len < (sizeof(r->errstr)-1) ? len : (sizeof(r->errstr)-1);
-    memcpy(r->errstr,str,len);
-    r->errstr[len] = '\0';
-}
-
-static size_t chrtos(char *buf, size_t size, char byte) {
-    size_t len = 0;
-
-    switch(byte) {
-    case '\\':
-    case '"':
-        len = snprintf(buf,size,"\"\\%c\"",byte);
-        break;
-    case '\n': len = snprintf(buf,size,"\"\\n\""); break;
-    case '\r': len = snprintf(buf,size,"\"\\r\""); break;
-    case '\t': len = snprintf(buf,size,"\"\\t\""); break;
-    case '\a': len = snprintf(buf,size,"\"\\a\""); break;
-    case '\b': len = snprintf(buf,size,"\"\\b\""); break;
-    default:
-        if (isprint(byte))
-            len = snprintf(buf,size,"\"%c\"",byte);
-        else
-            len = snprintf(buf,size,"\"\\x%02x\"",(unsigned char)byte);
-        break;
-    }
-
-    return len;
-}
-
-static void __redisReaderSetErrorProtocolByte(redisReader *r, char byte) {
-    char cbuf[8], sbuf[128];
-
-    chrtos(cbuf,sizeof(cbuf),byte);
-    snprintf(sbuf,sizeof(sbuf),
-        "Protocol error, got %s as reply type byte", cbuf);
-    __redisReaderSetError(r,REDIS_ERR_PROTOCOL,sbuf);
-}
-
-static void __redisReaderSetErrorOOM(redisReader *r) {
-    __redisReaderSetError(r,REDIS_ERR_OOM,"Out of memory");
-}
-
-static char *readBytes(redisReader *r, unsigned int bytes) {
-    char *p;
-    if (r->len-r->pos >= bytes) {
-        p = r->buf+r->pos;
-        r->pos += bytes;
-        return p;
-    }
-    return NULL;
-}
-
-/* Find pointer to \r\n. */
-static char *seekNewline(char *s, size_t len) {
-    int pos = 0;
-    int _len = len-1;
-
-    /* Position should be < len-1 because the character at "pos" should be
-     * followed by a \n. Note that strchr cannot be used because it doesn't
-     * allow to search a limited length and the buffer that is being searched
-     * might not have a trailing NULL character. */
-    while (pos < _len) {
-        while(pos < _len && s[pos] != '\r') pos++;
-        if (pos==_len) {
-            /* Not found. */
-            return NULL;
-        } else {
-            if (s[pos+1] == '\n') {
-                /* Found. */
-                return s+pos;
-            } else {
-                /* Continue searching. */
-                pos++;
-            }
-        }
-    }
-    return NULL;
-}
-
-/* Convert a string into a long long. Returns REDIS_OK if the string could be
- * parsed into a (non-overflowing) long long, REDIS_ERR otherwise. The value
- * will be set to the parsed value when appropriate.
- *
- * Note that this function demands that the string strictly represents
- * a long long: no spaces or other characters before or after the string
- * representing the number are accepted, nor zeroes at the start if not
- * for the string "0" representing the zero number.
- *
- * Because of its strictness, it is safe to use this function to check if
- * you can convert a string into a long long, and obtain back the string
- * from the number without any loss in the string representation. */
-static int string2ll(const char *s, size_t slen, long long *value) {
-    const char *p = s;
-    size_t plen = 0;
-    int negative = 0;
-    unsigned long long v;
-
-    if (plen == slen)
-        return REDIS_ERR;
-
-    /* Special case: first and only digit is 0. */
-    if (slen == 1 && p[0] == '0') {
-        if (value != NULL) *value = 0;
-        return REDIS_OK;
-    }
-
-    if (p[0] == '-') {
-        negative = 1;
-        p++; plen++;
-
-        /* Abort on only a negative sign. */
-        if (plen == slen)
-            return REDIS_ERR;
-    }
-
-    /* First digit should be 1-9, otherwise the string should just be 0. */
-    if (p[0] >= '1' && p[0] <= '9') {
-        v = p[0]-'0';
-        p++; plen++;
-    } else if (p[0] == '0' && slen == 1) {
-        *value = 0;
-        return REDIS_OK;
-    } else {
-        return REDIS_ERR;
-    }
-
-    while (plen < slen && p[0] >= '0' && p[0] <= '9') {
-        if (v > (ULLONG_MAX / 10)) /* Overflow. */
-            return REDIS_ERR;
-        v *= 10;
-
-        if (v > (ULLONG_MAX - (p[0]-'0'))) /* Overflow. */
-            return REDIS_ERR;
-        v += p[0]-'0';
-
-        p++; plen++;
-    }
-
-    /* Return if not all bytes were used. */
-    if (plen < slen)
-        return REDIS_ERR;
-
-    if (negative) {
-        if (v > ((unsigned long long)(-(LLONG_MIN+1))+1)) /* Overflow. */
-            return REDIS_ERR;
-        if (value != NULL) *value = -v;
-    } else {
-        if (v > LLONG_MAX) /* Overflow. */
-            return REDIS_ERR;
-        if (value != NULL) *value = v;
-    }
-    return REDIS_OK;
-}
-
-static char *readLine(redisReader *r, int *_len) {
-    char *p, *s;
-    int len;
-
-    p = r->buf+r->pos;
-    s = seekNewline(p,(r->len-r->pos));
-    if (s != NULL) {
-        len = s-(r->buf+r->pos);
-        r->pos += len+2; /* skip \r\n */
-        if (_len) *_len = len;
-        return p;
-    }
-    return NULL;
-}
-
-static void moveToNextTask(redisReader *r) {
-    redisReadTask *cur, *prv;
-    while (r->ridx >= 0) {
-        /* Return a.s.a.p. when the stack is now empty. */
-        if (r->ridx == 0) {
-            r->ridx--;
-            return;
-        }
-
-        cur = &(r->rstack[r->ridx]);
-        prv = &(r->rstack[r->ridx-1]);
-        assert(prv->type == REDIS_REPLY_ARRAY ||
-               prv->type == REDIS_REPLY_MAP ||
-               prv->type == REDIS_REPLY_SET);
-        if (cur->idx == prv->elements-1) {
-            r->ridx--;
-        } else {
-            /* Reset the type because the next item can be anything */
-            assert(cur->idx < prv->elements);
-            cur->type = -1;
-            cur->elements = -1;
-            cur->idx++;
-            return;
-        }
-    }
-}
-
-static int processLineItem(redisReader *r) {
-    redisReadTask *cur = &(r->rstack[r->ridx]);
-    void *obj;
-    char *p;
-    int len;
-
-    if ((p = readLine(r,&len)) != NULL) {
-        if (cur->type == REDIS_REPLY_INTEGER) {
-            if (r->fn && r->fn->createInteger) {
-                long long v;
-                if (string2ll(p, len, &v) == REDIS_ERR) {
-                    __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
-                            "Bad integer value");
-                    return REDIS_ERR;
-                }
-                obj = r->fn->createInteger(cur,v);
-            } else {
-                obj = (void*)REDIS_REPLY_INTEGER;
-            }
-        } else if (cur->type == REDIS_REPLY_DOUBLE) {
-            if (r->fn && r->fn->createDouble) {
-                char buf[326], *eptr;
-                double d;
-
-                if ((size_t)len >= sizeof(buf)) {
-                    __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
-                            "Double value is too large");
-                    return REDIS_ERR;
-                }
-
-                memcpy(buf,p,len);
-                buf[len] = '\0';
-
-                if (strcasecmp(buf,",inf") == 0) {
-                    d = INFINITY; /* Positive infinite. */
-                } else if (strcasecmp(buf,",-inf") == 0) {
-                    d = -INFINITY; /* Nevative infinite. */
-                } else {
-                    d = strtod((char*)buf,&eptr);
-                    if (buf[0] == '\0' || eptr[0] != '\0' || isnan(d)) {
-                        __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
-                                "Bad double value");
-                        return REDIS_ERR;
-                    }
-                }
-                obj = r->fn->createDouble(cur,d,buf,len);
-            } else {
-                obj = (void*)REDIS_REPLY_DOUBLE;
-            }
-        } else if (cur->type == REDIS_REPLY_NIL) {
-            if (r->fn && r->fn->createNil)
-                obj = r->fn->createNil(cur);
-            else
-                obj = (void*)REDIS_REPLY_NIL;
-        } else if (cur->type == REDIS_REPLY_BOOL) {
-            int bval = p[0] == 't' || p[0] == 'T';
-            if (r->fn && r->fn->createBool)
-                obj = r->fn->createBool(cur,bval);
-            else
-                obj = (void*)REDIS_REPLY_BOOL;
-        } else {
-            /* Type will be error or status. */
-            if (r->fn && r->fn->createString)
-                obj = r->fn->createString(cur,p,len);
-            else
-                obj = (void*)(size_t)(cur->type);
-        }
-
-        if (obj == NULL) {
-            __redisReaderSetErrorOOM(r);
-            return REDIS_ERR;
-        }
-
-        /* Set reply if this is the root object. */
-        if (r->ridx == 0) r->reply = obj;
-        moveToNextTask(r);
-        return REDIS_OK;
-    }
-
-    return REDIS_ERR;
-}
-
-static int processBulkItem(redisReader *r) {
-    redisReadTask *cur = &(r->rstack[r->ridx]);
-    void *obj = NULL;
-    char *p, *s;
-    long long len;
-    unsigned long bytelen;
-    int success = 0;
-
-    p = r->buf+r->pos;
-    s = seekNewline(p,r->len-r->pos);
-    if (s != NULL) {
-        p = r->buf+r->pos;
-        bytelen = s-(r->buf+r->pos)+2; /* include \r\n */
-
-        if (string2ll(p, bytelen - 2, &len) == REDIS_ERR) {
-            __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
-                    "Bad bulk string length");
-            return REDIS_ERR;
-        }
-
-        if (len < -1 || (LLONG_MAX > SIZE_MAX && len > (long long)SIZE_MAX)) {
-            __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
-                    "Bulk string length out of range");
-            return REDIS_ERR;
-        }
-
-        if (len == -1) {
-            /* The nil object can always be created. */
-            if (r->fn && r->fn->createNil)
-                obj = r->fn->createNil(cur);
-            else
-                obj = (void*)REDIS_REPLY_NIL;
-            success = 1;
-        } else {
-            /* Only continue when the buffer contains the entire bulk item. */
-            bytelen += len+2; /* include \r\n */
-            if (r->pos+bytelen <= r->len) {
-                if ((cur->type == REDIS_REPLY_VERB && len < 4) ||
-                    (cur->type == REDIS_REPLY_VERB && s[5] != ':'))
-                {
-                    __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
-                            "Verbatim string 4 bytes of content type are "
-                            "missing or incorrectly encoded.");
-                    return REDIS_ERR;
-                }
-                if (r->fn && r->fn->createString)
-                    obj = r->fn->createString(cur,s+2,len);
-                else
-                    obj = (void*)(long)cur->type;
-                success = 1;
-            }
-        }
-
-        /* Proceed when obj was created. */
-        if (success) {
-            if (obj == NULL) {
-                __redisReaderSetErrorOOM(r);
-                return REDIS_ERR;
-            }
-
-            r->pos += bytelen;
-
-            /* Set reply if this is the root object. */
-            if (r->ridx == 0) r->reply = obj;
-            moveToNextTask(r);
-            return REDIS_OK;
-        }
-    }
-
-    return REDIS_ERR;
-}
-
-/* Process the array, map and set types. */
-static int processAggregateItem(redisReader *r) {
-    redisReadTask *cur = &(r->rstack[r->ridx]);
-    void *obj;
-    char *p;
-    long long elements;
-    int root = 0, len;
-
-    /* Set error for nested multi bulks with depth > 7 */
-    if (r->ridx == 8) {
-        __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
-            "No support for nested multi bulk replies with depth > 7");
-        return REDIS_ERR;
-    }
-
-    if ((p = readLine(r,&len)) != NULL) {
-        if (string2ll(p, len, &elements) == REDIS_ERR) {
-            __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
-                    "Bad multi-bulk length");
-            return REDIS_ERR;
-        }
-
-        root = (r->ridx == 0);
-
-        if (elements < -1 || (LLONG_MAX > SIZE_MAX && elements > SIZE_MAX)) {
-            __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
-                    "Multi-bulk length out of range");
-            return REDIS_ERR;
-        }
-
-        if (elements == -1) {
-            if (r->fn && r->fn->createNil)
-                obj = r->fn->createNil(cur);
-            else
-                obj = (void*)REDIS_REPLY_NIL;
-
-            if (obj == NULL) {
-                __redisReaderSetErrorOOM(r);
-                return REDIS_ERR;
-            }
-
-            moveToNextTask(r);
-        } else {
-            if (cur->type == REDIS_REPLY_MAP) elements *= 2;
-
-            if (r->fn && r->fn->createArray)
-                obj = r->fn->createArray(cur,elements);
-            else
-                obj = (void*)(long)cur->type;
-
-            if (obj == NULL) {
-                __redisReaderSetErrorOOM(r);
-                return REDIS_ERR;
-            }
-
-            /* Modify task stack when there are more than 0 elements. */
-            if (elements > 0) {
-                cur->elements = elements;
-                cur->obj = obj;
-                r->ridx++;
-                r->rstack[r->ridx].type = -1;
-                r->rstack[r->ridx].elements = -1;
-                r->rstack[r->ridx].idx = 0;
-                r->rstack[r->ridx].obj = NULL;
-                r->rstack[r->ridx].parent = cur;
-                r->rstack[r->ridx].privdata = r->privdata;
-            } else {
-                moveToNextTask(r);
-            }
-        }
-
-        /* Set reply if this is the root object. */
-        if (root) r->reply = obj;
-        return REDIS_OK;
-    }
-
-    return REDIS_ERR;
-}
-
-static int processItem(redisReader *r) {
-    redisReadTask *cur = &(r->rstack[r->ridx]);
-    char *p;
-
-    /* check if we need to read type */
-    if (cur->type < 0) {
-        if ((p = readBytes(r,1)) != NULL) {
-            switch (p[0]) {
-            case '-':
-                cur->type = REDIS_REPLY_ERROR;
-                break;
-            case '+':
-                cur->type = REDIS_REPLY_STATUS;
-                break;
-            case ':':
-                cur->type = REDIS_REPLY_INTEGER;
-                break;
-            case ',':
-                cur->type = REDIS_REPLY_DOUBLE;
-                break;
-            case '_':
-                cur->type = REDIS_REPLY_NIL;
-                break;
-            case '$':
-                cur->type = REDIS_REPLY_STRING;
-                break;
-            case '*':
-                cur->type = REDIS_REPLY_ARRAY;
-                break;
-            case '%':
-                cur->type = REDIS_REPLY_MAP;
-                break;
-            case '~':
-                cur->type = REDIS_REPLY_SET;
-                break;
-            case '#':
-                cur->type = REDIS_REPLY_BOOL;
-                break;
-            case '=':
-                cur->type = REDIS_REPLY_VERB;
-                break;
-            default:
-                __redisReaderSetErrorProtocolByte(r,*p);
-                return REDIS_ERR;
-            }
-        } else {
-            /* could not consume 1 byte */
-            return REDIS_ERR;
-        }
-    }
-
-    /* process typed item */
-    switch(cur->type) {
-    case REDIS_REPLY_ERROR:
-    case REDIS_REPLY_STATUS:
-    case REDIS_REPLY_INTEGER:
-    case REDIS_REPLY_DOUBLE:
-    case REDIS_REPLY_NIL:
-    case REDIS_REPLY_BOOL:
-        return processLineItem(r);
-    case REDIS_REPLY_STRING:
-    case REDIS_REPLY_VERB:
-        return processBulkItem(r);
-    case REDIS_REPLY_ARRAY:
-    case REDIS_REPLY_MAP:
-    case REDIS_REPLY_SET:
-        return processAggregateItem(r);
-    default:
-        assert(NULL);
-        return REDIS_ERR; /* Avoid warning. */
-    }
-}
-
-redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) {
-    redisReader *r;
-
-    r = calloc(1,sizeof(redisReader));
-    if (r == NULL)
-        return NULL;
-
-    r->fn = fn;
-    r->buf = sdsempty();
-    r->maxbuf = REDIS_READER_MAX_BUF;
-    if (r->buf == NULL) {
-        free(r);
-        return NULL;
-    }
-
-    r->ridx = -1;
-    return r;
-}
-
-void redisReaderFree(redisReader *r) {
-    if (r == NULL)
-        return;
-    if (r->reply != NULL && r->fn && r->fn->freeObject)
-        r->fn->freeObject(r->reply);
-    sdsfree(r->buf);
-    free(r);
-}
-
-int redisReaderFeed(redisReader *r, const char *buf, size_t len) {
-    sds newbuf;
-
-    /* Return early when this reader is in an erroneous state. */
-    if (r->err)
-        return REDIS_ERR;
-
-    /* Copy the provided buffer. */
-    if (buf != NULL && len >= 1) {
-        /* Destroy internal buffer when it is empty and is quite large. */
-        if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) {
-            sdsfree(r->buf);
-            r->buf = sdsempty();
-            r->pos = 0;
-
-            /* r->buf should not be NULL since we just free'd a larger one. */
-            assert(r->buf != NULL);
-        }
-
-        newbuf = sdscatlen(r->buf,buf,len);
-        if (newbuf == NULL) {
-            __redisReaderSetErrorOOM(r);
-            return REDIS_ERR;
-        }
-
-        r->buf = newbuf;
-        r->len = sdslen(r->buf);
-    }
-
-    return REDIS_OK;
-}
-
-int redisReaderGetReply(redisReader *r, void **reply) {
-    /* Default target pointer to NULL. */
-    if (reply != NULL)
-        *reply = NULL;
-
-    /* Return early when this reader is in an erroneous state. */
-    if (r->err)
-        return REDIS_ERR;
-
-    /* When the buffer is empty, there will never be a reply. */
-    if (r->len == 0)
-        return REDIS_OK;
-
-    /* Set first item to process when the stack is empty. */
-    if (r->ridx == -1) {
-        r->rstack[0].type = -1;
-        r->rstack[0].elements = -1;
-        r->rstack[0].idx = -1;
-        r->rstack[0].obj = NULL;
-        r->rstack[0].parent = NULL;
-        r->rstack[0].privdata = r->privdata;
-        r->ridx = 0;
-    }
-
-    /* Process items in reply. */
-    while (r->ridx >= 0)
-        if (processItem(r) != REDIS_OK)
-            break;
-
-    /* Return ASAP when an error occurred. */
-    if (r->err)
-        return REDIS_ERR;
-
-    /* Discard part of the buffer when we've consumed at least 1k, to avoid
-     * doing unnecessary calls to memmove() in sds.c. */
-    if (r->pos >= 1024) {
-        sdsrange(r->buf,r->pos,-1);
-        r->pos = 0;
-        r->len = sdslen(r->buf);
-    }
-
-    /* Emit a reply when there is one. */
-    if (r->ridx == -1) {
-        if (reply != NULL) {
-            *reply = r->reply;
-        } else if (r->reply != NULL && r->fn && r->fn->freeObject) {
-            r->fn->freeObject(r->reply);
-        }
-        r->reply = NULL;
-    }
-    return REDIS_OK;
-}
diff --git a/deps/hiredis/read.h b/deps/hiredis/read.h
deleted file mode 100644
index 58105312a..000000000
--- a/deps/hiredis/read.h
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#ifndef __HIREDIS_READ_H
-#define __HIREDIS_READ_H
-#include <stdio.h> /* for size_t */
-
-#define REDIS_ERR -1
-#define REDIS_OK 0
-
-/* When an error occurs, the err flag in a context is set to hold the type of
- * error that occurred. REDIS_ERR_IO means there was an I/O error and you
- * should use the "errno" variable to find out what is wrong.
- * For other values, the "errstr" field will hold a description. */
-#define REDIS_ERR_IO 1 /* Error in read or write */
-#define REDIS_ERR_EOF 3 /* End of file */
-#define REDIS_ERR_PROTOCOL 4 /* Protocol error */
-#define REDIS_ERR_OOM 5 /* Out of memory */
-#define REDIS_ERR_TIMEOUT 6 /* Timed out */
-#define REDIS_ERR_OTHER 2 /* Everything else... */
-
-#define REDIS_REPLY_STRING 1
-#define REDIS_REPLY_ARRAY 2
-#define REDIS_REPLY_INTEGER 3
-#define REDIS_REPLY_NIL 4
-#define REDIS_REPLY_STATUS 5
-#define REDIS_REPLY_ERROR 6
-#define REDIS_REPLY_DOUBLE 7
-#define REDIS_REPLY_BOOL 8
-#define REDIS_REPLY_MAP 9
-#define REDIS_REPLY_SET 10
-#define REDIS_REPLY_ATTR 11
-#define REDIS_REPLY_PUSH 12
-#define REDIS_REPLY_BIGNUM 13
-#define REDIS_REPLY_VERB 14
-
-#define REDIS_READER_MAX_BUF (1024*16)  /* Default max unused reader buffer. */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct redisReadTask {
-    int type;
-    int elements; /* number of elements in multibulk container */
-    int idx; /* index in parent (array) object */
-    void *obj; /* holds user-generated value for a read task */
-    struct redisReadTask *parent; /* parent task */
-    void *privdata; /* user-settable arbitrary field */
-} redisReadTask;
-
-typedef struct redisReplyObjectFunctions {
-    void *(*createString)(const redisReadTask*, char*, size_t);
-    void *(*createArray)(const redisReadTask*, size_t);
-    void *(*createInteger)(const redisReadTask*, long long);
-    void *(*createDouble)(const redisReadTask*, double, char*, size_t);
-    void *(*createNil)(const redisReadTask*);
-    void *(*createBool)(const redisReadTask*, int);
-    void (*freeObject)(void*);
-} redisReplyObjectFunctions;
-
-typedef struct redisReader {
-    int err; /* Error flags, 0 when there is no error */
-    char errstr[128]; /* String representation of error when applicable */
-
-    char *buf; /* Read buffer */
-    size_t pos; /* Buffer cursor */
-    size_t len; /* Buffer length */
-    size_t maxbuf; /* Max length of unused buffer */
-
-    redisReadTask rstack[9];
-    int ridx; /* Index of current read task */
-    void *reply; /* Temporary reply pointer */
-
-    redisReplyObjectFunctions *fn;
-    void *privdata;
-} redisReader;
-
-/* Public API for the protocol parser. */
-redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn);
-void redisReaderFree(redisReader *r);
-int redisReaderFeed(redisReader *r, const char *buf, size_t len);
-int redisReaderGetReply(redisReader *r, void **reply);
-
-#define redisReaderSetPrivdata(_r, _p) (int)(((redisReader*)(_r))->privdata = (_p))
-#define redisReaderGetObject(_r) (((redisReader*)(_r))->reply)
-#define redisReaderGetError(_r) (((redisReader*)(_r))->errstr)
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/deps/hiredis/sds.c b/deps/hiredis/sds.c
deleted file mode 100644
index 6cf75841c..000000000
--- a/deps/hiredis/sds.c
+++ /dev/null
@@ -1,1291 +0,0 @@
-/* SDSLib 2.0 -- A C dynamic strings library
- *
- * Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2015, Oran Agra
- * Copyright (c) 2015, Redis Labs, Inc
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <ctype.h>
-#include <assert.h>
-#include "sds.h"
-#include "sdsalloc.h"
-
-static inline int sdsHdrSize(char type) {
-    switch(type&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
-            return sizeof(struct sdshdr5);
-        case SDS_TYPE_8:
-            return sizeof(struct sdshdr8);
-        case SDS_TYPE_16:
-            return sizeof(struct sdshdr16);
-        case SDS_TYPE_32:
-            return sizeof(struct sdshdr32);
-        case SDS_TYPE_64:
-            return sizeof(struct sdshdr64);
-    }
-    return 0;
-}
-
-static inline char sdsReqType(size_t string_size) {
-    if (string_size < 32)
-        return SDS_TYPE_5;
-    if (string_size < 0xff)
-        return SDS_TYPE_8;
-    if (string_size < 0xffff)
-        return SDS_TYPE_16;
-    if (string_size < 0xffffffff)
-        return SDS_TYPE_32;
-    return SDS_TYPE_64;
-}
-
-/* Create a new sds string with the content specified by the 'init' pointer
- * and 'initlen'.
- * If NULL is used for 'init' the string is initialized with zero bytes.
- *
- * The string is always null-termined (all the sds strings are, always) so
- * even if you create an sds string with:
- *
- * mystring = sdsnewlen("abc",3);
- *
- * You can print the string with printf() as there is an implicit \0 at the
- * end of the string. However the string is binary safe and can contain
- * \0 characters in the middle, as the length is stored in the sds header. */
-sds sdsnewlen(const void *init, size_t initlen) {
-    void *sh;
-    sds s;
-    char type = sdsReqType(initlen);
-    /* Empty strings are usually created in order to append. Use type 8
-     * since type 5 is not good at this. */
-    if (type == SDS_TYPE_5 && initlen == 0) type = SDS_TYPE_8;
-    int hdrlen = sdsHdrSize(type);
-    unsigned char *fp; /* flags pointer. */
-
-    sh = s_malloc(hdrlen+initlen+1);
-    if (sh == NULL) return NULL;
-    if (!init)
-        memset(sh, 0, hdrlen+initlen+1);
-    s = (char*)sh+hdrlen;
-    fp = ((unsigned char*)s)-1;
-    switch(type) {
-        case SDS_TYPE_5: {
-            *fp = type | (initlen << SDS_TYPE_BITS);
-            break;
-        }
-        case SDS_TYPE_8: {
-            SDS_HDR_VAR(8,s);
-            sh->len = initlen;
-            sh->alloc = initlen;
-            *fp = type;
-            break;
-        }
-        case SDS_TYPE_16: {
-            SDS_HDR_VAR(16,s);
-            sh->len = initlen;
-            sh->alloc = initlen;
-            *fp = type;
-            break;
-        }
-        case SDS_TYPE_32: {
-            SDS_HDR_VAR(32,s);
-            sh->len = initlen;
-            sh->alloc = initlen;
-            *fp = type;
-            break;
-        }
-        case SDS_TYPE_64: {
-            SDS_HDR_VAR(64,s);
-            sh->len = initlen;
-            sh->alloc = initlen;
-            *fp = type;
-            break;
-        }
-    }
-    if (initlen && init)
-        memcpy(s, init, initlen);
-    s[initlen] = '\0';
-    return s;
-}
-
-/* Create an empty (zero length) sds string. Even in this case the string
- * always has an implicit null term. */
-sds sdsempty(void) {
-    return sdsnewlen("",0);
-}
-
-/* Create a new sds string starting from a null terminated C string. */
-sds sdsnew(const char *init) {
-    size_t initlen = (init == NULL) ? 0 : strlen(init);
-    return sdsnewlen(init, initlen);
-}
-
-/* Duplicate an sds string. */
-sds sdsdup(const sds s) {
-    return sdsnewlen(s, sdslen(s));
-}
-
-/* Free an sds string. No operation is performed if 's' is NULL. */
-void sdsfree(sds s) {
-    if (s == NULL) return;
-    s_free((char*)s-sdsHdrSize(s[-1]));
-}
-
-/* Set the sds string length to the length as obtained with strlen(), so
- * considering as content only up to the first null term character.
- *
- * This function is useful when the sds string is hacked manually in some
- * way, like in the following example:
- *
- * s = sdsnew("foobar");
- * s[2] = '\0';
- * sdsupdatelen(s);
- * printf("%d\n", sdslen(s));
- *
- * The output will be "2", but if we comment out the call to sdsupdatelen()
- * the output will be "6" as the string was modified but the logical length
- * remains 6 bytes. */
-void sdsupdatelen(sds s) {
-    int reallen = strlen(s);
-    sdssetlen(s, reallen);
-}
-
-/* Modify an sds string in-place to make it empty (zero length).
- * However all the existing buffer is not discarded but set as free space
- * so that next append operations will not require allocations up to the
- * number of bytes previously available. */
-void sdsclear(sds s) {
-    sdssetlen(s, 0);
-    s[0] = '\0';
-}
-
-/* Enlarge the free space at the end of the sds string so that the caller
- * is sure that after calling this function can overwrite up to addlen
- * bytes after the end of the string, plus one more byte for nul term.
- *
- * Note: this does not change the *length* of the sds string as returned
- * by sdslen(), but only the free buffer space we have. */
-sds sdsMakeRoomFor(sds s, size_t addlen) {
-    void *sh, *newsh;
-    size_t avail = sdsavail(s);
-    size_t len, newlen;
-    char type, oldtype = s[-1] & SDS_TYPE_MASK;
-    int hdrlen;
-
-    /* Return ASAP if there is enough space left. */
-    if (avail >= addlen) return s;
-
-    len = sdslen(s);
-    sh = (char*)s-sdsHdrSize(oldtype);
-    newlen = (len+addlen);
-    if (newlen < SDS_MAX_PREALLOC)
-        newlen *= 2;
-    else
-        newlen += SDS_MAX_PREALLOC;
-
-    type = sdsReqType(newlen);
-
-    /* Don't use type 5: the user is appending to the string and type 5 is
-     * not able to remember empty space, so sdsMakeRoomFor() must be called
-     * at every appending operation. */
-    if (type == SDS_TYPE_5) type = SDS_TYPE_8;
-
-    hdrlen = sdsHdrSize(type);
-    if (oldtype==type) {
-        newsh = s_realloc(sh, hdrlen+newlen+1);
-        if (newsh == NULL) {
-            s_free(sh);
-            return NULL;
-        }
-        s = (char*)newsh+hdrlen;
-    } else {
-        /* Since the header size changes, need to move the string forward,
-         * and can't use realloc */
-        newsh = s_malloc(hdrlen+newlen+1);
-        if (newsh == NULL) return NULL;
-        memcpy((char*)newsh+hdrlen, s, len+1);
-        s_free(sh);
-        s = (char*)newsh+hdrlen;
-        s[-1] = type;
-        sdssetlen(s, len);
-    }
-    sdssetalloc(s, newlen);
-    return s;
-}
-
-/* Reallocate the sds string so that it has no free space at the end. The
- * contained string remains not altered, but next concatenation operations
- * will require a reallocation.
- *
- * After the call, the passed sds string is no longer valid and all the
- * references must be substituted with the new pointer returned by the call. */
-sds sdsRemoveFreeSpace(sds s) {
-    void *sh, *newsh;
-    char type, oldtype = s[-1] & SDS_TYPE_MASK;
-    int hdrlen;
-    size_t len = sdslen(s);
-    sh = (char*)s-sdsHdrSize(oldtype);
-
-    type = sdsReqType(len);
-    hdrlen = sdsHdrSize(type);
-    if (oldtype==type) {
-        newsh = s_realloc(sh, hdrlen+len+1);
-        if (newsh == NULL) return NULL;
-        s = (char*)newsh+hdrlen;
-    } else {
-        newsh = s_malloc(hdrlen+len+1);
-        if (newsh == NULL) return NULL;
-        memcpy((char*)newsh+hdrlen, s, len+1);
-        s_free(sh);
-        s = (char*)newsh+hdrlen;
-        s[-1] = type;
-        sdssetlen(s, len);
-    }
-    sdssetalloc(s, len);
-    return s;
-}
-
-/* Return the total size of the allocation of the specifed sds string,
- * including:
- * 1) The sds header before the pointer.
- * 2) The string.
- * 3) The free buffer at the end if any.
- * 4) The implicit null term.
- */
-size_t sdsAllocSize(sds s) {
-    size_t alloc = sdsalloc(s);
-    return sdsHdrSize(s[-1])+alloc+1;
-}
-
-/* Return the pointer of the actual SDS allocation (normally SDS strings
- * are referenced by the start of the string buffer). */
-void *sdsAllocPtr(sds s) {
-    return (void*) (s-sdsHdrSize(s[-1]));
-}
-
-/* Increment the sds length and decrements the left free space at the
- * end of the string according to 'incr'. Also set the null term
- * in the new end of the string.
- *
- * This function is used in order to fix the string length after the
- * user calls sdsMakeRoomFor(), writes something after the end of
- * the current string, and finally needs to set the new length.
- *
- * Note: it is possible to use a negative increment in order to
- * right-trim the string.
- *
- * Usage example:
- *
- * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the
- * following schema, to cat bytes coming from the kernel to the end of an
- * sds string without copying into an intermediate buffer:
- *
- * oldlen = sdslen(s);
- * s = sdsMakeRoomFor(s, BUFFER_SIZE);
- * nread = read(fd, s+oldlen, BUFFER_SIZE);
- * ... check for nread <= 0 and handle it ...
- * sdsIncrLen(s, nread);
- */
-void sdsIncrLen(sds s, int incr) {
-    unsigned char flags = s[-1];
-    size_t len;
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5: {
-            unsigned char *fp = ((unsigned char*)s)-1;
-            unsigned char oldlen = SDS_TYPE_5_LEN(flags);
-            assert((incr > 0 && oldlen+incr < 32) || (incr < 0 && oldlen >= (unsigned int)(-incr)));
-            *fp = SDS_TYPE_5 | ((oldlen+incr) << SDS_TYPE_BITS);
-            len = oldlen+incr;
-            break;
-        }
-        case SDS_TYPE_8: {
-            SDS_HDR_VAR(8,s);
-            assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr)));
-            len = (sh->len += incr);
-            break;
-        }
-        case SDS_TYPE_16: {
-            SDS_HDR_VAR(16,s);
-            assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr)));
-            len = (sh->len += incr);
-            break;
-        }
-        case SDS_TYPE_32: {
-            SDS_HDR_VAR(32,s);
-            assert((incr >= 0 && sh->alloc-sh->len >= (unsigned int)incr) || (incr < 0 && sh->len >= (unsigned int)(-incr)));
-            len = (sh->len += incr);
-            break;
-        }
-        case SDS_TYPE_64: {
-            SDS_HDR_VAR(64,s);
-            assert((incr >= 0 && sh->alloc-sh->len >= (uint64_t)incr) || (incr < 0 && sh->len >= (uint64_t)(-incr)));
-            len = (sh->len += incr);
-            break;
-        }
-        default: len = 0; /* Just to avoid compilation warnings. */
-    }
-    s[len] = '\0';
-}
-
-/* Grow the sds to have the specified length. Bytes that were not part of
- * the original length of the sds will be set to zero.
- *
- * if the specified length is smaller than the current length, no operation
- * is performed. */
-sds sdsgrowzero(sds s, size_t len) {
-    size_t curlen = sdslen(s);
-
-    if (len <= curlen) return s;
-    s = sdsMakeRoomFor(s,len-curlen);
-    if (s == NULL) return NULL;
-
-    /* Make sure added region doesn't contain garbage */
-    memset(s+curlen,0,(len-curlen+1)); /* also set trailing \0 byte */
-    sdssetlen(s, len);
-    return s;
-}
-
-/* Append the specified binary-safe string pointed by 't' of 'len' bytes to the
- * end of the specified sds string 's'.
- *
- * After the call, the passed sds string is no longer valid and all the
- * references must be substituted with the new pointer returned by the call. */
-sds sdscatlen(sds s, const void *t, size_t len) {
-    size_t curlen = sdslen(s);
-
-    s = sdsMakeRoomFor(s,len);
-    if (s == NULL) return NULL;
-    memcpy(s+curlen, t, len);
-    sdssetlen(s, curlen+len);
-    s[curlen+len] = '\0';
-    return s;
-}
-
-/* Append the specified null termianted C string to the sds string 's'.
- *
- * After the call, the passed sds string is no longer valid and all the
- * references must be substituted with the new pointer returned by the call. */
-sds sdscat(sds s, const char *t) {
-    return sdscatlen(s, t, strlen(t));
-}
-
-/* Append the specified sds 't' to the existing sds 's'.
- *
- * After the call, the modified sds string is no longer valid and all the
- * references must be substituted with the new pointer returned by the call. */
-sds sdscatsds(sds s, const sds t) {
-    return sdscatlen(s, t, sdslen(t));
-}
-
-/* Destructively modify the sds string 's' to hold the specified binary
- * safe string pointed by 't' of length 'len' bytes. */
-sds sdscpylen(sds s, const char *t, size_t len) {
-    if (sdsalloc(s) < len) {
-        s = sdsMakeRoomFor(s,len-sdslen(s));
-        if (s == NULL) return NULL;
-    }
-    memcpy(s, t, len);
-    s[len] = '\0';
-    sdssetlen(s, len);
-    return s;
-}
-
-/* Like sdscpylen() but 't' must be a null-termined string so that the length
- * of the string is obtained with strlen(). */
-sds sdscpy(sds s, const char *t) {
-    return sdscpylen(s, t, strlen(t));
-}
-
-/* Helper for sdscatlonglong() doing the actual number -> string
- * conversion. 's' must point to a string with room for at least
- * SDS_LLSTR_SIZE bytes.
- *
- * The function returns the length of the null-terminated string
- * representation stored at 's'. */
-#define SDS_LLSTR_SIZE 21
-int sdsll2str(char *s, long long value) {
-    char *p, aux;
-    unsigned long long v;
-    size_t l;
-
-    /* Generate the string representation, this method produces
-     * an reversed string. */
-    v = (value < 0) ? -value : value;
-    p = s;
-    do {
-        *p++ = '0'+(v%10);
-        v /= 10;
-    } while(v);
-    if (value < 0) *p++ = '-';
-
-    /* Compute length and add null term. */
-    l = p-s;
-    *p = '\0';
-
-    /* Reverse the string. */
-    p--;
-    while(s < p) {
-        aux = *s;
-        *s = *p;
-        *p = aux;
-        s++;
-        p--;
-    }
-    return l;
-}
-
-/* Identical sdsll2str(), but for unsigned long long type. */
-int sdsull2str(char *s, unsigned long long v) {
-    char *p, aux;
-    size_t l;
-
-    /* Generate the string representation, this method produces
-     * an reversed string. */
-    p = s;
-    do {
-        *p++ = '0'+(v%10);
-        v /= 10;
-    } while(v);
-
-    /* Compute length and add null term. */
-    l = p-s;
-    *p = '\0';
-
-    /* Reverse the string. */
-    p--;
-    while(s < p) {
-        aux = *s;
-        *s = *p;
-        *p = aux;
-        s++;
-        p--;
-    }
-    return l;
-}
-
-/* Create an sds string from a long long value. It is much faster than:
- *
- * sdscatprintf(sdsempty(),"%lld\n", value);
- */
-sds sdsfromlonglong(long long value) {
-    char buf[SDS_LLSTR_SIZE];
-    int len = sdsll2str(buf,value);
-
-    return sdsnewlen(buf,len);
-}
-
-/* Like sdscatprintf() but gets va_list instead of being variadic. */
-sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
-    va_list cpy;
-    char staticbuf[1024], *buf = staticbuf, *t;
-    size_t buflen = strlen(fmt)*2;
-
-    /* We try to start using a static buffer for speed.
-     * If not possible we revert to heap allocation. */
-    if (buflen > sizeof(staticbuf)) {
-        buf = s_malloc(buflen);
-        if (buf == NULL) return NULL;
-    } else {
-        buflen = sizeof(staticbuf);
-    }
-
-    /* Try with buffers two times bigger every time we fail to
-     * fit the string in the current buffer size. */
-    while(1) {
-        buf[buflen-2] = '\0';
-        va_copy(cpy,ap);
-        vsnprintf(buf, buflen, fmt, cpy);
-        va_end(cpy);
-        if (buf[buflen-2] != '\0') {
-            if (buf != staticbuf) s_free(buf);
-            buflen *= 2;
-            buf = s_malloc(buflen);
-            if (buf == NULL) return NULL;
-            continue;
-        }
-        break;
-    }
-
-    /* Finally concat the obtained string to the SDS string and return it. */
-    t = sdscat(s, buf);
-    if (buf != staticbuf) s_free(buf);
-    return t;
-}
-
-/* Append to the sds string 's' a string obtained using printf-alike format
- * specifier.
- *
- * After the call, the modified sds string is no longer valid and all the
- * references must be substituted with the new pointer returned by the call.
- *
- * Example:
- *
- * s = sdsnew("Sum is: ");
- * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b).
- *
- * Often you need to create a string from scratch with the printf-alike
- * format. When this is the need, just use sdsempty() as the target string:
- *
- * s = sdscatprintf(sdsempty(), "... your format ...", args);
- */
-sds sdscatprintf(sds s, const char *fmt, ...) {
-    va_list ap;
-    char *t;
-    va_start(ap, fmt);
-    t = sdscatvprintf(s,fmt,ap);
-    va_end(ap);
-    return t;
-}
-
-/* This function is similar to sdscatprintf, but much faster as it does
- * not rely on sprintf() family functions implemented by the libc that
- * are often very slow. Moreover directly handling the sds string as
- * new data is concatenated provides a performance improvement.
- *
- * However this function only handles an incompatible subset of printf-alike
- * format specifiers:
- *
- * %s - C String
- * %S - SDS string
- * %i - signed int
- * %I - 64 bit signed integer (long long, int64_t)
- * %u - unsigned int
- * %U - 64 bit unsigned integer (unsigned long long, uint64_t)
- * %% - Verbatim "%" character.
- */
-sds sdscatfmt(sds s, char const *fmt, ...) {
-    const char *f = fmt;
-    int i;
-    va_list ap;
-
-    va_start(ap,fmt);
-    i = sdslen(s); /* Position of the next byte to write to dest str. */
-    while(*f) {
-        char next, *str;
-        size_t l;
-        long long num;
-        unsigned long long unum;
-
-        /* Make sure there is always space for at least 1 char. */
-        if (sdsavail(s)==0) {
-            s = sdsMakeRoomFor(s,1);
-            if (s == NULL) goto fmt_error;
-        }
-
-        switch(*f) {
-        case '%':
-            next = *(f+1);
-            f++;
-            switch(next) {
-            case 's':
-            case 'S':
-                str = va_arg(ap,char*);
-                l = (next == 's') ? strlen(str) : sdslen(str);
-                if (sdsavail(s) < l) {
-                    s = sdsMakeRoomFor(s,l);
-                    if (s == NULL) goto fmt_error;
-                }
-                memcpy(s+i,str,l);
-                sdsinclen(s,l);
-                i += l;
-                break;
-            case 'i':
-            case 'I':
-                if (next == 'i')
-                    num = va_arg(ap,int);
-                else
-                    num = va_arg(ap,long long);
-                {
-                    char buf[SDS_LLSTR_SIZE];
-                    l = sdsll2str(buf,num);
-                    if (sdsavail(s) < l) {
-                        s = sdsMakeRoomFor(s,l);
-                        if (s == NULL) goto fmt_error;
-                    }
-                    memcpy(s+i,buf,l);
-                    sdsinclen(s,l);
-                    i += l;
-                }
-                break;
-            case 'u':
-            case 'U':
-                if (next == 'u')
-                    unum = va_arg(ap,unsigned int);
-                else
-                    unum = va_arg(ap,unsigned long long);
-                {
-                    char buf[SDS_LLSTR_SIZE];
-                    l = sdsull2str(buf,unum);
-                    if (sdsavail(s) < l) {
-                        s = sdsMakeRoomFor(s,l);
-                        if (s == NULL) goto fmt_error;
-                    }
-                    memcpy(s+i,buf,l);
-                    sdsinclen(s,l);
-                    i += l;
-                }
-                break;
-            default: /* Handle %% and generally %<unknown>. */
-                s[i++] = next;
-                sdsinclen(s,1);
-                break;
-            }
-            break;
-        default:
-            s[i++] = *f;
-            sdsinclen(s,1);
-            break;
-        }
-        f++;
-    }
-    va_end(ap);
-
-    /* Add null-term */
-    s[i] = '\0';
-    return s;
-
-fmt_error:
-    va_end(ap);
-    return NULL;
-}
-
-/* Remove the part of the string from left and from right composed just of
- * contiguous characters found in 'cset', that is a null terminted C string.
- *
- * After the call, the modified sds string is no longer valid and all the
- * references must be substituted with the new pointer returned by the call.
- *
- * Example:
- *
- * s = sdsnew("AA...AA.a.aa.aHelloWorld     :::");
- * s = sdstrim(s,"Aa. :");
- * printf("%s\n", s);
- *
- * Output will be just "Hello World".
- */
-sds sdstrim(sds s, const char *cset) {
-    char *start, *end, *sp, *ep;
-    size_t len;
-
-    sp = start = s;
-    ep = end = s+sdslen(s)-1;
-    while(sp <= end && strchr(cset, *sp)) sp++;
-    while(ep > sp && strchr(cset, *ep)) ep--;
-    len = (sp > ep) ? 0 : ((ep-sp)+1);
-    if (s != sp) memmove(s, sp, len);
-    s[len] = '\0';
-    sdssetlen(s,len);
-    return s;
-}
-
-/* Turn the string into a smaller (or equal) string containing only the
- * substring specified by the 'start' and 'end' indexes.
- *
- * start and end can be negative, where -1 means the last character of the
- * string, -2 the penultimate character, and so forth.
- *
- * The interval is inclusive, so the start and end characters will be part
- * of the resulting string.
- *
- * The string is modified in-place.
- *
- * Example:
- *
- * s = sdsnew("Hello World");
- * sdsrange(s,1,-1); => "ello World"
- */
-void sdsrange(sds s, int start, int end) {
-    size_t newlen, len = sdslen(s);
-
-    if (len == 0) return;
-    if (start < 0) {
-        start = len+start;
-        if (start < 0) start = 0;
-    }
-    if (end < 0) {
-        end = len+end;
-        if (end < 0) end = 0;
-    }
-    newlen = (start > end) ? 0 : (end-start)+1;
-    if (newlen != 0) {
-        if (start >= (signed)len) {
-            newlen = 0;
-        } else if (end >= (signed)len) {
-            end = len-1;
-            newlen = (start > end) ? 0 : (end-start)+1;
-        }
-    } else {
-        start = 0;
-    }
-    if (start && newlen) memmove(s, s+start, newlen);
-    s[newlen] = 0;
-    sdssetlen(s,newlen);
-}
-
-/* Apply tolower() to every character of the sds string 's'. */
-void sdstolower(sds s) {
-    int len = sdslen(s), j;
-
-    for (j = 0; j < len; j++) s[j] = tolower(s[j]);
-}
-
-/* Apply toupper() to every character of the sds string 's'. */
-void sdstoupper(sds s) {
-    int len = sdslen(s), j;
-
-    for (j = 0; j < len; j++) s[j] = toupper(s[j]);
-}
-
-/* Compare two sds strings s1 and s2 with memcmp().
- *
- * Return value:
- *
- *     positive if s1 > s2.
- *     negative if s1 < s2.
- *     0 if s1 and s2 are exactly the same binary string.
- *
- * If two strings share exactly the same prefix, but one of the two has
- * additional characters, the longer string is considered to be greater than
- * the smaller one. */
-int sdscmp(const sds s1, const sds s2) {
-    size_t l1, l2, minlen;
-    int cmp;
-
-    l1 = sdslen(s1);
-    l2 = sdslen(s2);
-    minlen = (l1 < l2) ? l1 : l2;
-    cmp = memcmp(s1,s2,minlen);
-    if (cmp == 0) return l1-l2;
-    return cmp;
-}
-
-/* Split 's' with separator in 'sep'. An array
- * of sds strings is returned. *count will be set
- * by reference to the number of tokens returned.
- *
- * On out of memory, zero length string, zero length
- * separator, NULL is returned.
- *
- * Note that 'sep' is able to split a string using
- * a multi-character separator. For example
- * sdssplit("foo_-_bar","_-_"); will return two
- * elements "foo" and "bar".
- *
- * This version of the function is binary-safe but
- * requires length arguments. sdssplit() is just the
- * same function but for zero-terminated strings.
- */
-sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) {
-    int elements = 0, slots = 5, start = 0, j;
-    sds *tokens;
-
-    if (seplen < 1 || len < 0) return NULL;
-
-    tokens = s_malloc(sizeof(sds)*slots);
-    if (tokens == NULL) return NULL;
-
-    if (len == 0) {
-        *count = 0;
-        return tokens;
-    }
-    for (j = 0; j < (len-(seplen-1)); j++) {
-        /* make sure there is room for the next element and the final one */
-        if (slots < elements+2) {
-            sds *newtokens;
-
-            slots *= 2;
-            newtokens = s_realloc(tokens,sizeof(sds)*slots);
-            if (newtokens == NULL) goto cleanup;
-            tokens = newtokens;
-        }
-        /* search the separator */
-        if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) {
-            tokens[elements] = sdsnewlen(s+start,j-start);
-            if (tokens[elements] == NULL) goto cleanup;
-            elements++;
-            start = j+seplen;
-            j = j+seplen-1; /* skip the separator */
-        }
-    }
-    /* Add the final element. We are sure there is room in the tokens array. */
-    tokens[elements] = sdsnewlen(s+start,len-start);
-    if (tokens[elements] == NULL) goto cleanup;
-    elements++;
-    *count = elements;
-    return tokens;
-
-cleanup:
-    {
-        int i;
-        for (i = 0; i < elements; i++) sdsfree(tokens[i]);
-        s_free(tokens);
-        *count = 0;
-        return NULL;
-    }
-}
-
-/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */
-void sdsfreesplitres(sds *tokens, int count) {
-    if (!tokens) return;
-    while(count--)
-        sdsfree(tokens[count]);
-    s_free(tokens);
-}
-
-/* Append to the sds string "s" an escaped string representation where
- * all the non-printable characters (tested with isprint()) are turned into
- * escapes in the form "\n\r\a...." or "\x<hex-number>".
- *
- * After the call, the modified sds string is no longer valid and all the
- * references must be substituted with the new pointer returned by the call. */
-sds sdscatrepr(sds s, const char *p, size_t len) {
-    s = sdscatlen(s,"\"",1);
-    while(len--) {
-        switch(*p) {
-        case '\\':
-        case '"':
-            s = sdscatprintf(s,"\\%c",*p);
-            break;
-        case '\n': s = sdscatlen(s,"\\n",2); break;
-        case '\r': s = sdscatlen(s,"\\r",2); break;
-        case '\t': s = sdscatlen(s,"\\t",2); break;
-        case '\a': s = sdscatlen(s,"\\a",2); break;
-        case '\b': s = sdscatlen(s,"\\b",2); break;
-        default:
-            if (isprint(*p))
-                s = sdscatprintf(s,"%c",*p);
-            else
-                s = sdscatprintf(s,"\\x%02x",(unsigned char)*p);
-            break;
-        }
-        p++;
-    }
-    return sdscatlen(s,"\"",1);
-}
-
-/* Helper function for sdssplitargs() that returns non zero if 'c'
- * is a valid hex digit. */
-int is_hex_digit(char c) {
-    return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') ||
-           (c >= 'A' && c <= 'F');
-}
-
-/* Helper function for sdssplitargs() that converts a hex digit into an
- * integer from 0 to 15 */
-int hex_digit_to_int(char c) {
-    switch(c) {
-    case '0': return 0;
-    case '1': return 1;
-    case '2': return 2;
-    case '3': return 3;
-    case '4': return 4;
-    case '5': return 5;
-    case '6': return 6;
-    case '7': return 7;
-    case '8': return 8;
-    case '9': return 9;
-    case 'a': case 'A': return 10;
-    case 'b': case 'B': return 11;
-    case 'c': case 'C': return 12;
-    case 'd': case 'D': return 13;
-    case 'e': case 'E': return 14;
-    case 'f': case 'F': return 15;
-    default: return 0;
-    }
-}
-
-/* Split a line into arguments, where every argument can be in the
- * following programming-language REPL-alike form:
- *
- * foo bar "newline are supported\n" and "\xff\x00otherstuff"
- *
- * The number of arguments is stored into *argc, and an array
- * of sds is returned.
- *
- * The caller should free the resulting array of sds strings with
- * sdsfreesplitres().
- *
- * Note that sdscatrepr() is able to convert back a string into
- * a quoted string in the same format sdssplitargs() is able to parse.
- *
- * The function returns the allocated tokens on success, even when the
- * input string is empty, or NULL if the input contains unbalanced
- * quotes or closed quotes followed by non space characters
- * as in: "foo"bar or "foo'
- */
-sds *sdssplitargs(const char *line, int *argc) {
-    const char *p = line;
-    char *current = NULL;
-    char **vector = NULL;
-
-    *argc = 0;
-    while(1) {
-        /* skip blanks */
-        while(*p && isspace(*p)) p++;
-        if (*p) {
-            /* get a token */
-            int inq=0;  /* set to 1 if we are in "quotes" */
-            int insq=0; /* set to 1 if we are in 'single quotes' */
-            int done=0;
-
-            if (current == NULL) current = sdsempty();
-            while(!done) {
-                if (inq) {
-                    if (*p == '\\' && *(p+1) == 'x' &&
-                                             is_hex_digit(*(p+2)) &&
-                                             is_hex_digit(*(p+3)))
-                    {
-                        unsigned char byte;
-
-                        byte = (hex_digit_to_int(*(p+2))*16)+
-                                hex_digit_to_int(*(p+3));
-                        current = sdscatlen(current,(char*)&byte,1);
-                        p += 3;
-                    } else if (*p == '\\' && *(p+1)) {
-                        char c;
-
-                        p++;
-                        switch(*p) {
-                        case 'n': c = '\n'; break;
-                        case 'r': c = '\r'; break;
-                        case 't': c = '\t'; break;
-                        case 'b': c = '\b'; break;
-                        case 'a': c = '\a'; break;
-                        default: c = *p; break;
-                        }
-                        current = sdscatlen(current,&c,1);
-                    } else if (*p == '"') {
-                        /* closing quote must be followed by a space or
-                         * nothing at all. */
-                        if (*(p+1) && !isspace(*(p+1))) goto err;
-                        done=1;
-                    } else if (!*p) {
-                        /* unterminated quotes */
-                        goto err;
-                    } else {
-                        current = sdscatlen(current,p,1);
-                    }
-                } else if (insq) {
-                    if (*p == '\\' && *(p+1) == '\'') {
-                        p++;
-                        current = sdscatlen(current,"'",1);
-                    } else if (*p == '\'') {
-                        /* closing quote must be followed by a space or
-                         * nothing at all. */
-                        if (*(p+1) && !isspace(*(p+1))) goto err;
-                        done=1;
-                    } else if (!*p) {
-                        /* unterminated quotes */
-                        goto err;
-                    } else {
-                        current = sdscatlen(current,p,1);
-                    }
-                } else {
-                    switch(*p) {
-                    case ' ':
-                    case '\n':
-                    case '\r':
-                    case '\t':
-                    case '\0':
-                        done=1;
-                        break;
-                    case '"':
-                        inq=1;
-                        break;
-                    case '\'':
-                        insq=1;
-                        break;
-                    default:
-                        current = sdscatlen(current,p,1);
-                        break;
-                    }
-                }
-                if (*p) p++;
-            }
-            /* add the token to the vector */
-            {
-                char **new_vector = s_realloc(vector,((*argc)+1)*sizeof(char*));
-                if (new_vector == NULL) {
-                    s_free(vector);
-                    return NULL;
-                }
-
-                vector = new_vector;
-                vector[*argc] = current;
-                (*argc)++;
-                current = NULL;
-            }
-        } else {
-            /* Even on empty input string return something not NULL. */
-            if (vector == NULL) vector = s_malloc(sizeof(void*));
-            return vector;
-        }
-    }
-
-err:
-    while((*argc)--)
-        sdsfree(vector[*argc]);
-    s_free(vector);
-    if (current) sdsfree(current);
-    *argc = 0;
-    return NULL;
-}
-
-/* Modify the string substituting all the occurrences of the set of
- * characters specified in the 'from' string to the corresponding character
- * in the 'to' array.
- *
- * For instance: sdsmapchars(mystring, "ho", "01", 2)
- * will have the effect of turning the string "hello" into "0ell1".
- *
- * The function returns the sds string pointer, that is always the same
- * as the input pointer since no resize is needed. */
-sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) {
-    size_t j, i, l = sdslen(s);
-
-    for (j = 0; j < l; j++) {
-        for (i = 0; i < setlen; i++) {
-            if (s[j] == from[i]) {
-                s[j] = to[i];
-                break;
-            }
-        }
-    }
-    return s;
-}
-
-/* Join an array of C strings using the specified separator (also a C string).
- * Returns the result as an sds string. */
-sds sdsjoin(char **argv, int argc, char *sep) {
-    sds join = sdsempty();
-    int j;
-
-    for (j = 0; j < argc; j++) {
-        join = sdscat(join, argv[j]);
-        if (j != argc-1) join = sdscat(join,sep);
-    }
-    return join;
-}
-
-/* Like sdsjoin, but joins an array of SDS strings. */
-sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) {
-    sds join = sdsempty();
-    int j;
-
-    for (j = 0; j < argc; j++) {
-        join = sdscatsds(join, argv[j]);
-        if (j != argc-1) join = sdscatlen(join,sep,seplen);
-    }
-    return join;
-}
-
-/* Wrappers to the allocators used by SDS. Note that SDS will actually
- * just use the macros defined into sdsalloc.h in order to avoid to pay
- * the overhead of function calls. Here we define these wrappers only for
- * the programs SDS is linked to, if they want to touch the SDS internals
- * even if they use a different allocator. */
-void *sds_malloc(size_t size) { return s_malloc(size); }
-void *sds_realloc(void *ptr, size_t size) { return s_realloc(ptr,size); }
-void sds_free(void *ptr) { s_free(ptr); }
-
-#if defined(SDS_TEST_MAIN)
-#include <stdio.h>
-#include "testhelp.h"
-#include "limits.h"
-
-#define UNUSED(x) (void)(x)
-int sdsTest(void) {
-    {
-        sds x = sdsnew("foo"), y;
-
-        test_cond("Create a string and obtain the length",
-            sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0)
-
-        sdsfree(x);
-        x = sdsnewlen("foo",2);
-        test_cond("Create a string with specified length",
-            sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0)
-
-        x = sdscat(x,"bar");
-        test_cond("Strings concatenation",
-            sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0);
-
-        x = sdscpy(x,"a");
-        test_cond("sdscpy() against an originally longer string",
-            sdslen(x) == 1 && memcmp(x,"a\0",2) == 0)
-
-        x = sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk");
-        test_cond("sdscpy() against an originally shorter string",
-            sdslen(x) == 33 &&
-            memcmp(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk\0",33) == 0)
-
-        sdsfree(x);
-        x = sdscatprintf(sdsempty(),"%d",123);
-        test_cond("sdscatprintf() seems working in the base case",
-            sdslen(x) == 3 && memcmp(x,"123\0",4) == 0)
-
-        sdsfree(x);
-        x = sdsnew("--");
-        x = sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX);
-        test_cond("sdscatfmt() seems working in the base case",
-            sdslen(x) == 60 &&
-            memcmp(x,"--Hello Hi! World -9223372036854775808,"
-                     "9223372036854775807--",60) == 0)
-        printf("[%s]\n",x);
-
-        sdsfree(x);
-        x = sdsnew("--");
-        x = sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX);
-        test_cond("sdscatfmt() seems working with unsigned numbers",
-            sdslen(x) == 35 &&
-            memcmp(x,"--4294967295,18446744073709551615--",35) == 0)
-
-        sdsfree(x);
-        x = sdsnew(" x ");
-        sdstrim(x," x");
-        test_cond("sdstrim() works when all chars match",
-            sdslen(x) == 0)
-
-        sdsfree(x);
-        x = sdsnew(" x ");
-        sdstrim(x," ");
-        test_cond("sdstrim() works when a single char remains",
-            sdslen(x) == 1 && x[0] == 'x')
-
-        sdsfree(x);
-        x = sdsnew("xxciaoyyy");
-        sdstrim(x,"xy");
-        test_cond("sdstrim() correctly trims characters",
-            sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0)
-
-        y = sdsdup(x);
-        sdsrange(y,1,1);
-        test_cond("sdsrange(...,1,1)",
-            sdslen(y) == 1 && memcmp(y,"i\0",2) == 0)
-
-        sdsfree(y);
-        y = sdsdup(x);
-        sdsrange(y,1,-1);
-        test_cond("sdsrange(...,1,-1)",
-            sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
-
-        sdsfree(y);
-        y = sdsdup(x);
-        sdsrange(y,-2,-1);
-        test_cond("sdsrange(...,-2,-1)",
-            sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0)
-
-        sdsfree(y);
-        y = sdsdup(x);
-        sdsrange(y,2,1);
-        test_cond("sdsrange(...,2,1)",
-            sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
-
-        sdsfree(y);
-        y = sdsdup(x);
-        sdsrange(y,1,100);
-        test_cond("sdsrange(...,1,100)",
-            sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
-
-        sdsfree(y);
-        y = sdsdup(x);
-        sdsrange(y,100,100);
-        test_cond("sdsrange(...,100,100)",
-            sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
-
-        sdsfree(y);
-        sdsfree(x);
-        x = sdsnew("foo");
-        y = sdsnew("foa");
-        test_cond("sdscmp(foo,foa)", sdscmp(x,y) > 0)
-
-        sdsfree(y);
-        sdsfree(x);
-        x = sdsnew("bar");
-        y = sdsnew("bar");
-        test_cond("sdscmp(bar,bar)", sdscmp(x,y) == 0)
-
-        sdsfree(y);
-        sdsfree(x);
-        x = sdsnew("aar");
-        y = sdsnew("bar");
-        test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0)
-
-        sdsfree(y);
-        sdsfree(x);
-        x = sdsnewlen("\a\n\0foo\r",7);
-        y = sdscatrepr(sdsempty(),x,sdslen(x));
-        test_cond("sdscatrepr(...data...)",
-            memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0)
-
-        {
-            unsigned int oldfree;
-            char *p;
-            int step = 10, j, i;
-
-            sdsfree(x);
-            sdsfree(y);
-            x = sdsnew("0");
-            test_cond("sdsnew() free/len buffers", sdslen(x) == 1 && sdsavail(x) == 0);
-
-            /* Run the test a few times in order to hit the first two
-             * SDS header types. */
-            for (i = 0; i < 10; i++) {
-                int oldlen = sdslen(x);
-                x = sdsMakeRoomFor(x,step);
-                int type = x[-1]&SDS_TYPE_MASK;
-
-                test_cond("sdsMakeRoomFor() len", sdslen(x) == oldlen);
-                if (type != SDS_TYPE_5) {
-                    test_cond("sdsMakeRoomFor() free", sdsavail(x) >= step);
-                    oldfree = sdsavail(x);
-                }
-                p = x+oldlen;
-                for (j = 0; j < step; j++) {
-                    p[j] = 'A'+j;
-                }
-                sdsIncrLen(x,step);
-            }
-            test_cond("sdsMakeRoomFor() content",
-                memcmp("0ABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJ",x,101) == 0);
-            test_cond("sdsMakeRoomFor() final length",sdslen(x)==101);
-
-            sdsfree(x);
-        }
-    }
-    test_report()
-    return 0;
-}
-#endif
-
-#ifdef SDS_TEST_MAIN
-int main(void) {
-    return sdsTest();
-}
-#endif
diff --git a/deps/hiredis/sds.h b/deps/hiredis/sds.h
deleted file mode 100644
index 3f9a96457..000000000
--- a/deps/hiredis/sds.h
+++ /dev/null
@@ -1,276 +0,0 @@
-/* SDSLib 2.0 -- A C dynamic strings library
- *
- * Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2015, Oran Agra
- * Copyright (c) 2015, Redis Labs, Inc
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __SDS_H
-#define __SDS_H
-
-#define SDS_MAX_PREALLOC (1024*1024)
-#ifdef _MSC_VER
-#define __attribute__(x)
-#endif
-
-#include <sys/types.h>
-#include <stdarg.h>
-#include <stdint.h>
-
-typedef char *sds;
-
-/* Note: sdshdr5 is never used, we just access the flags byte directly.
- * However is here to document the layout of type 5 SDS strings. */
-struct __attribute__ ((__packed__)) sdshdr5 {
-    unsigned char flags; /* 3 lsb of type, and 5 msb of string length */
-    char buf[];
-};
-struct __attribute__ ((__packed__)) sdshdr8 {
-    uint8_t len; /* used */
-    uint8_t alloc; /* excluding the header and null terminator */
-    unsigned char flags; /* 3 lsb of type, 5 unused bits */
-    char buf[];
-};
-struct __attribute__ ((__packed__)) sdshdr16 {
-    uint16_t len; /* used */
-    uint16_t alloc; /* excluding the header and null terminator */
-    unsigned char flags; /* 3 lsb of type, 5 unused bits */
-    char buf[];
-};
-struct __attribute__ ((__packed__)) sdshdr32 {
-    uint32_t len; /* used */
-    uint32_t alloc; /* excluding the header and null terminator */
-    unsigned char flags; /* 3 lsb of type, 5 unused bits */
-    char buf[];
-};
-struct __attribute__ ((__packed__)) sdshdr64 {
-    uint64_t len; /* used */
-    uint64_t alloc; /* excluding the header and null terminator */
-    unsigned char flags; /* 3 lsb of type, 5 unused bits */
-    char buf[];
-};
-
-#define SDS_TYPE_5  0
-#define SDS_TYPE_8  1
-#define SDS_TYPE_16 2
-#define SDS_TYPE_32 3
-#define SDS_TYPE_64 4
-#define SDS_TYPE_MASK 7
-#define SDS_TYPE_BITS 3
-#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T)));
-#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))))
-#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS)
-
-static inline size_t sdslen(const sds s) {
-    unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
-            return SDS_TYPE_5_LEN(flags);
-        case SDS_TYPE_8:
-            return SDS_HDR(8,s)->len;
-        case SDS_TYPE_16:
-            return SDS_HDR(16,s)->len;
-        case SDS_TYPE_32:
-            return SDS_HDR(32,s)->len;
-        case SDS_TYPE_64:
-            return SDS_HDR(64,s)->len;
-    }
-    return 0;
-}
-
-static inline size_t sdsavail(const sds s) {
-    unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5: {
-            return 0;
-        }
-        case SDS_TYPE_8: {
-            SDS_HDR_VAR(8,s);
-            return sh->alloc - sh->len;
-        }
-        case SDS_TYPE_16: {
-            SDS_HDR_VAR(16,s);
-            return sh->alloc - sh->len;
-        }
-        case SDS_TYPE_32: {
-            SDS_HDR_VAR(32,s);
-            return sh->alloc - sh->len;
-        }
-        case SDS_TYPE_64: {
-            SDS_HDR_VAR(64,s);
-            return sh->alloc - sh->len;
-        }
-    }
-    return 0;
-}
-
-static inline void sdssetlen(sds s, size_t newlen) {
-    unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
-            {
-                unsigned char *fp = ((unsigned char*)s)-1;
-                *fp = (unsigned char)(SDS_TYPE_5 | (newlen << SDS_TYPE_BITS));
-            }
-            break;
-        case SDS_TYPE_8:
-            SDS_HDR(8,s)->len = (uint8_t)newlen;
-            break;
-        case SDS_TYPE_16:
-            SDS_HDR(16,s)->len = (uint16_t)newlen;
-            break;
-        case SDS_TYPE_32:
-            SDS_HDR(32,s)->len = (uint32_t)newlen;
-            break;
-        case SDS_TYPE_64:
-            SDS_HDR(64,s)->len = (uint64_t)newlen;
-            break;
-    }
-}
-
-static inline void sdsinclen(sds s, size_t inc) {
-    unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
-            {
-                unsigned char *fp = ((unsigned char*)s)-1;
-                unsigned char newlen = SDS_TYPE_5_LEN(flags)+(unsigned char)inc;
-                *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
-            }
-            break;
-        case SDS_TYPE_8:
-            SDS_HDR(8,s)->len += (uint8_t)inc;
-            break;
-        case SDS_TYPE_16:
-            SDS_HDR(16,s)->len += (uint16_t)inc;
-            break;
-        case SDS_TYPE_32:
-            SDS_HDR(32,s)->len += (uint32_t)inc;
-            break;
-        case SDS_TYPE_64:
-            SDS_HDR(64,s)->len += (uint64_t)inc;
-            break;
-    }
-}
-
-/* sdsalloc() = sdsavail() + sdslen() */
-static inline size_t sdsalloc(const sds s) {
-    unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
-            return SDS_TYPE_5_LEN(flags);
-        case SDS_TYPE_8:
-            return SDS_HDR(8,s)->alloc;
-        case SDS_TYPE_16:
-            return SDS_HDR(16,s)->alloc;
-        case SDS_TYPE_32:
-            return SDS_HDR(32,s)->alloc;
-        case SDS_TYPE_64:
-            return SDS_HDR(64,s)->alloc;
-    }
-    return 0;
-}
-
-static inline void sdssetalloc(sds s, size_t newlen) {
-    unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
-            /* Nothing to do, this type has no total allocation info. */
-            break;
-        case SDS_TYPE_8:
-            SDS_HDR(8,s)->alloc = (uint8_t)newlen;
-            break;
-        case SDS_TYPE_16:
-            SDS_HDR(16,s)->alloc = (uint16_t)newlen;
-            break;
-        case SDS_TYPE_32:
-            SDS_HDR(32,s)->alloc = (uint32_t)newlen;
-            break;
-        case SDS_TYPE_64:
-            SDS_HDR(64,s)->alloc = (uint64_t)newlen;
-            break;
-    }
-}
-
-sds sdsnewlen(const void *init, size_t initlen);
-sds sdsnew(const char *init);
-sds sdsempty(void);
-sds sdsdup(const sds s);
-void sdsfree(sds s);
-sds sdsgrowzero(sds s, size_t len);
-sds sdscatlen(sds s, const void *t, size_t len);
-sds sdscat(sds s, const char *t);
-sds sdscatsds(sds s, const sds t);
-sds sdscpylen(sds s, const char *t, size_t len);
-sds sdscpy(sds s, const char *t);
-
-sds sdscatvprintf(sds s, const char *fmt, va_list ap);
-#ifdef __GNUC__
-sds sdscatprintf(sds s, const char *fmt, ...)
-    __attribute__((format(printf, 2, 3)));
-#else
-sds sdscatprintf(sds s, const char *fmt, ...);
-#endif
-
-sds sdscatfmt(sds s, char const *fmt, ...);
-sds sdstrim(sds s, const char *cset);
-void sdsrange(sds s, int start, int end);
-void sdsupdatelen(sds s);
-void sdsclear(sds s);
-int sdscmp(const sds s1, const sds s2);
-sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count);
-void sdsfreesplitres(sds *tokens, int count);
-void sdstolower(sds s);
-void sdstoupper(sds s);
-sds sdsfromlonglong(long long value);
-sds sdscatrepr(sds s, const char *p, size_t len);
-sds *sdssplitargs(const char *line, int *argc);
-sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen);
-sds sdsjoin(char **argv, int argc, char *sep);
-sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen);
-
-/* Low level functions exposed to the user API */
-sds sdsMakeRoomFor(sds s, size_t addlen);
-void sdsIncrLen(sds s, int incr);
-sds sdsRemoveFreeSpace(sds s);
-size_t sdsAllocSize(sds s);
-void *sdsAllocPtr(sds s);
-
-/* Export the allocator used by SDS to the program using SDS.
- * Sometimes the program SDS is linked to, may use a different set of
- * allocators, but may want to allocate or free things that SDS will
- * respectively free or allocate. */
-void *sds_malloc(size_t size);
-void *sds_realloc(void *ptr, size_t size);
-void sds_free(void *ptr);
-
-#ifdef REDIS_TEST
-int sdsTest(int argc, char *argv[]);
-#endif
-
-#endif
diff --git a/deps/hiredis/sdsalloc.h b/deps/hiredis/sdsalloc.h
deleted file mode 100644
index f43023c48..000000000
--- a/deps/hiredis/sdsalloc.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/* SDSLib 2.0 -- A C dynamic strings library
- *
- * Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2015, Oran Agra
- * Copyright (c) 2015, Redis Labs, Inc
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-/* SDS allocator selection.
- *
- * This file is used in order to change the SDS allocator at compile time.
- * Just define the following defines to what you want to use. Also add
- * the include of your alternate allocator if needed (not needed in order
- * to use the default libc allocator). */
-
-#define s_malloc malloc
-#define s_realloc realloc
-#define s_free free
diff --git a/deps/hiredis/sockcompat.c b/deps/hiredis/sockcompat.c
deleted file mode 100644
index 4cc2f414f..000000000
--- a/deps/hiredis/sockcompat.c
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Copyright (c) 2019, Marcus Geelnard <m at bitsnbites dot eu>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#define REDIS_SOCKCOMPAT_IMPLEMENTATION
-#include "sockcompat.h"
-
-#ifdef _WIN32
-static int _wsaErrorToErrno(int err) {
-    switch (err) {
-        case WSAEWOULDBLOCK:
-            return EWOULDBLOCK;
-        case WSAEINPROGRESS:
-            return EINPROGRESS;
-        case WSAEALREADY:
-            return EALREADY;
-        case WSAENOTSOCK:
-            return ENOTSOCK;
-        case WSAEDESTADDRREQ:
-            return EDESTADDRREQ;
-        case WSAEMSGSIZE:
-            return EMSGSIZE;
-        case WSAEPROTOTYPE:
-            return EPROTOTYPE;
-        case WSAENOPROTOOPT:
-            return ENOPROTOOPT;
-        case WSAEPROTONOSUPPORT:
-            return EPROTONOSUPPORT;
-        case WSAEOPNOTSUPP:
-            return EOPNOTSUPP;
-        case WSAEAFNOSUPPORT:
-            return EAFNOSUPPORT;
-        case WSAEADDRINUSE:
-            return EADDRINUSE;
-        case WSAEADDRNOTAVAIL:
-            return EADDRNOTAVAIL;
-        case WSAENETDOWN:
-            return ENETDOWN;
-        case WSAENETUNREACH:
-            return ENETUNREACH;
-        case WSAENETRESET:
-            return ENETRESET;
-        case WSAECONNABORTED:
-            return ECONNABORTED;
-        case WSAECONNRESET:
-            return ECONNRESET;
-        case WSAENOBUFS:
-            return ENOBUFS;
-        case WSAEISCONN:
-            return EISCONN;
-        case WSAENOTCONN:
-            return ENOTCONN;
-        case WSAETIMEDOUT:
-            return ETIMEDOUT;
-        case WSAECONNREFUSED:
-            return ECONNREFUSED;
-        case WSAELOOP:
-            return ELOOP;
-        case WSAENAMETOOLONG:
-            return ENAMETOOLONG;
-        case WSAEHOSTUNREACH:
-            return EHOSTUNREACH;
-        case WSAENOTEMPTY:
-            return ENOTEMPTY;
-        default:
-            /* We just return a generic I/O error if we could not find a relevant error. */
-            return EIO;
-    }
-}
-
-static void _updateErrno(int success) {
-    errno = success ? 0 : _wsaErrorToErrno(WSAGetLastError());
-}
-
-static int _initWinsock() {
-    static int s_initialized = 0;
-    if (!s_initialized) {
-        static WSADATA wsadata;
-        int err = WSAStartup(MAKEWORD(2,2), &wsadata);
-        if (err != 0) {
-            errno = _wsaErrorToErrno(err);
-            return 0;
-        }
-        s_initialized = 1;
-    }
-    return 1;
-}
-
-int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res) {
-    /* Note: This function is likely to be called before other functions, so run init here. */
-    if (!_initWinsock()) {
-        return EAI_FAIL;
-    }
-
-    switch (getaddrinfo(node, service, hints, res)) {
-        case 0:                     return 0;
-        case WSATRY_AGAIN:          return EAI_AGAIN;
-        case WSAEINVAL:             return EAI_BADFLAGS;
-        case WSAEAFNOSUPPORT:       return EAI_FAMILY;
-        case WSA_NOT_ENOUGH_MEMORY: return EAI_MEMORY;
-        case WSAHOST_NOT_FOUND:     return EAI_NONAME;
-        case WSATYPE_NOT_FOUND:     return EAI_SERVICE;
-        case WSAESOCKTNOSUPPORT:    return EAI_SOCKTYPE;
-        default:                    return EAI_FAIL;     /* Including WSANO_RECOVERY */
-    }
-}
-
-const char *win32_gai_strerror(int errcode) {
-    switch (errcode) {
-        case 0:            errcode = 0;                     break;
-        case EAI_AGAIN:    errcode = WSATRY_AGAIN;          break;
-        case EAI_BADFLAGS: errcode = WSAEINVAL;             break;
-        case EAI_FAMILY:   errcode = WSAEAFNOSUPPORT;       break;
-        case EAI_MEMORY:   errcode = WSA_NOT_ENOUGH_MEMORY; break;
-        case EAI_NONAME:   errcode = WSAHOST_NOT_FOUND;     break;
-        case EAI_SERVICE:  errcode = WSATYPE_NOT_FOUND;     break;
-        case EAI_SOCKTYPE: errcode = WSAESOCKTNOSUPPORT;    break;
-        default:           errcode = WSANO_RECOVERY;        break; /* Including EAI_FAIL */
-    }
-    return gai_strerror(errcode);
-}
-
-void win32_freeaddrinfo(struct addrinfo *res) {
-    freeaddrinfo(res);
-}
-
-SOCKET win32_socket(int domain, int type, int protocol) {
-    SOCKET s;
-
-    /* Note: This function is likely to be called before other functions, so run init here. */
-    if (!_initWinsock()) {
-        return INVALID_SOCKET;
-    }
-
-    _updateErrno((s = socket(domain, type, protocol)) != INVALID_SOCKET);
-    return s;
-}
-
-int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp) {
-    int ret = ioctlsocket(fd, (long)request, argp);
-    _updateErrno(ret != SOCKET_ERROR);
-    return ret != SOCKET_ERROR ? ret : -1;
-}
-
-int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) {
-    int ret = bind(sockfd, addr, addrlen);
-    _updateErrno(ret != SOCKET_ERROR);
-    return ret != SOCKET_ERROR ? ret : -1;
-}
-
-int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) {
-    int ret = connect(sockfd, addr, addrlen);
-    _updateErrno(ret != SOCKET_ERROR);
-
-    /* For Winsock connect(), the WSAEWOULDBLOCK error means the same thing as
-     * EINPROGRESS for POSIX connect(), so we do that translation to keep POSIX
-     * logic consistent. */
-    if (errno == EWOULDBLOCK) {
-        errno = EINPROGRESS;
-    }
-
-    return ret != SOCKET_ERROR ? ret : -1;
-}
-
-int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen) {
-    int ret = 0;
-    if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) {
-        if (*optlen >= sizeof (struct timeval)) {
-            struct timeval *tv = optval;
-            DWORD timeout = 0;
-            socklen_t dwlen = 0;
-            ret = getsockopt(sockfd, level, optname, (char *)&timeout, &dwlen);
-            tv->tv_sec = timeout / 1000;
-            tv->tv_usec = (timeout * 1000) % 1000000;
-        } else {
-            ret = WSAEFAULT;
-        }
-        *optlen = sizeof (struct timeval);
-    } else {
-        ret = getsockopt(sockfd, level, optname, (char*)optval, optlen);
-    }
-    _updateErrno(ret != SOCKET_ERROR);
-    return ret != SOCKET_ERROR ? ret : -1;
-}
-
-int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen) {
-    int ret = 0;
-    if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) {
-        struct timeval *tv = optval;
-        DWORD timeout = tv->tv_sec * 1000 + tv->tv_usec / 1000;
-        ret = setsockopt(sockfd, level, optname, (const char*)&timeout, sizeof(DWORD));
-    } else {
-        ret = setsockopt(sockfd, level, optname, (const char*)optval, optlen);
-    }
-    _updateErrno(ret != SOCKET_ERROR);
-    return ret != SOCKET_ERROR ? ret : -1;
-}
-
-int win32_close(SOCKET fd) {
-    int ret = closesocket(fd);
-    _updateErrno(ret != SOCKET_ERROR);
-    return ret != SOCKET_ERROR ? ret : -1;
-}
-
-ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags) {
-    int ret = recv(sockfd, (char*)buf, (int)len, flags);
-    _updateErrno(ret != SOCKET_ERROR);
-    return ret != SOCKET_ERROR ? ret : -1;
-}
-
-ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags) {
-    int ret = send(sockfd, (const char*)buf, (int)len, flags);
-    _updateErrno(ret != SOCKET_ERROR);
-    return ret != SOCKET_ERROR ? ret : -1;
-}
-
-int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout) {
-    int ret = WSAPoll(fds, nfds, timeout);
-    _updateErrno(ret != SOCKET_ERROR);
-    return ret != SOCKET_ERROR ? ret : -1;
-}
-#endif /* _WIN32 */
diff --git a/deps/hiredis/sockcompat.h b/deps/hiredis/sockcompat.h
deleted file mode 100644
index 56006c163..000000000
--- a/deps/hiredis/sockcompat.h
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright (c) 2019, Marcus Geelnard <m at bitsnbites dot eu>
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __SOCKCOMPAT_H
-#define __SOCKCOMPAT_H
-
-#ifndef _WIN32
-/* For POSIX systems we use the standard BSD socket API. */
-#include <unistd.h>
-#include <sys/socket.h>
-#include <sys/select.h>
-#include <sys/un.h>
-#include <netinet/in.h>
-#include <netinet/tcp.h>
-#include <arpa/inet.h>
-#include <netdb.h>
-#include <poll.h>
-#else
-/* For Windows we use winsock. */
-#undef _WIN32_WINNT
-#define _WIN32_WINNT 0x0600 /* To get WSAPoll etc. */
-#include <winsock2.h>
-#include <ws2tcpip.h>
-#include <stddef.h>
-
-#ifdef _MSC_VER
-typedef signed long ssize_t;
-#endif
-
-/* Emulate the parts of the BSD socket API that we need (override the winsock signatures). */
-int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res);
-const char *win32_gai_strerror(int errcode);
-void win32_freeaddrinfo(struct addrinfo *res);
-SOCKET win32_socket(int domain, int type, int protocol);
-int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp);
-int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen);
-int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen);
-int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen);
-int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen);
-int win32_close(SOCKET fd);
-ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags);
-ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags);
-typedef ULONG nfds_t;
-int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout);
-
-#ifndef REDIS_SOCKCOMPAT_IMPLEMENTATION
-#define getaddrinfo(node, service, hints, res) win32_getaddrinfo(node, service, hints, res)
-#undef gai_strerror
-#define gai_strerror(errcode) win32_gai_strerror(errcode)
-#define freeaddrinfo(res) win32_freeaddrinfo(res)
-#define socket(domain, type, protocol) win32_socket(domain, type, protocol)
-#define ioctl(fd, request, argp) win32_ioctl(fd, request, argp)
-#define bind(sockfd, addr, addrlen) win32_bind(sockfd, addr, addrlen)
-#define connect(sockfd, addr, addrlen) win32_connect(sockfd, addr, addrlen)
-#define getsockopt(sockfd, level, optname, optval, optlen) win32_getsockopt(sockfd, level, optname, optval, optlen)
-#define setsockopt(sockfd, level, optname, optval, optlen) win32_setsockopt(sockfd, level, optname, optval, optlen)
-#define close(fd) win32_close(fd)
-#define recv(sockfd, buf, len, flags) win32_recv(sockfd, buf, len, flags)
-#define send(sockfd, buf, len, flags) win32_send(sockfd, buf, len, flags)
-#define poll(fds, nfds, timeout) win32_poll(fds, nfds, timeout)
-#endif /* REDIS_SOCKCOMPAT_IMPLEMENTATION */
-#endif /* _WIN32 */
-
-#endif /* __SOCKCOMPAT_H */
diff --git a/deps/hiredis/ssl.c b/deps/hiredis/ssl.c
deleted file mode 100644
index 78ab9e43e..000000000
--- a/deps/hiredis/ssl.c
+++ /dev/null
@@ -1,448 +0,0 @@
-/*
- * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
- * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
- * Copyright (c) 2019, Redis Labs
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "hiredis.h"
-#include "async.h"
-
-#include <assert.h>
-#include <pthread.h>
-#include <errno.h>
-#include <string.h>
-
-#include <openssl/ssl.h>
-#include <openssl/err.h>
-
-#include "async_private.h"
-
-void __redisSetError(redisContext *c, int type, const char *str);
-
-/* The SSL context is attached to SSL/TLS connections as a privdata. */
-typedef struct redisSSLContext {
-    /**
-     * OpenSSL SSL_CTX; It is optional and will not be set when using
-     * user-supplied SSL.
-     */
-    SSL_CTX *ssl_ctx;
-
-    /**
-     * OpenSSL SSL object.
-     */
-    SSL *ssl;
-
-    /**
-     * SSL_write() requires to be called again with the same arguments it was
-     * previously called with in the event of an SSL_read/SSL_write situation
-     */
-    size_t lastLen;
-
-    /** Whether the SSL layer requires read (possibly before a write) */
-    int wantRead;
-
-    /**
-     * Whether a write was requested prior to a read. If set, the write()
-     * should resume whenever a read takes place, if possible
-     */
-    int pendingWrite;
-} redisSSLContext;
-
-/* Forward declaration */
-redisContextFuncs redisContextSSLFuncs;
-
-#ifdef HIREDIS_SSL_TRACE
-/**
- * Callback used for debugging
- */
-static void sslLogCallback(const SSL *ssl, int where, int ret) {
-    const char *retstr = "";
-    int should_log = 1;
-    /* Ignore low-level SSL stuff */
-
-    if (where & SSL_CB_ALERT) {
-        should_log = 1;
-    }
-    if (where == SSL_CB_HANDSHAKE_START || where == SSL_CB_HANDSHAKE_DONE) {
-        should_log = 1;
-    }
-    if ((where & SSL_CB_EXIT) && ret == 0) {
-        should_log = 1;
-    }
-
-    if (!should_log) {
-        return;
-    }
-
-    retstr = SSL_alert_type_string(ret);
-    printf("ST(0x%x). %s. R(0x%x)%s\n", where, SSL_state_string_long(ssl), ret, retstr);
-
-    if (where == SSL_CB_HANDSHAKE_DONE) {
-        printf("Using SSL version %s. Cipher=%s\n", SSL_get_version(ssl), SSL_get_cipher_name(ssl));
-    }
-}
-#endif
-
-/**
- * OpenSSL global initialization and locking handling callbacks.
- * Note that this is only required for OpenSSL < 1.1.0.
- */
-
-#if OPENSSL_VERSION_NUMBER < 0x10100000L
-#define HIREDIS_USE_CRYPTO_LOCKS
-#endif
-
-#ifdef HIREDIS_USE_CRYPTO_LOCKS
-typedef pthread_mutex_t sslLockType;
-static void sslLockInit(sslLockType *l) {
-    pthread_mutex_init(l, NULL);
-}
-static void sslLockAcquire(sslLockType *l) {
-    pthread_mutex_lock(l);
-}
-static void sslLockRelease(sslLockType *l) {
-    pthread_mutex_unlock(l);
-}
-static pthread_mutex_t *ossl_locks;
-
-static void opensslDoLock(int mode, int lkid, const char *f, int line) {
-    sslLockType *l = ossl_locks + lkid;
-
-    if (mode & CRYPTO_LOCK) {
-        sslLockAcquire(l);
-    } else {
-        sslLockRelease(l);
-    }
-
-    (void)f;
-    (void)line;
-}
-
-static void initOpensslLocks(void) {
-    unsigned ii, nlocks;
-    if (CRYPTO_get_locking_callback() != NULL) {
-        /* Someone already set the callback before us. Don't destroy it! */
-        return;
-    }
-    nlocks = CRYPTO_num_locks();
-    ossl_locks = malloc(sizeof(*ossl_locks) * nlocks);
-    for (ii = 0; ii < nlocks; ii++) {
-        sslLockInit(ossl_locks + ii);
-    }
-    CRYPTO_set_locking_callback(opensslDoLock);
-}
-#endif /* HIREDIS_USE_CRYPTO_LOCKS */
-
-/**
- * SSL Connection initialization.
- */
-
-static int redisSSLConnect(redisContext *c, SSL_CTX *ssl_ctx, SSL *ssl) {
-    if (c->privdata) {
-        __redisSetError(c, REDIS_ERR_OTHER, "redisContext was already associated");
-        return REDIS_ERR;
-    }
-    c->privdata = calloc(1, sizeof(redisSSLContext));
-
-    c->funcs = &redisContextSSLFuncs;
-    redisSSLContext *rssl = c->privdata;
-
-    rssl->ssl_ctx = ssl_ctx;
-    rssl->ssl = ssl;
-
-    SSL_set_mode(rssl->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
-    SSL_set_fd(rssl->ssl, c->fd);
-    SSL_set_connect_state(rssl->ssl);
-
-    ERR_clear_error();
-    int rv = SSL_connect(rssl->ssl);
-    if (rv == 1) {
-        return REDIS_OK;
-    }
-
-    rv = SSL_get_error(rssl->ssl, rv);
-    if (((c->flags & REDIS_BLOCK) == 0) &&
-        (rv == SSL_ERROR_WANT_READ || rv == SSL_ERROR_WANT_WRITE)) {
-        return REDIS_OK;
-    }
-
-    if (c->err == 0) {
-        char err[512];
-        if (rv == SSL_ERROR_SYSCALL)
-            snprintf(err,sizeof(err)-1,"SSL_connect failed: %s",strerror(errno));
-        else {
-            unsigned long e = ERR_peek_last_error();
-            snprintf(err,sizeof(err)-1,"SSL_connect failed: %s",
-                    ERR_reason_error_string(e));
-        }
-        __redisSetError(c, REDIS_ERR_IO, err);
-    }
-    return REDIS_ERR;
-}
-
-int redisInitiateSSL(redisContext *c, SSL *ssl) {
-    return redisSSLConnect(c, NULL, ssl);
-}
-
-int redisSecureConnection(redisContext *c, const char *capath,
-                          const char *certpath, const char *keypath, const char *servername) {
-
-    SSL_CTX *ssl_ctx = NULL;
-    SSL *ssl = NULL;
-
-    /* Initialize global OpenSSL stuff */
-    static int isInit = 0;
-    if (!isInit) {
-        isInit = 1;
-        SSL_library_init();
-#ifdef HIREDIS_USE_CRYPTO_LOCKS
-        initOpensslLocks();
-#endif
-    }
-
-    ssl_ctx = SSL_CTX_new(SSLv23_client_method());
-    if (!ssl_ctx) {
-        __redisSetError(c, REDIS_ERR_OTHER, "Failed to create SSL_CTX");
-        goto error;
-    }
-
-#ifdef HIREDIS_SSL_TRACE
-    SSL_CTX_set_info_callback(ssl_ctx, sslLogCallback);
-#endif
-    SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3);
-    SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL);
-    if ((certpath != NULL && keypath == NULL) || (keypath != NULL && certpath == NULL)) {
-        __redisSetError(c, REDIS_ERR_OTHER, "certpath and keypath must be specified together");
-        goto error;
-    }
-
-    if (capath) {
-        if (!SSL_CTX_load_verify_locations(ssl_ctx, capath, NULL)) {
-            __redisSetError(c, REDIS_ERR_OTHER, "Invalid CA certificate");
-            goto error;
-        }
-    }
-    if (certpath) {
-        if (!SSL_CTX_use_certificate_chain_file(ssl_ctx, certpath)) {
-            __redisSetError(c, REDIS_ERR_OTHER, "Invalid client certificate");
-            goto error;
-        }
-        if (!SSL_CTX_use_PrivateKey_file(ssl_ctx, keypath, SSL_FILETYPE_PEM)) {
-            __redisSetError(c, REDIS_ERR_OTHER, "Invalid client key");
-            goto error;
-        }
-    }
-
-    ssl = SSL_new(ssl_ctx);
-    if (!ssl) {
-        __redisSetError(c, REDIS_ERR_OTHER, "Couldn't create new SSL instance");
-        goto error;
-    }
-    if (servername) {
-        if (!SSL_set_tlsext_host_name(ssl, servername)) {
-            __redisSetError(c, REDIS_ERR_OTHER, "Couldn't set server name indication");
-            goto error;
-        }
-    }
-
-    return redisSSLConnect(c, ssl_ctx, ssl);
-
-error:
-    if (ssl) SSL_free(ssl);
-    if (ssl_ctx) SSL_CTX_free(ssl_ctx);
-    return REDIS_ERR;
-}
-
-static int maybeCheckWant(redisSSLContext *rssl, int rv) {
-    /**
-     * If the error is WANT_READ or WANT_WRITE, the appropriate flags are set
-     * and true is returned. False is returned otherwise
-     */
-    if (rv == SSL_ERROR_WANT_READ) {
-        rssl->wantRead = 1;
-        return 1;
-    } else if (rv == SSL_ERROR_WANT_WRITE) {
-        rssl->pendingWrite = 1;
-        return 1;
-    } else {
-        return 0;
-    }
-}
-
-/**
- * Implementation of redisContextFuncs for SSL connections.
- */
-
-static void redisSSLFreeContext(void *privdata){
-    redisSSLContext *rsc = privdata;
-
-    if (!rsc) return;
-    if (rsc->ssl) {
-        SSL_free(rsc->ssl);
-        rsc->ssl = NULL;
-    }
-    if (rsc->ssl_ctx) {
-        SSL_CTX_free(rsc->ssl_ctx);
-        rsc->ssl_ctx = NULL;
-    }
-    free(rsc);
-}
-
-static int redisSSLRead(redisContext *c, char *buf, size_t bufcap) {
-    redisSSLContext *rssl = c->privdata;
-
-    int nread = SSL_read(rssl->ssl, buf, bufcap);
-    if (nread > 0) {
-        return nread;
-    } else if (nread == 0) {
-        __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection");
-        return -1;
-    } else {
-        int err = SSL_get_error(rssl->ssl, nread);
-        if (c->flags & REDIS_BLOCK) {
-            /**
-             * In blocking mode, we should never end up in a situation where
-             * we get an error without it being an actual error, except
-             * in the case of EINTR, which can be spuriously received from
-             * debuggers or whatever.
-             */
-            if (errno == EINTR) {
-                return 0;
-            } else {
-                const char *msg = NULL;
-                if (errno == EAGAIN) {
-                    msg = "Resource temporarily unavailable";
-                }
-                __redisSetError(c, REDIS_ERR_IO, msg);
-                return -1;
-            }
-        }
-
-        /**
-         * We can very well get an EWOULDBLOCK/EAGAIN, however
-         */
-        if (maybeCheckWant(rssl, err)) {
-            return 0;
-        } else {
-            __redisSetError(c, REDIS_ERR_IO, NULL);
-            return -1;
-        }
-    }
-}
-
-static int redisSSLWrite(redisContext *c) {
-    redisSSLContext *rssl = c->privdata;
-
-    size_t len = rssl->lastLen ? rssl->lastLen : sdslen(c->obuf);
-    int rv = SSL_write(rssl->ssl, c->obuf, len);
-
-    if (rv > 0) {
-        rssl->lastLen = 0;
-    } else if (rv < 0) {
-        rssl->lastLen = len;
-
-        int err = SSL_get_error(rssl->ssl, rv);
-        if ((c->flags & REDIS_BLOCK) == 0 && maybeCheckWant(rssl, err)) {
-            return 0;
-        } else {
-            __redisSetError(c, REDIS_ERR_IO, NULL);
-            return -1;
-        }
-    }
-    return rv;
-}
-
-static void redisSSLAsyncRead(redisAsyncContext *ac) {
-    int rv;
-    redisSSLContext *rssl = ac->c.privdata;
-    redisContext *c = &ac->c;
-
-    rssl->wantRead = 0;
-
-    if (rssl->pendingWrite) {
-        int done;
-
-        /* This is probably just a write event */
-        rssl->pendingWrite = 0;
-        rv = redisBufferWrite(c, &done);
-        if (rv == REDIS_ERR) {
-            __redisAsyncDisconnect(ac);
-            return;
-        } else if (!done) {
-            _EL_ADD_WRITE(ac);
-        }
-    }
-
-    rv = redisBufferRead(c);
-    if (rv == REDIS_ERR) {
-        __redisAsyncDisconnect(ac);
-    } else {
-        _EL_ADD_READ(ac);
-        redisProcessCallbacks(ac);
-    }
-}
-
-static void redisSSLAsyncWrite(redisAsyncContext *ac) {
-    int rv, done = 0;
-    redisSSLContext *rssl = ac->c.privdata;
-    redisContext *c = &ac->c;
-
-    rssl->pendingWrite = 0;
-    rv = redisBufferWrite(c, &done);
-    if (rv == REDIS_ERR) {
-        __redisAsyncDisconnect(ac);
-        return;
-    }
-
-    if (!done) {
-        if (rssl->wantRead) {
-            /* Need to read-before-write */
-            rssl->pendingWrite = 1;
-            _EL_DEL_WRITE(ac);
-        } else {
-            /* No extra reads needed, just need to write more */
-            _EL_ADD_WRITE(ac);
-        }
-    } else {
-        /* Already done! */
-        _EL_DEL_WRITE(ac);
-    }
-
-    /* Always reschedule a read */
-    _EL_ADD_READ(ac);
-}
-
-redisContextFuncs redisContextSSLFuncs = {
-    .free_privdata = redisSSLFreeContext,
-    .async_read = redisSSLAsyncRead,
-    .async_write = redisSSLAsyncWrite,
-    .read = redisSSLRead,
-    .write = redisSSLWrite
-};
-
diff --git a/deps/hiredis/test.c b/deps/hiredis/test.c
deleted file mode 100644
index 8668e1856..000000000
--- a/deps/hiredis/test.c
+++ /dev/null
@@ -1,1012 +0,0 @@
-#include "fmacros.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <strings.h>
-#include <sys/socket.h>
-#include <sys/time.h>
-#include <netdb.h>
-#include <assert.h>
-#include <unistd.h>
-#include <signal.h>
-#include <errno.h>
-#include <limits.h>
-
-#include "hiredis.h"
-#ifdef HIREDIS_TEST_SSL
-#include "hiredis_ssl.h"
-#endif
-#include "net.h"
-
-enum connection_type {
-    CONN_TCP,
-    CONN_UNIX,
-    CONN_FD,
-    CONN_SSL
-};
-
-struct config {
-    enum connection_type type;
-
-    struct {
-        const char *host;
-        int port;
-        struct timeval timeout;
-    } tcp;
-
-    struct {
-        const char *path;
-    } unix_sock;
-
-    struct {
-        const char *host;
-        int port;
-        const char *ca_cert;
-        const char *cert;
-        const char *key;
-    } ssl;
-};
-
-/* The following lines make up our testing "framework" :) */
-static int tests = 0, fails = 0;
-#define test(_s) { printf("#%02d ", ++tests); printf(_s); }
-#define test_cond(_c) if(_c) printf("\033[0;32mPASSED\033[0;0m\n"); else {printf("\033[0;31mFAILED\033[0;0m\n"); fails++;}
-
-static long long usec(void) {
-    struct timeval tv;
-    gettimeofday(&tv,NULL);
-    return (((long long)tv.tv_sec)*1000000)+tv.tv_usec;
-}
-
-/* The assert() calls below have side effects, so we need assert()
- * even if we are compiling without asserts (-DNDEBUG). */
-#ifdef NDEBUG
-#undef assert
-#define assert(e) (void)(e)
-#endif
-
-static redisContext *select_database(redisContext *c) {
-    redisReply *reply;
-
-    /* Switch to DB 9 for testing, now that we know we can chat. */
-    reply = redisCommand(c,"SELECT 9");
-    assert(reply != NULL);
-    freeReplyObject(reply);
-
-    /* Make sure the DB is emtpy */
-    reply = redisCommand(c,"DBSIZE");
-    assert(reply != NULL);
-    if (reply->type == REDIS_REPLY_INTEGER && reply->integer == 0) {
-        /* Awesome, DB 9 is empty and we can continue. */
-        freeReplyObject(reply);
-    } else {
-        printf("Database #9 is not empty, test can not continue\n");
-        exit(1);
-    }
-
-    return c;
-}
-
-static int disconnect(redisContext *c, int keep_fd) {
-    redisReply *reply;
-
-    /* Make sure we're on DB 9. */
-    reply = redisCommand(c,"SELECT 9");
-    assert(reply != NULL);
-    freeReplyObject(reply);
-    reply = redisCommand(c,"FLUSHDB");
-    assert(reply != NULL);
-    freeReplyObject(reply);
-
-    /* Free the context as well, but keep the fd if requested. */
-    if (keep_fd)
-        return redisFreeKeepFd(c);
-    redisFree(c);
-    return -1;
-}
-
-static void do_ssl_handshake(redisContext *c, struct config config) {
-#ifdef HIREDIS_TEST_SSL
-    redisSecureConnection(c, config.ssl.ca_cert, config.ssl.cert, config.ssl.key, NULL);
-    if (c->err) {
-        printf("SSL error: %s\n", c->errstr);
-        redisFree(c);
-        exit(1);
-    }
-#else
-    (void) c;
-    (void) config;
-#endif
-}
-
-static redisContext *do_connect(struct config config) {
-    redisContext *c = NULL;
-
-    if (config.type == CONN_TCP) {
-        c = redisConnect(config.tcp.host, config.tcp.port);
-    } else if (config.type == CONN_SSL) {
-        c = redisConnect(config.ssl.host, config.ssl.port);
-    } else if (config.type == CONN_UNIX) {
-        c = redisConnectUnix(config.unix_sock.path);
-    } else if (config.type == CONN_FD) {
-        /* Create a dummy connection just to get an fd to inherit */
-        redisContext *dummy_ctx = redisConnectUnix(config.unix_sock.path);
-        if (dummy_ctx) {
-            int fd = disconnect(dummy_ctx, 1);
-            printf("Connecting to inherited fd %d\n", fd);
-            c = redisConnectFd(fd);
-        }
-    } else {
-        assert(NULL);
-    }
-
-    if (c == NULL) {
-        printf("Connection error: can't allocate redis context\n");
-        exit(1);
-    } else if (c->err) {
-        printf("Connection error: %s\n", c->errstr);
-        redisFree(c);
-        exit(1);
-    }
-
-    if (config.type == CONN_SSL) {
-        do_ssl_handshake(c, config);
-    }
-
-    return select_database(c);
-}
-
-static void do_reconnect(redisContext *c, struct config config) {
-    redisReconnect(c);
-
-    if (config.type == CONN_SSL) {
-        do_ssl_handshake(c, config);
-    }
-}
-
-static void test_format_commands(void) {
-    char *cmd;
-    int len;
-
-    test("Format command without interpolation: ");
-    len = redisFormatCommand(&cmd,"SET foo bar");
-    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 &&
-        len == 4+4+(3+2)+4+(3+2)+4+(3+2));
-    free(cmd);
-
-    test("Format command with %%s string interpolation: ");
-    len = redisFormatCommand(&cmd,"SET %s %s","foo","bar");
-    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 &&
-        len == 4+4+(3+2)+4+(3+2)+4+(3+2));
-    free(cmd);
-
-    test("Format command with %%s and an empty string: ");
-    len = redisFormatCommand(&cmd,"SET %s %s","foo","");
-    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 &&
-        len == 4+4+(3+2)+4+(3+2)+4+(0+2));
-    free(cmd);
-
-    test("Format command with an empty string in between proper interpolations: ");
-    len = redisFormatCommand(&cmd,"SET %s %s","","foo");
-    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$0\r\n\r\n$3\r\nfoo\r\n",len) == 0 &&
-        len == 4+4+(3+2)+4+(0+2)+4+(3+2));
-    free(cmd);
-
-    test("Format command with %%b string interpolation: ");
-    len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"b\0r",(size_t)3);
-    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nb\0r\r\n",len) == 0 &&
-        len == 4+4+(3+2)+4+(3+2)+4+(3+2));
-    free(cmd);
-
-    test("Format command with %%b and an empty string: ");
-    len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"",(size_t)0);
-    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 &&
-        len == 4+4+(3+2)+4+(3+2)+4+(0+2));
-    free(cmd);
-
-    test("Format command with literal %%: ");
-    len = redisFormatCommand(&cmd,"SET %% %%");
-    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$1\r\n%\r\n$1\r\n%\r\n",len) == 0 &&
-        len == 4+4+(3+2)+4+(1+2)+4+(1+2));
-    free(cmd);
-
-    /* Vararg width depends on the type. These tests make sure that the
-     * width is correctly determined using the format and subsequent varargs
-     * can correctly be interpolated. */
-#define INTEGER_WIDTH_TEST(fmt, type) do {                                                \
-    type value = 123;                                                                     \
-    test("Format command with printf-delegation (" #type "): ");                          \
-    len = redisFormatCommand(&cmd,"key:%08" fmt " str:%s", value, "hello");               \
-    test_cond(strncmp(cmd,"*2\r\n$12\r\nkey:00000123\r\n$9\r\nstr:hello\r\n",len) == 0 && \
-        len == 4+5+(12+2)+4+(9+2));                                                       \
-    free(cmd);                                                                            \
-} while(0)
-
-#define FLOAT_WIDTH_TEST(type) do {                                                       \
-    type value = 123.0;                                                                   \
-    test("Format command with printf-delegation (" #type "): ");                          \
-    len = redisFormatCommand(&cmd,"key:%08.3f str:%s", value, "hello");                   \
-    test_cond(strncmp(cmd,"*2\r\n$12\r\nkey:0123.000\r\n$9\r\nstr:hello\r\n",len) == 0 && \
-        len == 4+5+(12+2)+4+(9+2));                                                       \
-    free(cmd);                                                                            \
-} while(0)
-
-    INTEGER_WIDTH_TEST("d", int);
-    INTEGER_WIDTH_TEST("hhd", char);
-    INTEGER_WIDTH_TEST("hd", short);
-    INTEGER_WIDTH_TEST("ld", long);
-    INTEGER_WIDTH_TEST("lld", long long);
-    INTEGER_WIDTH_TEST("u", unsigned int);
-    INTEGER_WIDTH_TEST("hhu", unsigned char);
-    INTEGER_WIDTH_TEST("hu", unsigned short);
-    INTEGER_WIDTH_TEST("lu", unsigned long);
-    INTEGER_WIDTH_TEST("llu", unsigned long long);
-    FLOAT_WIDTH_TEST(float);
-    FLOAT_WIDTH_TEST(double);
-
-    test("Format command with invalid printf format: ");
-    len = redisFormatCommand(&cmd,"key:%08p %b",(void*)1234,"foo",(size_t)3);
-    test_cond(len == -1);
-
-    const char *argv[3];
-    argv[0] = "SET";
-    argv[1] = "foo\0xxx";
-    argv[2] = "bar";
-    size_t lens[3] = { 3, 7, 3 };
-    int argc = 3;
-
-    test("Format command by passing argc/argv without lengths: ");
-    len = redisFormatCommandArgv(&cmd,argc,argv,NULL);
-    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 &&
-        len == 4+4+(3+2)+4+(3+2)+4+(3+2));
-    free(cmd);
-
-    test("Format command by passing argc/argv with lengths: ");
-    len = redisFormatCommandArgv(&cmd,argc,argv,lens);
-    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 &&
-        len == 4+4+(3+2)+4+(7+2)+4+(3+2));
-    free(cmd);
-
-    sds sds_cmd;
-
-    sds_cmd = sdsempty();
-    test("Format command into sds by passing argc/argv without lengths: ");
-    len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,NULL);
-    test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 &&
-        len == 4+4+(3+2)+4+(3+2)+4+(3+2));
-    sdsfree(sds_cmd);
-
-    sds_cmd = sdsempty();
-    test("Format command into sds by passing argc/argv with lengths: ");
-    len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,lens);
-    test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 &&
-        len == 4+4+(3+2)+4+(7+2)+4+(3+2));
-    sdsfree(sds_cmd);
-}
-
-static void test_append_formatted_commands(struct config config) {
-    redisContext *c;
-    redisReply *reply;
-    char *cmd;
-    int len;
-
-    c = do_connect(config);
-
-    test("Append format command: ");
-
-    len = redisFormatCommand(&cmd, "SET foo bar");
-
-    test_cond(redisAppendFormattedCommand(c, cmd, len) == REDIS_OK);
-
-    assert(redisGetReply(c, (void*)&reply) == REDIS_OK);
-
-    free(cmd);
-    freeReplyObject(reply);
-
-    disconnect(c, 0);
-}
-
-static void test_reply_reader(void) {
-    redisReader *reader;
-    void *reply;
-    int ret;
-    int i;
-
-    test("Error handling in reply parser: ");
-    reader = redisReaderCreate();
-    redisReaderFeed(reader,(char*)"@foo\r\n",6);
-    ret = redisReaderGetReply(reader,NULL);
-    test_cond(ret == REDIS_ERR &&
-              strcasecmp(reader->errstr,"Protocol error, got \"@\" as reply type byte") == 0);
-    redisReaderFree(reader);
-
-    /* when the reply already contains multiple items, they must be free'd
-     * on an error. valgrind will bark when this doesn't happen. */
-    test("Memory cleanup in reply parser: ");
-    reader = redisReaderCreate();
-    redisReaderFeed(reader,(char*)"*2\r\n",4);
-    redisReaderFeed(reader,(char*)"$5\r\nhello\r\n",11);
-    redisReaderFeed(reader,(char*)"@foo\r\n",6);
-    ret = redisReaderGetReply(reader,NULL);
-    test_cond(ret == REDIS_ERR &&
-              strcasecmp(reader->errstr,"Protocol error, got \"@\" as reply type byte") == 0);
-    redisReaderFree(reader);
-
-    test("Set error on nested multi bulks with depth > 7: ");
-    reader = redisReaderCreate();
-
-    for (i = 0; i < 9; i++) {
-        redisReaderFeed(reader,(char*)"*1\r\n",4);
-    }
-
-    ret = redisReaderGetReply(reader,NULL);
-    test_cond(ret == REDIS_ERR &&
-              strncasecmp(reader->errstr,"No support for",14) == 0);
-    redisReaderFree(reader);
-
-    test("Correctly parses LLONG_MAX: ");
-    reader = redisReaderCreate();
-    redisReaderFeed(reader, ":9223372036854775807\r\n",22);
-    ret = redisReaderGetReply(reader,&reply);
-    test_cond(ret == REDIS_OK &&
-            ((redisReply*)reply)->type == REDIS_REPLY_INTEGER &&
-            ((redisReply*)reply)->integer == LLONG_MAX);
-    freeReplyObject(reply);
-    redisReaderFree(reader);
-
-    test("Set error when > LLONG_MAX: ");
-    reader = redisReaderCreate();
-    redisReaderFeed(reader, ":9223372036854775808\r\n",22);
-    ret = redisReaderGetReply(reader,&reply);
-    test_cond(ret == REDIS_ERR &&
-              strcasecmp(reader->errstr,"Bad integer value") == 0);
-    freeReplyObject(reply);
-    redisReaderFree(reader);
-
-    test("Correctly parses LLONG_MIN: ");
-    reader = redisReaderCreate();
-    redisReaderFeed(reader, ":-9223372036854775808\r\n",23);
-    ret = redisReaderGetReply(reader,&reply);
-    test_cond(ret == REDIS_OK &&
-            ((redisReply*)reply)->type == REDIS_REPLY_INTEGER &&
-            ((redisReply*)reply)->integer == LLONG_MIN);
-    freeReplyObject(reply);
-    redisReaderFree(reader);
-
-    test("Set error when < LLONG_MIN: ");
-    reader = redisReaderCreate();
-    redisReaderFeed(reader, ":-9223372036854775809\r\n",23);
-    ret = redisReaderGetReply(reader,&reply);
-    test_cond(ret == REDIS_ERR &&
-              strcasecmp(reader->errstr,"Bad integer value") == 0);
-    freeReplyObject(reply);
-    redisReaderFree(reader);
-
-    test("Set error when array < -1: ");
-    reader = redisReaderCreate();
-    redisReaderFeed(reader, "*-2\r\n+asdf\r\n",12);
-    ret = redisReaderGetReply(reader,&reply);
-    test_cond(ret == REDIS_ERR &&
-              strcasecmp(reader->errstr,"Multi-bulk length out of range") == 0);
-    freeReplyObject(reply);
-    redisReaderFree(reader);
-
-    test("Set error when bulk < -1: ");
-    reader = redisReaderCreate();
-    redisReaderFeed(reader, "$-2\r\nasdf\r\n",11);
-    ret = redisReaderGetReply(reader,&reply);
-    test_cond(ret == REDIS_ERR &&
-              strcasecmp(reader->errstr,"Bulk string length out of range") == 0);
-    freeReplyObject(reply);
-    redisReaderFree(reader);
-
-#if LLONG_MAX > SIZE_MAX
-    test("Set error when array > SIZE_MAX: ");
-    reader = redisReaderCreate();
-    redisReaderFeed(reader, "*9223372036854775807\r\n+asdf\r\n",29);
-    ret = redisReaderGetReply(reader,&reply);
-    test_cond(ret == REDIS_ERR &&
-            strcasecmp(reader->errstr,"Multi-bulk length out of range") == 0);
-    freeReplyObject(reply);
-    redisReaderFree(reader);
-
-    test("Set error when bulk > SIZE_MAX: ");
-    reader = redisReaderCreate();
-    redisReaderFeed(reader, "$9223372036854775807\r\nasdf\r\n",28);
-    ret = redisReaderGetReply(reader,&reply);
-    test_cond(ret == REDIS_ERR &&
-            strcasecmp(reader->errstr,"Bulk string length out of range") == 0);
-    freeReplyObject(reply);
-    redisReaderFree(reader);
-#endif
-
-    test("Works with NULL functions for reply: ");
-    reader = redisReaderCreate();
-    reader->fn = NULL;
-    redisReaderFeed(reader,(char*)"+OK\r\n",5);
-    ret = redisReaderGetReply(reader,&reply);
-    test_cond(ret == REDIS_OK && reply == (void*)REDIS_REPLY_STATUS);
-    redisReaderFree(reader);
-
-    test("Works when a single newline (\\r\\n) covers two calls to feed: ");
-    reader = redisReaderCreate();
-    reader->fn = NULL;
-    redisReaderFeed(reader,(char*)"+OK\r",4);
-    ret = redisReaderGetReply(reader,&reply);
-    assert(ret == REDIS_OK && reply == NULL);
-    redisReaderFeed(reader,(char*)"\n",1);
-    ret = redisReaderGetReply(reader,&reply);
-    test_cond(ret == REDIS_OK && reply == (void*)REDIS_REPLY_STATUS);
-    redisReaderFree(reader);
-
-    test("Don't reset state after protocol error: ");
-    reader = redisReaderCreate();
-    reader->fn = NULL;
-    redisReaderFeed(reader,(char*)"x",1);
-    ret = redisReaderGetReply(reader,&reply);
-    assert(ret == REDIS_ERR);
-    ret = redisReaderGetReply(reader,&reply);
-    test_cond(ret == REDIS_ERR && reply == NULL);
-    redisReaderFree(reader);
-
-    /* Regression test for issue #45 on GitHub. */
-    test("Don't do empty allocation for empty multi bulk: ");
-    reader = redisReaderCreate();
-    redisReaderFeed(reader,(char*)"*0\r\n",4);
-    ret = redisReaderGetReply(reader,&reply);
-    test_cond(ret == REDIS_OK &&
-        ((redisReply*)reply)->type == REDIS_REPLY_ARRAY &&
-        ((redisReply*)reply)->elements == 0);
-    freeReplyObject(reply);
-    redisReaderFree(reader);
-}
-
-static void test_free_null(void) {
-    void *redisCtx = NULL;
-    void *reply = NULL;
-
-    test("Don't fail when redisFree is passed a NULL value: ");
-    redisFree(redisCtx);
-    test_cond(redisCtx == NULL);
-
-    test("Don't fail when freeReplyObject is passed a NULL value: ");
-    freeReplyObject(reply);
-    test_cond(reply == NULL);
-}
-
-#define HIREDIS_BAD_DOMAIN "idontexist-noreally.com"
-static void test_blocking_connection_errors(void) {
-    redisContext *c;
-    struct addrinfo hints = {.ai_family = AF_INET};
-    struct addrinfo *ai_tmp = NULL;
-
-    int rv = getaddrinfo(HIREDIS_BAD_DOMAIN, "6379", &hints, &ai_tmp);
-    if (rv != 0) {
-        // Address does *not* exist
-        test("Returns error when host cannot be resolved: ");
-        // First see if this domain name *actually* resolves to NXDOMAIN
-        c = redisConnect(HIREDIS_BAD_DOMAIN, 6379);
-        test_cond(
-            c->err == REDIS_ERR_OTHER &&
-            (strcmp(c->errstr, "Name or service not known") == 0 ||
-             strcmp(c->errstr, "Can't resolve: " HIREDIS_BAD_DOMAIN) == 0 ||
-             strcmp(c->errstr, "Name does not resolve") == 0 ||
-             strcmp(c->errstr,
-                    "nodename nor servname provided, or not known") == 0 ||
-             strcmp(c->errstr, "No address associated with hostname") == 0 ||
-             strcmp(c->errstr, "Temporary failure in name resolution") == 0 ||
-             strcmp(c->errstr,
-                    "hostname nor servname provided, or not known") == 0 ||
-             strcmp(c->errstr, "no address associated with name") == 0));
-        redisFree(c);
-    } else {
-        printf("Skipping NXDOMAIN test. Found evil ISP!\n");
-        freeaddrinfo(ai_tmp);
-    }
-
-    test("Returns error when the port is not open: ");
-    c = redisConnect((char*)"localhost", 1);
-    test_cond(c->err == REDIS_ERR_IO &&
-        strcmp(c->errstr,"Connection refused") == 0);
-    redisFree(c);
-
-    test("Returns error when the unix_sock socket path doesn't accept connections: ");
-    c = redisConnectUnix((char*)"/tmp/idontexist.sock");
-    test_cond(c->err == REDIS_ERR_IO); /* Don't care about the message... */
-    redisFree(c);
-}
-
-static void test_blocking_connection(struct config config) {
-    redisContext *c;
-    redisReply *reply;
-
-    c = do_connect(config);
-
-    test("Is able to deliver commands: ");
-    reply = redisCommand(c,"PING");
-    test_cond(reply->type == REDIS_REPLY_STATUS &&
-        strcasecmp(reply->str,"pong") == 0)
-    freeReplyObject(reply);
-
-    test("Is a able to send commands verbatim: ");
-    reply = redisCommand(c,"SET foo bar");
-    test_cond (reply->type == REDIS_REPLY_STATUS &&
-        strcasecmp(reply->str,"ok") == 0)
-    freeReplyObject(reply);
-
-    test("%%s String interpolation works: ");
-    reply = redisCommand(c,"SET %s %s","foo","hello world");
-    freeReplyObject(reply);
-    reply = redisCommand(c,"GET foo");
-    test_cond(reply->type == REDIS_REPLY_STRING &&
-        strcmp(reply->str,"hello world") == 0);
-    freeReplyObject(reply);
-
-    test("%%b String interpolation works: ");
-    reply = redisCommand(c,"SET %b %b","foo",(size_t)3,"hello\x00world",(size_t)11);
-    freeReplyObject(reply);
-    reply = redisCommand(c,"GET foo");
-    test_cond(reply->type == REDIS_REPLY_STRING &&
-        memcmp(reply->str,"hello\x00world",11) == 0)
-
-    test("Binary reply length is correct: ");
-    test_cond(reply->len == 11)
-    freeReplyObject(reply);
-
-    test("Can parse nil replies: ");
-    reply = redisCommand(c,"GET nokey");
-    test_cond(reply->type == REDIS_REPLY_NIL)
-    freeReplyObject(reply);
-
-    /* test 7 */
-    test("Can parse integer replies: ");
-    reply = redisCommand(c,"INCR mycounter");
-    test_cond(reply->type == REDIS_REPLY_INTEGER && reply->integer == 1)
-    freeReplyObject(reply);
-
-    test("Can parse multi bulk replies: ");
-    freeReplyObject(redisCommand(c,"LPUSH mylist foo"));
-    freeReplyObject(redisCommand(c,"LPUSH mylist bar"));
-    reply = redisCommand(c,"LRANGE mylist 0 -1");
-    test_cond(reply->type == REDIS_REPLY_ARRAY &&
-              reply->elements == 2 &&
-              !memcmp(reply->element[0]->str,"bar",3) &&
-              !memcmp(reply->element[1]->str,"foo",3))
-    freeReplyObject(reply);
-
-    /* m/e with multi bulk reply *before* other reply.
-     * specifically test ordering of reply items to parse. */
-    test("Can handle nested multi bulk replies: ");
-    freeReplyObject(redisCommand(c,"MULTI"));
-    freeReplyObject(redisCommand(c,"LRANGE mylist 0 -1"));
-    freeReplyObject(redisCommand(c,"PING"));
-    reply = (redisCommand(c,"EXEC"));
-    test_cond(reply->type == REDIS_REPLY_ARRAY &&
-              reply->elements == 2 &&
-              reply->element[0]->type == REDIS_REPLY_ARRAY &&
-              reply->element[0]->elements == 2 &&
-              !memcmp(reply->element[0]->element[0]->str,"bar",3) &&
-              !memcmp(reply->element[0]->element[1]->str,"foo",3) &&
-              reply->element[1]->type == REDIS_REPLY_STATUS &&
-              strcasecmp(reply->element[1]->str,"pong") == 0);
-    freeReplyObject(reply);
-
-    disconnect(c, 0);
-}
-
-static void test_blocking_connection_timeouts(struct config config) {
-    redisContext *c;
-    redisReply *reply;
-    ssize_t s;
-    const char *cmd = "DEBUG SLEEP 3\r\n";
-    struct timeval tv;
-
-    c = do_connect(config);
-    test("Successfully completes a command when the timeout is not exceeded: ");
-    reply = redisCommand(c,"SET foo fast");
-    freeReplyObject(reply);
-    tv.tv_sec = 0;
-    tv.tv_usec = 10000;
-    redisSetTimeout(c, tv);
-    reply = redisCommand(c, "GET foo");
-    test_cond(reply != NULL && reply->type == REDIS_REPLY_STRING && memcmp(reply->str, "fast", 4) == 0);
-    freeReplyObject(reply);
-    disconnect(c, 0);
-
-    c = do_connect(config);
-    test("Does not return a reply when the command times out: ");
-    redisAppendFormattedCommand(c, cmd, strlen(cmd));
-    s = c->funcs->write(c);
-    tv.tv_sec = 0;
-    tv.tv_usec = 10000;
-    redisSetTimeout(c, tv);
-    reply = redisCommand(c, "GET foo");
-    test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_IO && strcmp(c->errstr, "Resource temporarily unavailable") == 0);
-    freeReplyObject(reply);
-
-    test("Reconnect properly reconnects after a timeout: ");
-    do_reconnect(c, config);
-    reply = redisCommand(c, "PING");
-    test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0);
-    freeReplyObject(reply);
-
-    test("Reconnect properly uses owned parameters: ");
-    config.tcp.host = "foo";
-    config.unix_sock.path = "foo";
-    do_reconnect(c, config);
-    reply = redisCommand(c, "PING");
-    test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0);
-    freeReplyObject(reply);
-
-    disconnect(c, 0);
-}
-
-static void test_blocking_io_errors(struct config config) {
-    redisContext *c;
-    redisReply *reply;
-    void *_reply;
-    int major, minor;
-
-    /* Connect to target given by config. */
-    c = do_connect(config);
-    {
-        /* Find out Redis version to determine the path for the next test */
-        const char *field = "redis_version:";
-        char *p, *eptr;
-
-        reply = redisCommand(c,"INFO");
-        p = strstr(reply->str,field);
-        major = strtol(p+strlen(field),&eptr,10);
-        p = eptr+1; /* char next to the first "." */
-        minor = strtol(p,&eptr,10);
-        freeReplyObject(reply);
-    }
-
-    test("Returns I/O error when the connection is lost: ");
-    reply = redisCommand(c,"QUIT");
-    if (major > 2 || (major == 2 && minor > 0)) {
-        /* > 2.0 returns OK on QUIT and read() should be issued once more
-         * to know the descriptor is at EOF. */
-        test_cond(strcasecmp(reply->str,"OK") == 0 &&
-            redisGetReply(c,&_reply) == REDIS_ERR);
-        freeReplyObject(reply);
-    } else {
-        test_cond(reply == NULL);
-    }
-
-    /* On 2.0, QUIT will cause the connection to be closed immediately and
-     * the read(2) for the reply on QUIT will set the error to EOF.
-     * On >2.0, QUIT will return with OK and another read(2) needed to be
-     * issued to find out the socket was closed by the server. In both
-     * conditions, the error will be set to EOF. */
-    assert(c->err == REDIS_ERR_EOF &&
-        strcmp(c->errstr,"Server closed the connection") == 0);
-    redisFree(c);
-
-    c = do_connect(config);
-    test("Returns I/O error on socket timeout: ");
-    struct timeval tv = { 0, 1000 };
-    assert(redisSetTimeout(c,tv) == REDIS_OK);
-    test_cond(redisGetReply(c,&_reply) == REDIS_ERR &&
-        c->err == REDIS_ERR_IO && errno == EAGAIN);
-    redisFree(c);
-}
-
-static void test_invalid_timeout_errors(struct config config) {
-    redisContext *c;
-
-    test("Set error when an invalid timeout usec value is given to redisConnectWithTimeout: ");
-
-    config.tcp.timeout.tv_sec = 0;
-    config.tcp.timeout.tv_usec = 10000001;
-
-    c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout);
-
-    test_cond(c->err == REDIS_ERR_IO && strcmp(c->errstr, "Invalid timeout specified") == 0);
-    redisFree(c);
-
-    test("Set error when an invalid timeout sec value is given to redisConnectWithTimeout: ");
-
-    config.tcp.timeout.tv_sec = (((LONG_MAX) - 999) / 1000) + 1;
-    config.tcp.timeout.tv_usec = 0;
-
-    c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout);
-
-    test_cond(c->err == REDIS_ERR_IO && strcmp(c->errstr, "Invalid timeout specified") == 0);
-    redisFree(c);
-}
-
-static void test_throughput(struct config config) {
-    redisContext *c = do_connect(config);
-    redisReply **replies;
-    int i, num;
-    long long t1, t2;
-
-    test("Throughput:\n");
-    for (i = 0; i < 500; i++)
-        freeReplyObject(redisCommand(c,"LPUSH mylist foo"));
-
-    num = 1000;
-    replies = malloc(sizeof(redisReply*)*num);
-    t1 = usec();
-    for (i = 0; i < num; i++) {
-        replies[i] = redisCommand(c,"PING");
-        assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_STATUS);
-    }
-    t2 = usec();
-    for (i = 0; i < num; i++) freeReplyObject(replies[i]);
-    free(replies);
-    printf("\t(%dx PING: %.3fs)\n", num, (t2-t1)/1000000.0);
-
-    replies = malloc(sizeof(redisReply*)*num);
-    t1 = usec();
-    for (i = 0; i < num; i++) {
-        replies[i] = redisCommand(c,"LRANGE mylist 0 499");
-        assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_ARRAY);
-        assert(replies[i] != NULL && replies[i]->elements == 500);
-    }
-    t2 = usec();
-    for (i = 0; i < num; i++) freeReplyObject(replies[i]);
-    free(replies);
-    printf("\t(%dx LRANGE with 500 elements: %.3fs)\n", num, (t2-t1)/1000000.0);
-
-    replies = malloc(sizeof(redisReply*)*num);
-    t1 = usec();
-    for (i = 0; i < num; i++) {
-        replies[i] = redisCommand(c, "INCRBY incrkey %d", 1000000);
-        assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_INTEGER);
-    }
-    t2 = usec();
-    for (i = 0; i < num; i++) freeReplyObject(replies[i]);
-    free(replies);
-    printf("\t(%dx INCRBY: %.3fs)\n", num, (t2-t1)/1000000.0);
-
-    num = 10000;
-    replies = malloc(sizeof(redisReply*)*num);
-    for (i = 0; i < num; i++)
-        redisAppendCommand(c,"PING");
-    t1 = usec();
-    for (i = 0; i < num; i++) {
-        assert(redisGetReply(c, (void*)&replies[i]) == REDIS_OK);
-        assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_STATUS);
-    }
-    t2 = usec();
-    for (i = 0; i < num; i++) freeReplyObject(replies[i]);
-    free(replies);
-    printf("\t(%dx PING (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0);
-
-    replies = malloc(sizeof(redisReply*)*num);
-    for (i = 0; i < num; i++)
-        redisAppendCommand(c,"LRANGE mylist 0 499");
-    t1 = usec();
-    for (i = 0; i < num; i++) {
-        assert(redisGetReply(c, (void*)&replies[i]) == REDIS_OK);
-        assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_ARRAY);
-        assert(replies[i] != NULL && replies[i]->elements == 500);
-    }
-    t2 = usec();
-    for (i = 0; i < num; i++) freeReplyObject(replies[i]);
-    free(replies);
-    printf("\t(%dx LRANGE with 500 elements (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0);
-
-    replies = malloc(sizeof(redisReply*)*num);
-    for (i = 0; i < num; i++)
-        redisAppendCommand(c,"INCRBY incrkey %d", 1000000);
-    t1 = usec();
-    for (i = 0; i < num; i++) {
-        assert(redisGetReply(c, (void*)&replies[i]) == REDIS_OK);
-        assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_INTEGER);
-    }
-    t2 = usec();
-    for (i = 0; i < num; i++) freeReplyObject(replies[i]);
-    free(replies);
-    printf("\t(%dx INCRBY (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0);
-
-    disconnect(c, 0);
-}
-
-// static long __test_callback_flags = 0;
-// static void __test_callback(redisContext *c, void *privdata) {
-//     ((void)c);
-//     /* Shift to detect execution order */
-//     __test_callback_flags <<= 8;
-//     __test_callback_flags |= (long)privdata;
-// }
-//
-// static void __test_reply_callback(redisContext *c, redisReply *reply, void *privdata) {
-//     ((void)c);
-//     /* Shift to detect execution order */
-//     __test_callback_flags <<= 8;
-//     __test_callback_flags |= (long)privdata;
-//     if (reply) freeReplyObject(reply);
-// }
-//
-// static redisContext *__connect_nonblock() {
-//     /* Reset callback flags */
-//     __test_callback_flags = 0;
-//     return redisConnectNonBlock("127.0.0.1", port, NULL);
-// }
-//
-// static void test_nonblocking_connection() {
-//     redisContext *c;
-//     int wdone = 0;
-//
-//     test("Calls command callback when command is issued: ");
-//     c = __connect_nonblock();
-//     redisSetCommandCallback(c,__test_callback,(void*)1);
-//     redisCommand(c,"PING");
-//     test_cond(__test_callback_flags == 1);
-//     redisFree(c);
-//
-//     test("Calls disconnect callback on redisDisconnect: ");
-//     c = __connect_nonblock();
-//     redisSetDisconnectCallback(c,__test_callback,(void*)2);
-//     redisDisconnect(c);
-//     test_cond(__test_callback_flags == 2);
-//     redisFree(c);
-//
-//     test("Calls disconnect callback and free callback on redisFree: ");
-//     c = __connect_nonblock();
-//     redisSetDisconnectCallback(c,__test_callback,(void*)2);
-//     redisSetFreeCallback(c,__test_callback,(void*)4);
-//     redisFree(c);
-//     test_cond(__test_callback_flags == ((2 << 8) | 4));
-//
-//     test("redisBufferWrite against empty write buffer: ");
-//     c = __connect_nonblock();
-//     test_cond(redisBufferWrite(c,&wdone) == REDIS_OK && wdone == 1);
-//     redisFree(c);
-//
-//     test("redisBufferWrite against not yet connected fd: ");
-//     c = __connect_nonblock();
-//     redisCommand(c,"PING");
-//     test_cond(redisBufferWrite(c,NULL) == REDIS_ERR &&
-//               strncmp(c->error,"write:",6) == 0);
-//     redisFree(c);
-//
-//     test("redisBufferWrite against closed fd: ");
-//     c = __connect_nonblock();
-//     redisCommand(c,"PING");
-//     redisDisconnect(c);
-//     test_cond(redisBufferWrite(c,NULL) == REDIS_ERR &&
-//               strncmp(c->error,"write:",6) == 0);
-//     redisFree(c);
-//
-//     test("Process callbacks in the right sequence: ");
-//     c = __connect_nonblock();
-//     redisCommandWithCallback(c,__test_reply_callback,(void*)1,"PING");
-//     redisCommandWithCallback(c,__test_reply_callback,(void*)2,"PING");
-//     redisCommandWithCallback(c,__test_reply_callback,(void*)3,"PING");
-//
-//     /* Write output buffer */
-//     wdone = 0;
-//     while(!wdone) {
-//         usleep(500);
-//         redisBufferWrite(c,&wdone);
-//     }
-//
-//     /* Read until at least one callback is executed (the 3 replies will
-//      * arrive in a single packet, causing all callbacks to be executed in
-//      * a single pass). */
-//     while(__test_callback_flags == 0) {
-//         assert(redisBufferRead(c) == REDIS_OK);
-//         redisProcessCallbacks(c);
-//     }
-//     test_cond(__test_callback_flags == 0x010203);
-//     redisFree(c);
-//
-//     test("redisDisconnect executes pending callbacks with NULL reply: ");
-//     c = __connect_nonblock();
-//     redisSetDisconnectCallback(c,__test_callback,(void*)1);
-//     redisCommandWithCallback(c,__test_reply_callback,(void*)2,"PING");
-//     redisDisconnect(c);
-//     test_cond(__test_callback_flags == 0x0201);
-//     redisFree(c);
-// }
-
-int main(int argc, char **argv) {
-    struct config cfg = {
-        .tcp = {
-            .host = "127.0.0.1",
-            .port = 6379
-        },
-        .unix_sock = {
-            .path = "/tmp/redis.sock"
-        }
-    };
-    int throughput = 1;
-    int test_inherit_fd = 1;
-
-    /* Ignore broken pipe signal (for I/O error tests). */
-    signal(SIGPIPE, SIG_IGN);
-
-    /* Parse command line options. */
-    argv++; argc--;
-    while (argc) {
-        if (argc >= 2 && !strcmp(argv[0],"-h")) {
-            argv++; argc--;
-            cfg.tcp.host = argv[0];
-        } else if (argc >= 2 && !strcmp(argv[0],"-p")) {
-            argv++; argc--;
-            cfg.tcp.port = atoi(argv[0]);
-        } else if (argc >= 2 && !strcmp(argv[0],"-s")) {
-            argv++; argc--;
-            cfg.unix_sock.path = argv[0];
-        } else if (argc >= 1 && !strcmp(argv[0],"--skip-throughput")) {
-            throughput = 0;
-        } else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) {
-            test_inherit_fd = 0;
-#ifdef HIREDIS_TEST_SSL
-        } else if (argc >= 2 && !strcmp(argv[0],"--ssl-port")) {
-            argv++; argc--;
-            cfg.ssl.port = atoi(argv[0]);
-        } else if (argc >= 2 && !strcmp(argv[0],"--ssl-host")) {
-            argv++; argc--;
-            cfg.ssl.host = argv[0];
-        } else if (argc >= 2 && !strcmp(argv[0],"--ssl-ca-cert")) {
-            argv++; argc--;
-            cfg.ssl.ca_cert  = argv[0];
-        } else if (argc >= 2 && !strcmp(argv[0],"--ssl-cert")) {
-            argv++; argc--;
-            cfg.ssl.cert = argv[0];
-        } else if (argc >= 2 && !strcmp(argv[0],"--ssl-key")) {
-            argv++; argc--;
-            cfg.ssl.key = argv[0];
-#endif
-        } else {
-            fprintf(stderr, "Invalid argument: %s\n", argv[0]);
-            exit(1);
-        }
-        argv++; argc--;
-    }
-
-    test_format_commands();
-    test_reply_reader();
-    test_blocking_connection_errors();
-    test_free_null();
-
-    printf("\nTesting against TCP connection (%s:%d):\n", cfg.tcp.host, cfg.tcp.port);
-    cfg.type = CONN_TCP;
-    test_blocking_connection(cfg);
-    test_blocking_connection_timeouts(cfg);
-    test_blocking_io_errors(cfg);
-    test_invalid_timeout_errors(cfg);
-    test_append_formatted_commands(cfg);
-    if (throughput) test_throughput(cfg);
-
-    printf("\nTesting against Unix socket connection (%s):\n", cfg.unix_sock.path);
-    cfg.type = CONN_UNIX;
-    test_blocking_connection(cfg);
-    test_blocking_connection_timeouts(cfg);
-    test_blocking_io_errors(cfg);
-    if (throughput) test_throughput(cfg);
-
-#ifdef HIREDIS_TEST_SSL
-    if (cfg.ssl.port && cfg.ssl.host) {
-        printf("\nTesting against SSL connection (%s:%d):\n", cfg.ssl.host, cfg.ssl.port);
-        cfg.type = CONN_SSL;
-
-        test_blocking_connection(cfg);
-        test_blocking_connection_timeouts(cfg);
-        test_blocking_io_errors(cfg);
-        test_invalid_timeout_errors(cfg);
-        test_append_formatted_commands(cfg);
-        if (throughput) test_throughput(cfg);
-    }
-#endif
-
-    if (test_inherit_fd) {
-        printf("\nTesting against inherited fd (%s):\n", cfg.unix_sock.path);
-        cfg.type = CONN_FD;
-        test_blocking_connection(cfg);
-    }
-
-
-    if (fails) {
-        printf("*** %d TESTS FAILED ***\n", fails);
-        return 1;
-    }
-
-    printf("ALL TESTS PASSED\n");
-    return 0;
-}
diff --git a/deps/hiredis/test.sh b/deps/hiredis/test.sh
deleted file mode 100755
index 2cab9e6fb..000000000
--- a/deps/hiredis/test.sh
+++ /dev/null
@@ -1,70 +0,0 @@
-#!/bin/sh -ue
-
-REDIS_SERVER=${REDIS_SERVER:-redis-server}
-REDIS_PORT=${REDIS_PORT:-56379}
-REDIS_SSL_PORT=${REDIS_SSL_PORT:-56443}
-TEST_SSL=${TEST_SSL:-0}
-SSL_TEST_ARGS=
-
-tmpdir=$(mktemp -d)
-PID_FILE=${tmpdir}/hiredis-test-redis.pid
-SOCK_FILE=${tmpdir}/hiredis-test-redis.sock
-
-if [ "$TEST_SSL" = "1" ]; then
-    SSL_CA_CERT=${tmpdir}/ca.crt
-    SSL_CA_KEY=${tmpdir}/ca.key
-    SSL_CERT=${tmpdir}/redis.crt
-    SSL_KEY=${tmpdir}/redis.key
-
-    openssl genrsa -out ${tmpdir}/ca.key 4096
-    openssl req \
-        -x509 -new -nodes -sha256 \
-        -key ${SSL_CA_KEY} \
-        -days 3650 \
-        -subj '/CN=Hiredis Test CA' \
-        -out ${SSL_CA_CERT}
-    openssl genrsa -out ${SSL_KEY} 2048
-    openssl req \
-        -new -sha256 \
-        -key ${SSL_KEY} \
-        -subj '/CN=Hiredis Test Cert' | \
-        openssl x509 \
-            -req -sha256 \
-            -CA ${SSL_CA_CERT} \
-            -CAkey ${SSL_CA_KEY} \
-            -CAserial ${tmpdir}/ca.txt \
-            -CAcreateserial \
-            -days 365 \
-            -out ${SSL_CERT}
-
-    SSL_TEST_ARGS="--ssl-host 127.0.0.1 --ssl-port ${REDIS_SSL_PORT} --ssl-ca-cert ${SSL_CA_CERT} --ssl-cert ${SSL_CERT} --ssl-key ${SSL_KEY}"
-fi
-
-cleanup() {
-  set +e
-  kill $(cat ${PID_FILE})
-  rm -rf ${tmpdir}
-}
-trap cleanup INT TERM EXIT
-
-cat > ${tmpdir}/redis.conf <<EOF
-daemonize yes
-pidfile ${PID_FILE}
-port ${REDIS_PORT}
-bind 127.0.0.1
-unixsocket ${SOCK_FILE}
-EOF
-
-if [ "$TEST_SSL" = "1" ]; then
-    cat >> ${tmpdir}/redis.conf <<EOF
-tls-port ${REDIS_SSL_PORT}
-tls-ca-cert-file ${SSL_CA_CERT}
-tls-cert-file ${SSL_CERT}
-tls-key-file ${SSL_KEY}
-EOF
-fi
-
-cat ${tmpdir}/redis.conf
-${REDIS_SERVER} ${tmpdir}/redis.conf
-
-${TEST_PREFIX:-} ./hiredis-test -h 127.0.0.1 -p ${REDIS_PORT} -s ${SOCK_FILE} ${SSL_TEST_ARGS}
diff --git a/deps/hiredis/win32.h b/deps/hiredis/win32.h
deleted file mode 100644
index 04289c696..000000000
--- a/deps/hiredis/win32.h
+++ /dev/null
@@ -1,56 +0,0 @@
-#ifndef _WIN32_HELPER_INCLUDE
-#define _WIN32_HELPER_INCLUDE
-#ifdef _MSC_VER
-
-#include <winsock2.h> /* for struct timeval */
-
-#ifndef inline
-#define inline __inline
-#endif
-
-#ifndef strcasecmp
-#define strcasecmp stricmp
-#endif
-
-#ifndef strncasecmp
-#define strncasecmp strnicmp
-#endif
-
-#ifndef va_copy
-#define va_copy(d,s) ((d) = (s))
-#endif
-
-#ifndef snprintf
-#define snprintf c99_snprintf
-
-__inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap)
-{
-    int count = -1;
-
-    if (size != 0)
-        count = _vsnprintf_s(str, size, _TRUNCATE, format, ap);
-    if (count == -1)
-        count = _vscprintf(format, ap);
-
-    return count;
-}
-
-__inline int c99_snprintf(char* str, size_t size, const char* format, ...)
-{
-    int count;
-    va_list ap;
-
-    va_start(ap, format);
-    count = c99_vsnprintf(str, size, format, ap);
-    va_end(ap);
-
-    return count;
-}
-#endif
-#endif /* _MSC_VER */
-
-#ifdef _WIN32
-#define strerror_r(errno,buf,len) strerror_s(buf,len,errno)
-#endif /* _WIN32 */
-
-#endif /* _WIN32_HELPER_INCLUDE */

From 7ee5a41aac7e5abc90d050fa509fa953ca7f1da1 Mon Sep 17 00:00:00 2001
From: michael-grunder <michael.grunder@gmail.com>
Date: Thu, 6 Aug 2020 12:41:58 -0700
Subject: [PATCH 0545/1098] Squashed 'deps/hiredis/' content from commit
 39de5267c

git-subtree-dir: deps/hiredis
git-subtree-split: 39de5267c092859b4cab4bdf79081e9634b70e39
---
 .gitignore                      |    9 +
 .travis.yml                     |  131 +++
 CHANGELOG.md                    |  345 ++++++++
 CMakeLists.txt                  |  165 ++++
 COPYING                         |   29 +
 Makefile                        |  308 +++++++
 README.md                       |  658 +++++++++++++++
 adapters/ae.h                   |  130 +++
 adapters/glib.h                 |  156 ++++
 adapters/ivykis.h               |   84 ++
 adapters/libev.h                |  179 ++++
 adapters/libevent.h             |  175 ++++
 adapters/libuv.h                |  117 +++
 adapters/macosx.h               |  115 +++
 adapters/qt.h                   |  135 +++
 alloc.c                         |   86 ++
 alloc.h                         |   91 ++
 appveyor.yml                    |   24 +
 async.c                         |  887 ++++++++++++++++++++
 async.h                         |  147 ++++
 async_private.h                 |   75 ++
 dict.c                          |  352 ++++++++
 dict.h                          |  126 +++
 examples/CMakeLists.txt         |   49 ++
 examples/example-ae.c           |   62 ++
 examples/example-glib.c         |   73 ++
 examples/example-ivykis.c       |   60 ++
 examples/example-libev.c        |   54 ++
 examples/example-libevent-ssl.c |   90 ++
 examples/example-libevent.c     |   67 ++
 examples/example-libuv.c        |   56 ++
 examples/example-macosx.c       |   66 ++
 examples/example-push.c         |  160 ++++
 examples/example-qt.cpp         |   46 +
 examples/example-qt.h           |   32 +
 examples/example-ssl.c          |  110 +++
 examples/example.c              |   91 ++
 fmacros.h                       |   12 +
 hiredis-config.cmake.in         |   13 +
 hiredis.c                       | 1173 ++++++++++++++++++++++++++
 hiredis.h                       |  336 ++++++++
 hiredis.pc.in                   |   12 +
 hiredis_ssl-config.cmake.in     |   13 +
 hiredis_ssl.h                   |  127 +++
 hiredis_ssl.pc.in               |   12 +
 net.c                           |  612 ++++++++++++++
 net.h                           |   56 ++
 read.c                          |  739 ++++++++++++++++
 read.h                          |  129 +++
 sds.c                           | 1289 ++++++++++++++++++++++++++++
 sds.h                           |  278 +++++++
 sdsalloc.h                      |   44 +
 sockcompat.c                    |  248 ++++++
 sockcompat.h                    |   92 ++
 ssl.c                           |  526 ++++++++++++
 test.c                          | 1387 +++++++++++++++++++++++++++++++
 test.sh                         |   78 ++
 win32.h                         |   56 ++
 58 files changed, 12772 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 .travis.yml
 create mode 100644 CHANGELOG.md
 create mode 100644 CMakeLists.txt
 create mode 100644 COPYING
 create mode 100644 Makefile
 create mode 100644 README.md
 create mode 100644 adapters/ae.h
 create mode 100644 adapters/glib.h
 create mode 100644 adapters/ivykis.h
 create mode 100644 adapters/libev.h
 create mode 100644 adapters/libevent.h
 create mode 100644 adapters/libuv.h
 create mode 100644 adapters/macosx.h
 create mode 100644 adapters/qt.h
 create mode 100644 alloc.c
 create mode 100644 alloc.h
 create mode 100644 appveyor.yml
 create mode 100644 async.c
 create mode 100644 async.h
 create mode 100644 async_private.h
 create mode 100644 dict.c
 create mode 100644 dict.h
 create mode 100644 examples/CMakeLists.txt
 create mode 100644 examples/example-ae.c
 create mode 100644 examples/example-glib.c
 create mode 100644 examples/example-ivykis.c
 create mode 100644 examples/example-libev.c
 create mode 100644 examples/example-libevent-ssl.c
 create mode 100644 examples/example-libevent.c
 create mode 100644 examples/example-libuv.c
 create mode 100644 examples/example-macosx.c
 create mode 100644 examples/example-push.c
 create mode 100644 examples/example-qt.cpp
 create mode 100644 examples/example-qt.h
 create mode 100644 examples/example-ssl.c
 create mode 100644 examples/example.c
 create mode 100644 fmacros.h
 create mode 100644 hiredis-config.cmake.in
 create mode 100644 hiredis.c
 create mode 100644 hiredis.h
 create mode 100644 hiredis.pc.in
 create mode 100644 hiredis_ssl-config.cmake.in
 create mode 100644 hiredis_ssl.h
 create mode 100644 hiredis_ssl.pc.in
 create mode 100644 net.c
 create mode 100644 net.h
 create mode 100644 read.c
 create mode 100644 read.h
 create mode 100644 sds.c
 create mode 100644 sds.h
 create mode 100644 sdsalloc.h
 create mode 100644 sockcompat.c
 create mode 100644 sockcompat.h
 create mode 100644 ssl.c
 create mode 100644 test.c
 create mode 100755 test.sh
 create mode 100644 win32.h

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 000000000..056959ffe
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,9 @@
+/hiredis-test
+/examples/hiredis-example*
+/*.o
+/*.so
+/*.dylib
+/*.a
+/*.pc
+*.dSYM
+tags
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 000000000..f9a9460ff
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,131 @@
+language: c
+compiler:
+  - gcc
+  - clang
+
+os:
+  - linux
+  - osx
+
+dist: bionic
+
+branches:
+  only:
+    - staging
+    - trying
+    - master
+    - /^release\/.*$/
+
+install:
+    - if [ "$BITS" == "64" ]; then
+        wget https://github.com/redis/redis/archive/6.0.6.tar.gz;
+        tar -xzvf 6.0.6.tar.gz;
+        pushd redis-6.0.6 && BUILD_TLS=yes make && export PATH=$PWD/src:$PATH && popd;
+      fi
+
+before_script:
+    - if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+        curl -O https://distfiles.macports.org/MacPorts/MacPorts-2.6.2-10.13-HighSierra.pkg;
+        sudo installer -pkg MacPorts-2.6.2-10.13-HighSierra.pkg -target /;
+        export PATH=$PATH:/opt/local/bin && sudo port -v selfupdate;
+        sudo port -N install openssl redis;
+      fi;
+
+addons:
+  apt:
+    sources:
+    - sourceline: 'ppa:chris-lea/redis-server'
+    packages:
+    - libc6-dbg
+    - libc6-dev
+    - libc6:i386
+    - libc6-dev-i386
+    - libc6-dbg:i386
+    - gcc-multilib
+    - g++-multilib
+    - libssl-dev
+    - libssl-dev:i386
+    - valgrind
+    - redis
+
+env:
+  - BITS="32"
+  - BITS="64"
+
+script:
+  - EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON";
+    if [ "$BITS" == "64" ]; then
+      EXTRA_CMAKE_OPTS="$EXTRA_CMAKE_OPTS -DENABLE_SSL_TESTS:BOOL=ON";
+    fi;
+    if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+      if [ "$BITS" == "32" ]; then
+        CFLAGS="-m32 -Werror";
+        CXXFLAGS="-m32 -Werror";
+        LDFLAGS="-m32";
+        EXTRA_CMAKE_OPTS=;
+      else
+        CFLAGS="-Werror";
+        CXXFLAGS="-Werror";
+      fi;
+    else
+      TEST_PREFIX="valgrind --track-origins=yes --leak-check=full";
+      if [ "$BITS" == "32" ]; then
+        CFLAGS="-m32 -Werror";
+        CXXFLAGS="-m32 -Werror";
+        LDFLAGS="-m32";
+        EXTRA_CMAKE_OPTS=;
+      else
+        CFLAGS="-Werror";
+        CXXFLAGS="-Werror";
+      fi;
+    fi;
+    export CFLAGS CXXFLAGS LDFLAGS TEST_PREFIX EXTRA_CMAKE_OPTS
+  - make && make clean;
+    if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+      if [ "$BITS" == "64" ]; then
+        OPENSSL_PREFIX="$(ls -d /usr/local/Cellar/openssl@1.1/*)" USE_SSL=1 make;
+      fi;
+    else
+      USE_SSL=1 make;
+    fi;
+  - mkdir build/ && cd build/
+  - cmake .. ${EXTRA_CMAKE_OPTS}
+  - make VERBOSE=1
+  - if [ "$BITS" == "64" ]; then
+      TEST_SSL=1 SKIPS_AS_FAILS=1 ctest -V;
+    else
+      SKIPS_AS_FAILS=1 ctest -V;
+    fi;
+
+jobs:
+  include:
+    # Windows MinGW cross compile on Linux
+    - os: linux
+      dist: xenial
+      compiler: mingw
+      addons:
+        apt:
+          packages:
+            - ninja-build
+            - gcc-mingw-w64-x86-64
+            - g++-mingw-w64-x86-64
+      script:
+        - mkdir build && cd build
+        - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_WITH_INSTALL_RPATH=on
+        - ninja -v
+
+    # Windows MSVC 2017
+    - os: windows
+      compiler: msvc
+      env:
+        - MATRIX_EVAL="CC=cl.exe && CXX=cl.exe"
+      before_install:
+        - eval "${MATRIX_EVAL}"
+      install:
+        - choco install ninja
+        - choco install -y memurai-developer
+      script:
+        - mkdir build && cd build
+        - cmd.exe //C 'C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat' amd64 '&&'
+          cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DENABLE_EXAMPLES=ON '&&' ninja -v
+        - ./hiredis-test.exe
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 000000000..271f1fcf3
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,345 @@
+## [1.0.0](https://github.com/redis/hiredis/tree/v1.0.0) - (2020-08-03)
+
+Announcing Hiredis v1.0.0, which adds support for RESP3, SSL connections, allocator injection, and better Windows support! :tada:
+
+_A big thanks to everyone who helped with this release.  The following list includes everyone who contributed at least five lines, sorted by lines contributed._ :sparkling_heart:
+
+[Michael Grunder](https://github.com/michael-grunder), [Yossi Gottlieb](https://github.com/yossigo),
+[Mark Nunberg](https://github.com/mnunberg), [Marcus Geelnard](https://github.com/mbitsnbites),
+[Justin Brewer](https://github.com/justinbrewer), [Valentino Geron](https://github.com/valentinogeron),
+[Minun Dragonation](https://github.com/dragonation), [Omri Steiner](https://github.com/OmriSteiner),
+[Sangmoon Yi](https://github.com/jman-krafton), [Jinjiazh](https://github.com/jinjiazhang),
+[Odin Hultgren Van Der Horst](https://github.com/Miniwoffer), [Muhammad Zahalqa](https://github.com/tryfinally),
+[Nick Rivera](https://github.com/heronr), [Qi Yang](https://github.com/movebean),
+[kevin1018](https://github.com/kevin1018)
+
+[Full Changelog](https://github.com/redis/hiredis/compare/v0.14.1...v1.0.0)
+
+**BREAKING CHANGES**:
+
+* `redisOptions` now has two timeout fields.  One for connecting, and one for commands.  If you're presently using `options->timeout` you will need to change it to use `options->connect_timeout`. (See [example](https://github.com/redis/hiredis/commit/38b5ae543f5c99eb4ccabbe277770fc6bc81226f#diff-86ba39d37aa829c8c82624cce4f049fbL36))
+
+* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now protocol errors. This is consistent
+  with the RESP specification. On 32-bit platforms, the upper bound is lowered to `SIZE_MAX`.
+
+* `redisReplyObjectFunctions.createArray` now takes `size_t` for its length parameter.
+
+**New features:**
+- Support for RESP3
+  [\#697](https://github.com/redis/hiredis/pull/697),
+  [\#805](https://github.com/redis/hiredis/pull/805),
+  [\#819](https://github.com/redis/hiredis/pull/819),
+  [\#841](https://github.com/redis/hiredis/pull/841)
+  ([Yossi Gottlieb](https://github.com/yossigo), [Michael Grunder](https://github.com/michael-grunder))
+- Support for SSL connections
+  [\#645](https://github.com/redis/hiredis/pull/645),
+  [\#699](https://github.com/redis/hiredis/pull/699),
+  [\#702](https://github.com/redis/hiredis/pull/702),
+  [\#708](https://github.com/redis/hiredis/pull/708),
+  [\#711](https://github.com/redis/hiredis/pull/711),
+  [\#821](https://github.com/redis/hiredis/pull/821),
+  [more](https://github.com/redis/hiredis/pulls?q=is%3Apr+is%3Amerged+SSL)
+  ([Mark Nunberg](https://github.com/mnunberg), [Yossi Gottlieb](https://github.com/yossigo))
+- Run-time allocator injection
+  [\#800](https://github.com/redis/hiredis/pull/800)
+  ([Michael Grunder](https://github.com/michael-grunder))
+- Improved Windows support (including MinGW and Windows CI)
+  [\#652](https://github.com/redis/hiredis/pull/652),
+  [\#663](https://github.com/redis/hiredis/pull/663)
+  ([Marcus Geelnard](https://www.bitsnbites.eu/author/m/))
+- Adds support for distinct connect and command timeouts
+  [\#839](https://github.com/redis/hiredis/pull/839),
+  [\#829](https://github.com/redis/hiredis/pull/829)
+  ([Valentino Geron](https://github.com/valentinogeron))
+- Add generic pointer and destructor to `redisContext` that users can use for context.
+  [\#855](https://github.com/redis/hiredis/pull/855)
+  ([Michael Grunder](https://github.com/michael-grunder))
+
+**Closed issues (that involved code changes):**
+
+- Makefile does not install TLS libraries  [\#809](https://github.com/redis/hiredis/issues/809)
+- redisConnectWithOptions should not set command timeout [\#722](https://github.com/redis/hiredis/issues/722), [\#829](https://github.com/redis/hiredis/pull/829) ([valentinogeron](https://github.com/valentinogeron))
+- Fix integer overflow in `sdsrange` [\#827](https://github.com/redis/hiredis/issues/827)
+- INFO & CLUSTER commands failed when using RESP3 [\#802](https://github.com/redis/hiredis/issues/802)
+- Windows compatibility patches [\#687](https://github.com/redis/hiredis/issues/687), [\#838](https://github.com/redis/hiredis/issues/838), [\#842](https://github.com/redis/hiredis/issues/842)
+- RESP3 PUSH messages incorrectly use pending callback [\#825](https://github.com/redis/hiredis/issues/825)
+- Asynchronous PSUBSCRIBE command fails when using RESP3 [\#815](https://github.com/redis/hiredis/issues/815)
+- New SSL API [\#804](https://github.com/redis/hiredis/issues/804), [\#813](https://github.com/redis/hiredis/issues/813)
+- Hard-coded limit of nested reply depth [\#794](https://github.com/redis/hiredis/issues/794)
+- Fix TCP_NODELAY in Windows/OSX [\#679](https://github.com/redis/hiredis/issues/679), [\#690](https://github.com/redis/hiredis/issues/690), [\#779](https://github.com/redis/hiredis/issues/779), [\#785](https://github.com/redis/hiredis/issues/785),
+- Added timers to libev adapter.  [\#778](https://github.com/redis/hiredis/issues/778), [\#795](https://github.com/redis/hiredis/pull/795)
+- Initialization discards const qualifier [\#777](https://github.com/redis/hiredis/issues/777)
+- \[BUG\]\[MinGW64\] Error setting socket timeout  [\#775](https://github.com/redis/hiredis/issues/775)
+- undefined reference to hi_malloc [\#769](https://github.com/redis/hiredis/issues/769)
+- hiredis pkg-config file incorrectly ignores multiarch libdir spec'n [\#767](https://github.com/redis/hiredis/issues/767)
+- Don't use -G to build shared object on Solaris [\#757](https://github.com/redis/hiredis/issues/757)
+- error when make USE\_SSL=1 [\#748](https://github.com/redis/hiredis/issues/748)
+- Allow to change SSL Mode [\#646](https://github.com/redis/hiredis/issues/646)
+- hiredis/adapters/libevent.h memleak [\#618](https://github.com/redis/hiredis/issues/618)
+- redisLibuvPoll crash when server closes the connetion [\#545](https://github.com/redis/hiredis/issues/545)
+- about redisAsyncDisconnect question [\#518](https://github.com/redis/hiredis/issues/518)
+- hiredis adapters libuv error for help [\#508](https://github.com/redis/hiredis/issues/508)
+- API/ABI changes analysis [\#506](https://github.com/redis/hiredis/issues/506)
+- Memory leak patch in Redis [\#502](https://github.com/redis/hiredis/issues/502)
+- Remove the depth limitation [\#421](https://github.com/redis/hiredis/issues/421)
+
+**Merged pull requests:**
+
+- Move SSL management to a distinct private pointer [\#855](https://github.com/redis/hiredis/pull/855) ([michael-grunder](https://github.com/michael-grunder))
+- Move include to sockcompat.h to maintain style [\#850](https://github.com/redis/hiredis/pull/850) ([michael-grunder](https://github.com/michael-grunder))
+- Remove erroneous tag and add license to push example [\#849](https://github.com/redis/hiredis/pull/849) ([michael-grunder](https://github.com/michael-grunder))
+- fix windows compiling with mingw [\#848](https://github.com/redis/hiredis/pull/848) ([rmalizia44](https://github.com/rmalizia44))
+- Some Windows quality of life improvements. [\#846](https://github.com/redis/hiredis/pull/846) ([michael-grunder](https://github.com/michael-grunder))
+- Use \_WIN32 define instead of WIN32 [\#845](https://github.com/redis/hiredis/pull/845) ([michael-grunder](https://github.com/michael-grunder))
+- Non Linux CI fixes [\#844](https://github.com/redis/hiredis/pull/844) ([michael-grunder](https://github.com/michael-grunder))
+- Resp3 oob push support [\#841](https://github.com/redis/hiredis/pull/841) ([michael-grunder](https://github.com/michael-grunder))
+- fix \#785: defer TCP\_NODELAY in async tcp connections [\#836](https://github.com/redis/hiredis/pull/836) ([OmriSteiner](https://github.com/OmriSteiner))
+- sdsrange overflow fix [\#830](https://github.com/redis/hiredis/pull/830) ([michael-grunder](https://github.com/michael-grunder))
+- Use explicit pointer casting for c++ compatibility [\#826](https://github.com/redis/hiredis/pull/826) ([aureus1](https://github.com/aureus1))
+- Document allocator injection and completeness fix in test.c [\#824](https://github.com/redis/hiredis/pull/824) ([michael-grunder](https://github.com/michael-grunder))
+- Use unique names for allocator struct members [\#823](https://github.com/redis/hiredis/pull/823) ([michael-grunder](https://github.com/michael-grunder))
+- New SSL API to replace redisSecureConnection\(\). [\#821](https://github.com/redis/hiredis/pull/821) ([yossigo](https://github.com/yossigo))
+- Add logic to handle RESP3 push messages [\#819](https://github.com/redis/hiredis/pull/819) ([michael-grunder](https://github.com/michael-grunder))
+- Use standrad isxdigit instead of custom helper function. [\#814](https://github.com/redis/hiredis/pull/814) ([tryfinally](https://github.com/tryfinally))
+- Fix missing SSL build/install options. [\#812](https://github.com/redis/hiredis/pull/812) ([yossigo](https://github.com/yossigo))
+- Add link to ABI tracker [\#808](https://github.com/redis/hiredis/pull/808) ([michael-grunder](https://github.com/michael-grunder))
+- Resp3 verbatim string support [\#805](https://github.com/redis/hiredis/pull/805) ([michael-grunder](https://github.com/michael-grunder))
+- Allow users to replace allocator and handle OOM everywhere. [\#800](https://github.com/redis/hiredis/pull/800) ([michael-grunder](https://github.com/michael-grunder))
+- Remove nested depth limitation. [\#797](https://github.com/redis/hiredis/pull/797) ([michael-grunder](https://github.com/michael-grunder))
+- Attempt to fix compilation on Solaris [\#796](https://github.com/redis/hiredis/pull/796) ([michael-grunder](https://github.com/michael-grunder))
+- Support timeouts in libev adapater [\#795](https://github.com/redis/hiredis/pull/795) ([michael-grunder](https://github.com/michael-grunder))
+- Fix pkgconfig when installing to a custom lib dir [\#793](https://github.com/redis/hiredis/pull/793) ([michael-grunder](https://github.com/michael-grunder))
+- Fix USE\_SSL=1 make/cmake on OSX and CMake tests [\#789](https://github.com/redis/hiredis/pull/789) ([michael-grunder](https://github.com/michael-grunder))
+- Use correct libuv call on Windows [\#784](https://github.com/redis/hiredis/pull/784) ([michael-grunder](https://github.com/michael-grunder))
+- Added CMake package config and fixed hiredis\_ssl on Windows [\#783](https://github.com/redis/hiredis/pull/783) ([michael-grunder](https://github.com/michael-grunder))
+- CMake: Set hiredis\_ssl shared object version. [\#780](https://github.com/redis/hiredis/pull/780) ([yossigo](https://github.com/yossigo))
+- Win32 tests and timeout fix [\#776](https://github.com/redis/hiredis/pull/776) ([michael-grunder](https://github.com/michael-grunder))
+- Provides an optional cleanup callback for async data. [\#768](https://github.com/redis/hiredis/pull/768) ([heronr](https://github.com/heronr))
+- Housekeeping fixes [\#764](https://github.com/redis/hiredis/pull/764) ([michael-grunder](https://github.com/michael-grunder))
+- install alloc.h [\#756](https://github.com/redis/hiredis/pull/756) ([ch1aki](https://github.com/ch1aki))
+- fix spelling mistakes [\#746](https://github.com/redis/hiredis/pull/746) ([ShooterIT](https://github.com/ShooterIT))
+- Free the reply in redisGetReply when passed NULL [\#741](https://github.com/redis/hiredis/pull/741) ([michael-grunder](https://github.com/michael-grunder))
+- Fix dead code in sslLogCallback relating to should\_log variable. [\#737](https://github.com/redis/hiredis/pull/737) ([natoscott](https://github.com/natoscott))
+- Fix typo in dict.c. [\#731](https://github.com/redis/hiredis/pull/731) ([Kevin-Xi](https://github.com/Kevin-Xi))
+- Adding an option to DISABLE\_TESTS [\#727](https://github.com/redis/hiredis/pull/727) ([pbotros](https://github.com/pbotros))
+- Update README with SSL support. [\#720](https://github.com/redis/hiredis/pull/720) ([yossigo](https://github.com/yossigo))
+- Fixes leaks in unit tests [\#715](https://github.com/redis/hiredis/pull/715) ([michael-grunder](https://github.com/michael-grunder))
+- SSL Tests [\#711](https://github.com/redis/hiredis/pull/711) ([yossigo](https://github.com/yossigo))
+- SSL Reorganization [\#708](https://github.com/redis/hiredis/pull/708) ([yossigo](https://github.com/yossigo))
+- Fix MSVC build. [\#706](https://github.com/redis/hiredis/pull/706) ([yossigo](https://github.com/yossigo))
+- SSL: Properly report SSL\_connect\(\) errors. [\#702](https://github.com/redis/hiredis/pull/702) ([yossigo](https://github.com/yossigo))
+- Silent SSL trace to stdout by default. [\#699](https://github.com/redis/hiredis/pull/699) ([yossigo](https://github.com/yossigo))
+- Port RESP3 support from Redis. [\#697](https://github.com/redis/hiredis/pull/697) ([yossigo](https://github.com/yossigo))
+- Removed whitespace before newline [\#691](https://github.com/redis/hiredis/pull/691) ([Miniwoffer](https://github.com/Miniwoffer))
+- Add install adapters header files [\#688](https://github.com/redis/hiredis/pull/688) ([kevin1018](https://github.com/kevin1018))
+- Remove unnecessary null check before free [\#684](https://github.com/redis/hiredis/pull/684) ([qlyoung](https://github.com/qlyoung))
+- redisReaderGetReply leak memory [\#671](https://github.com/redis/hiredis/pull/671) ([movebean](https://github.com/movebean))
+- fix timeout code in windows [\#670](https://github.com/redis/hiredis/pull/670) ([jman-krafton](https://github.com/jman-krafton))
+- test: fix errstr matching for musl libc [\#665](https://github.com/redis/hiredis/pull/665) ([ghost](https://github.com/ghost))
+- Windows: MinGW fixes and Windows Travis builders [\#663](https://github.com/redis/hiredis/pull/663) ([mbitsnbites](https://github.com/mbitsnbites))
+- The setsockopt and getsockopt API diffs from BSD socket and WSA one [\#662](https://github.com/redis/hiredis/pull/662) ([dragonation](https://github.com/dragonation))
+- Fix Compile Error On Windows \(Visual Studio\) [\#658](https://github.com/redis/hiredis/pull/658) ([jinjiazhang](https://github.com/jinjiazhang))
+- Fix NXDOMAIN test case [\#653](https://github.com/redis/hiredis/pull/653) ([michael-grunder](https://github.com/michael-grunder))
+- Add MinGW support [\#652](https://github.com/redis/hiredis/pull/652) ([mbitsnbites](https://github.com/mbitsnbites))
+- SSL Support [\#645](https://github.com/redis/hiredis/pull/645) ([mnunberg](https://github.com/mnunberg))
+- Fix Invalid argument after redisAsyncConnectUnix [\#644](https://github.com/redis/hiredis/pull/644) ([codehz](https://github.com/codehz))
+- Makefile: use predefined AR [\#632](https://github.com/redis/hiredis/pull/632) ([Mic92](https://github.com/Mic92))
+- FreeBSD  build fix [\#628](https://github.com/redis/hiredis/pull/628) ([devnexen](https://github.com/devnexen))
+- Fix errors not propagating properly with libuv.h. [\#624](https://github.com/redis/hiredis/pull/624) ([yossigo](https://github.com/yossigo))
+- Update README.md [\#621](https://github.com/redis/hiredis/pull/621) ([Crunsher](https://github.com/Crunsher))
+- Fix redisBufferRead documentation [\#620](https://github.com/redis/hiredis/pull/620) ([hacst](https://github.com/hacst))
+- Add CPPFLAGS to REAL\_CFLAGS [\#614](https://github.com/redis/hiredis/pull/614) ([thomaslee](https://github.com/thomaslee))
+- Update createArray to take size\_t [\#597](https://github.com/redis/hiredis/pull/597) ([justinbrewer](https://github.com/justinbrewer))
+- fix common realloc mistake and add null check more [\#580](https://github.com/redis/hiredis/pull/580) ([charsyam](https://github.com/charsyam))
+- Proper error reporting for connect failures [\#578](https://github.com/redis/hiredis/pull/578) ([mnunberg](https://github.com/mnunberg))
+
+\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*
+
+## [1.0.0-rc1](https://github.com/redis/hiredis/tree/v1.0.0-rc1) - (2020-07-29)
+
+_Note:  There were no changes to code between v1.0.0-rc1 and v1.0.0 so see v1.0.0 for changelog_
+
+### 0.14.1 (2020-03-13)
+
+* Adds safe allocation wrappers (CVE-2020-7105, #747, #752) (Michael Grunder)
+
+### 0.14.0 (2018-09-25)
+**BREAKING CHANGES**:
+
+* Change `redisReply.len` to `size_t`, as it denotes the the size of a string
+
+  User code should compare this to `size_t` values as well.
+  If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before.
+
+* Make string2ll static to fix conflict with Redis (Tom Lee [c3188b])
+* Use -dynamiclib instead of -shared for OSX (Ryan Schmidt [a65537])
+* Use string2ll from Redis w/added tests (Michael Grunder [7bef04, 60f622])
+* Makefile - OSX compilation fixes (Ryan Schmidt [881fcb, 0e9af8])
+* Remove redundant NULL checks (Justin Brewer [54acc8, 58e6b8])
+* Fix bulk and multi-bulk length truncation (Justin Brewer [109197])
+* Fix SIGSEGV in OpenBSD by checking for NULL before calling freeaddrinfo (Justin Brewer [546d94])
+* Several POSIX compatibility fixes (Justin Brewer [bbeab8, 49bbaa, d1c1b6])
+* Makefile - Compatibility fixes (Dimitri Vorobiev [3238cf, 12a9d1])
+* Makefile - Fix make install on FreeBSD (Zach Shipko [a2ef2b])
+* Makefile - don't assume $(INSTALL) is cp (Igor Gnatenko [725a96])
+* Separate side-effect causing function from assert and small cleanup (amallia [b46413, 3c3234])
+* Don't send negative values to `__redisAsyncCommand` (Frederik Deweerdt [706129])
+* Fix leak if setsockopt fails (Frederik Deweerdt [e21c9c])
+* Fix libevent leak (zfz [515228])
+* Clean up GCC warning (Ichito Nagata [2ec774])
+* Keep track of errno in `__redisSetErrorFromErrno()` as snprintf may use it (Jin Qing [25cd88])
+* Solaris compilation fix (Donald Whyte [41b07d])
+* Reorder linker arguments when building examples (Tustfarm-heart [06eedd])
+* Keep track of subscriptions in case of rapid subscribe/unsubscribe (Hyungjin Kim [073dc8, be76c5, d46999])
+* libuv use after free fix (Paul Scott [cbb956])
+* Properly close socket fd on reconnect attempt (WSL [64d1ec])
+* Skip valgrind in OSX tests (Jan-Erik Rediger [9deb78])
+* Various updates for Travis testing OSX (Ted Nyman [fa3774, 16a459, bc0ea5])
+* Update libevent (Chris Xin [386802])
+* Change sds.h for building in C++ projects (Ali Volkan ATLI [f5b32e])
+* Use proper format specifier in redisFormatSdsCommandArgv (Paulino Huerta, Jan-Erik Rediger [360a06, 8655a6])
+* Better handling of NULL reply in example code (Jan-Erik Rediger [1b8ed3])
+* Prevent overflow when formatting an error (Jan-Erik Rediger [0335cb])
+* Compatibility fix for strerror_r (Tom Lee [bb1747])
+* Properly detect integer parse/overflow errors (Justin Brewer [93421f])
+* Adds CI for Windows and cygwin fixes (owent, [6c53d6, 6c3e40])
+* Catch a buffer overflow when formatting the error message
+* Import latest upstream sds. This breaks applications that are linked against the old hiredis v0.13
+* Fix warnings, when compiled with -Wshadow
+* Make hiredis compile in Cygwin on Windows, now CI-tested
+* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now
+  protocol errors. This is consistent with the RESP specification. On 32-bit
+  platforms, the upper bound is lowered to `SIZE_MAX`.
+
+* Remove backwards compatibility macro's
+
+This removes the following old function aliases, use the new name now:
+
+| Old                         | New                    |
+| --------------------------- | ---------------------- |
+| redisReplyReaderCreate      | redisReaderCreate      |
+| redisReplyReaderCreate      | redisReaderCreate      |
+| redisReplyReaderFree        | redisReaderFree        |
+| redisReplyReaderFeed        | redisReaderFeed        |
+| redisReplyReaderGetReply    | redisReaderGetReply    |
+| redisReplyReaderSetPrivdata | redisReaderSetPrivdata |
+| redisReplyReaderGetObject   | redisReaderGetObject   |
+| redisReplyReaderGetError    | redisReaderGetError    |
+
+* The `DEBUG` variable in the Makefile was renamed to `DEBUG_FLAGS`
+
+Previously it broke some builds for people that had `DEBUG` set to some arbitrary value,
+due to debugging other software.
+By renaming we avoid unintentional name clashes.
+
+Simply rename `DEBUG` to `DEBUG_FLAGS` in your environment to make it working again.
+
+### 0.13.3 (2015-09-16)
+
+* Revert "Clear `REDIS_CONNECTED` flag when connection is closed".
+* Make tests pass on FreeBSD (Thanks, Giacomo Olgeni)
+
+
+If the `REDIS_CONNECTED` flag is cleared,
+the async onDisconnect callback function will never be called.
+This causes problems as the disconnect is never reported back to the user.
+
+### 0.13.2 (2015-08-25)
+
+* Prevent crash on pending replies in async code (Thanks, @switch-st)
+* Clear `REDIS_CONNECTED` flag when connection is closed (Thanks, Jerry Jacobs)
+* Add MacOS X addapter (Thanks, @dizzus)
+* Add Qt adapter (Thanks, Pietro Cerutti)
+* Add Ivykis adapter (Thanks, Gergely Nagy)
+
+All adapters are provided as is and are only tested where possible.
+
+### 0.13.1 (2015-05-03)
+
+This is a bug fix release.
+The new `reconnect` method introduced new struct members, which clashed with pre-defined names in pre-C99 code.
+Another commit forced C99 compilation just to make it work, but of course this is not desirable for outside projects.
+Other non-C99 code can now use hiredis as usual again.
+Sorry for the inconvenience.
+
+* Fix memory leak in async reply handling (Salvatore Sanfilippo)
+* Rename struct member to avoid name clash with pre-c99 code (Alex Balashov, ncopa)
+
+### 0.13.0 (2015-04-16)
+
+This release adds a minimal Windows compatibility layer.
+The parser, standalone since v0.12.0, can now be compiled on Windows
+(and thus used in other client libraries as well)
+
+* Windows compatibility layer for parser code (tzickel)
+* Properly escape data printed to PKGCONF file (Dan Skorupski)
+* Fix tests when assert() undefined (Keith Bennett, Matt Stancliff)
+* Implement a reconnect method for the client context, this changes the structure of `redisContext` (Aaron Bedra)
+
+### 0.12.1 (2015-01-26)
+
+* Fix `make install`: DESTDIR support, install all required files, install PKGCONF in proper location
+* Fix `make test` as 32 bit build on 64 bit platform
+
+### 0.12.0 (2015-01-22)
+
+* Add optional KeepAlive support
+
+* Try again on EINTR errors
+
+* Add libuv adapter
+
+* Add IPv6 support
+
+* Remove possibility of multiple close on same fd
+
+* Add ability to bind source address on connect
+
+* Add redisConnectFd() and redisFreeKeepFd()
+
+* Fix getaddrinfo() memory leak
+
+* Free string if it is unused (fixes memory leak)
+
+* Improve redisAppendCommandArgv performance 2.5x
+
+* Add support for SO_REUSEADDR
+
+* Fix redisvFormatCommand format parsing
+
+* Add GLib 2.0 adapter
+
+* Refactor reading code into read.c
+
+* Fix errno error buffers to not clobber errors
+
+* Generate pkgconf during build
+
+* Silence _BSD_SOURCE warnings
+
+* Improve digit counting for multibulk creation
+
+
+### 0.11.0
+
+* Increase the maximum multi-bulk reply depth to 7.
+
+* Increase the read buffer size from 2k to 16k.
+
+* Use poll(2) instead of select(2) to support large fds (>= 1024).
+
+### 0.10.1
+
+* Makefile overhaul. Important to check out if you override one or more
+  variables using environment variables or via arguments to the "make" tool.
+
+* Issue #45: Fix potential memory leak for a multi bulk reply with 0 elements
+  being created by the default reply object functions.
+
+* Issue #43: Don't crash in an asynchronous context when Redis returns an error
+  reply after the connection has been made (this happens when the maximum
+  number of connections is reached).
+
+### 0.10.0
+
+* See commit log.
diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 000000000..f86c9b70b
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,165 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 3.4.0)
+INCLUDE(GNUInstallDirs)
+PROJECT(hiredis)
+
+OPTION(ENABLE_SSL "Build hiredis_ssl for SSL support" OFF)
+OPTION(DISABLE_TESTS "If tests should be compiled or not" OFF)
+OPTION(ENABLE_SSL_TESTS, "Should we test SSL connections" OFF)
+
+MACRO(getVersionBit name)
+  SET(VERSION_REGEX "^#define ${name} (.+)$")
+  FILE(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/hiredis.h"
+    VERSION_BIT REGEX ${VERSION_REGEX})
+  STRING(REGEX REPLACE ${VERSION_REGEX} "\\1" ${name} "${VERSION_BIT}")
+ENDMACRO(getVersionBit)
+
+getVersionBit(HIREDIS_MAJOR)
+getVersionBit(HIREDIS_MINOR)
+getVersionBit(HIREDIS_PATCH)
+getVersionBit(HIREDIS_SONAME)
+SET(VERSION "${HIREDIS_MAJOR}.${HIREDIS_MINOR}.${HIREDIS_PATCH}")
+MESSAGE("Detected version: ${VERSION}")
+
+PROJECT(hiredis VERSION "${VERSION}")
+
+SET(ENABLE_EXAMPLES OFF CACHE BOOL "Enable building hiredis examples")
+
+SET(hiredis_sources
+    alloc.c
+    async.c
+    dict.c
+    hiredis.c
+    net.c
+    read.c
+    sds.c
+    sockcompat.c)
+
+SET(hiredis_sources ${hiredis_sources})
+
+IF(WIN32)
+    ADD_COMPILE_DEFINITIONS(_CRT_SECURE_NO_WARNINGS WIN32_LEAN_AND_MEAN)
+ENDIF()
+
+ADD_LIBRARY(hiredis SHARED ${hiredis_sources})
+
+SET_TARGET_PROPERTIES(hiredis
+    PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE
+    VERSION "${HIREDIS_SONAME}")
+IF(WIN32 OR MINGW)
+    TARGET_LINK_LIBRARIES(hiredis PRIVATE ws2_32)
+ENDIF()
+
+TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC $<INSTALL_INTERFACE:.> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
+
+CONFIGURE_FILE(hiredis.pc.in hiredis.pc @ONLY)
+
+INSTALL(TARGETS hiredis
+    EXPORT hiredis-targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
+
+INSTALL(FILES hiredis.h read.h sds.h async.h alloc.h
+    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis)
+    
+INSTALL(DIRECTORY adapters
+    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis)
+    
+INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis.pc
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
+
+export(EXPORT hiredis-targets
+    FILE "${CMAKE_CURRENT_BINARY_DIR}/hiredis-targets.cmake"
+    NAMESPACE hiredis::)
+
+SET(CMAKE_CONF_INSTALL_DIR share/hiredis)
+SET(INCLUDE_INSTALL_DIR include)
+include(CMakePackageConfigHelpers)
+configure_package_config_file(hiredis-config.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/hiredis-config.cmake
+                              INSTALL_DESTINATION ${CMAKE_CONF_INSTALL_DIR}
+                              PATH_VARS INCLUDE_INSTALL_DIR)
+
+INSTALL(EXPORT hiredis-targets
+        FILE hiredis-targets.cmake
+        NAMESPACE hiredis::
+        DESTINATION ${CMAKE_CONF_INSTALL_DIR})
+
+INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis-config.cmake
+        DESTINATION ${CMAKE_CONF_INSTALL_DIR})
+
+
+IF(ENABLE_SSL)
+    IF (NOT OPENSSL_ROOT_DIR)
+        IF (APPLE)
+            SET(OPENSSL_ROOT_DIR "/usr/local/opt/openssl")
+        ENDIF()
+    ENDIF()
+    FIND_PACKAGE(OpenSSL REQUIRED)
+    SET(hiredis_ssl_sources 
+        ssl.c)
+    ADD_LIBRARY(hiredis_ssl SHARED
+            ${hiredis_ssl_sources})
+
+    IF (APPLE)
+        SET_PROPERTY(TARGET hiredis_ssl PROPERTY LINK_FLAGS "-Wl,-undefined -Wl,dynamic_lookup")
+    ENDIF()
+
+    SET_TARGET_PROPERTIES(hiredis_ssl
+        PROPERTIES
+        WINDOWS_EXPORT_ALL_SYMBOLS TRUE
+        VERSION "${HIREDIS_SONAME}")
+
+    TARGET_INCLUDE_DIRECTORIES(hiredis_ssl PRIVATE "${OPENSSL_INCLUDE_DIR}")
+    TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE ${OPENSSL_LIBRARIES})
+    IF (WIN32 OR MINGW)
+        TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE hiredis)
+    ENDIF()
+    CONFIGURE_FILE(hiredis_ssl.pc.in hiredis_ssl.pc @ONLY)
+
+    INSTALL(TARGETS hiredis_ssl
+        EXPORT hiredis_ssl-targets
+        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
+
+    INSTALL(FILES hiredis_ssl.h
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis)
+    
+    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl.pc
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
+
+    export(EXPORT hiredis_ssl-targets
+           FILE "${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl-targets.cmake"
+           NAMESPACE hiredis::)
+
+    SET(CMAKE_CONF_INSTALL_DIR share/hiredis_ssl)
+    configure_package_config_file(hiredis_ssl-config.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl-config.cmake
+                                  INSTALL_DESTINATION ${CMAKE_CONF_INSTALL_DIR}
+                                  PATH_VARS INCLUDE_INSTALL_DIR)
+
+    INSTALL(EXPORT hiredis_ssl-targets
+        FILE hiredis_ssl-targets.cmake
+        NAMESPACE hiredis::
+        DESTINATION ${CMAKE_CONF_INSTALL_DIR})
+
+    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl-config.cmake
+        DESTINATION ${CMAKE_CONF_INSTALL_DIR})
+ENDIF()
+
+IF(NOT DISABLE_TESTS)
+    ENABLE_TESTING()
+    ADD_EXECUTABLE(hiredis-test test.c)
+    IF(ENABLE_SSL_TESTS)
+        ADD_DEFINITIONS(-DHIREDIS_TEST_SSL=1)
+        TARGET_LINK_LIBRARIES(hiredis-test hiredis hiredis_ssl)
+    ELSE()
+        TARGET_LINK_LIBRARIES(hiredis-test hiredis)
+    ENDIF()
+    ADD_TEST(NAME hiredis-test
+        COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test.sh)
+ENDIF()
+
+# Add examples
+IF(ENABLE_EXAMPLES)
+  ADD_SUBDIRECTORY(examples)
+ENDIF(ENABLE_EXAMPLES)
diff --git a/COPYING b/COPYING
new file mode 100644
index 000000000..a5fc97395
--- /dev/null
+++ b/COPYING
@@ -0,0 +1,29 @@
+Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of Redis nor the names of its contributors may be used
+  to endorse or promote products derived from this software without specific
+  prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/Makefile b/Makefile
new file mode 100644
index 000000000..a8d37a2eb
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,308 @@
+# Hiredis Makefile
+# Copyright (C) 2010-2011 Salvatore Sanfilippo <antirez at gmail dot com>
+# Copyright (C) 2010-2011 Pieter Noordhuis <pcnoordhuis at gmail dot com>
+# This file is released under the BSD license, see the COPYING file
+
+OBJ=alloc.o net.o hiredis.o sds.o async.o read.o sockcompat.o
+SSL_OBJ=ssl.o
+EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib hiredis-example-push
+ifeq ($(USE_SSL),1)
+EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl
+endif
+TESTS=hiredis-test
+LIBNAME=libhiredis
+PKGCONFNAME=hiredis.pc
+SSL_LIBNAME=libhiredis_ssl
+SSL_PKGCONFNAME=hiredis_ssl.pc
+
+HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}')
+HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}')
+HIREDIS_PATCH=$(shell grep HIREDIS_PATCH hiredis.h | awk '{print $$3}')
+HIREDIS_SONAME=$(shell grep HIREDIS_SONAME hiredis.h | awk '{print $$3}')
+
+# Installation related variables and target
+PREFIX?=/usr/local
+INCLUDE_PATH?=include/hiredis
+LIBRARY_PATH?=lib
+PKGCONF_PATH?=pkgconfig
+INSTALL_INCLUDE_PATH= $(DESTDIR)$(PREFIX)/$(INCLUDE_PATH)
+INSTALL_LIBRARY_PATH= $(DESTDIR)$(PREFIX)/$(LIBRARY_PATH)
+INSTALL_PKGCONF_PATH= $(INSTALL_LIBRARY_PATH)/$(PKGCONF_PATH)
+
+# redis-server configuration used for testing
+REDIS_PORT=56379
+REDIS_SERVER=redis-server
+define REDIS_TEST_CONFIG
+	daemonize yes
+	pidfile /tmp/hiredis-test-redis.pid
+	port $(REDIS_PORT)
+	bind 127.0.0.1
+	unixsocket /tmp/hiredis-test-redis.sock
+endef
+export REDIS_TEST_CONFIG
+
+# Fallback to gcc when $CC is not in $PATH.
+CC:=$(shell sh -c 'type $${CC%% *} >/dev/null 2>/dev/null && echo $(CC) || echo gcc')
+CXX:=$(shell sh -c 'type $${CXX%% *} >/dev/null 2>/dev/null && echo $(CXX) || echo g++')
+OPTIMIZATION?=-O3
+WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings -Wno-missing-field-initializers
+DEBUG_FLAGS?= -g -ggdb
+REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CPPFLAGS) $(CFLAGS) $(WARNINGS) $(DEBUG_FLAGS)
+REAL_LDFLAGS=$(LDFLAGS)
+
+DYLIBSUFFIX=so
+STLIBSUFFIX=a
+DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME)
+DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR)
+DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX)
+
+DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME)
+STLIBNAME=$(LIBNAME).$(STLIBSUFFIX)
+STLIB_MAKE_CMD=$(AR) rcs
+
+SSL_DYLIB_MINOR_NAME=$(SSL_LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME)
+SSL_DYLIB_MAJOR_NAME=$(SSL_LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR)
+SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX)
+SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX)
+SSL_DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(SSL_DYLIB_MINOR_NAME)
+
+# Platform-specific overrides
+uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
+
+USE_SSL?=0
+
+# This is required for test.c only
+ifeq ($(USE_SSL),1)
+  CFLAGS+=-DHIREDIS_TEST_SSL
+endif
+
+ifeq ($(uname_S),Linux)
+  SSL_LDFLAGS=-lssl -lcrypto
+else
+  OPENSSL_PREFIX?=/usr/local/opt/openssl
+  CFLAGS+=-I$(OPENSSL_PREFIX)/include
+  SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto
+endif
+
+ifeq ($(uname_S),SunOS)
+  IS_SUN_CC=$(shell sh -c '$(CC) -V 2>&1 |egrep -i -c "sun|studio"')
+  ifeq ($(IS_SUN_CC),1)
+    SUN_SHARED_FLAG=-G
+  else
+    SUN_SHARED_FLAG=-shared
+  endif
+  REAL_LDFLAGS+= -ldl -lnsl -lsocket
+  DYLIB_MAKE_CMD=$(CC) $(SUN_SHARED_FLAG) -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS)
+  SSL_DYLIB_MAKE_CMD=$(CC) $(SUN_SHARED_FLAG) -o $(SSL_DYLIBNAME) -h $(SSL_DYLIB_MINOR_NAME) $(LDFLAGS) $(SSL_LDFLAGS)
+endif
+ifeq ($(uname_S),Darwin)
+  DYLIBSUFFIX=dylib
+  DYLIB_MINOR_NAME=$(LIBNAME).$(HIREDIS_SONAME).$(DYLIBSUFFIX)
+  DYLIB_MAKE_CMD=$(CC) -dynamiclib -Wl,-install_name,$(PREFIX)/$(LIBRARY_PATH)/$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS)
+  SSL_DYLIB_MAKE_CMD=$(CC) -dynamiclib -Wl,-install_name,$(PREFIX)/$(LIBRARY_PATH)/$(SSL_DYLIB_MINOR_NAME) -o $(SSL_DYLIBNAME) $(LDFLAGS) $(SSL_LDFLAGS)
+  DYLIB_PLUGIN=-Wl,-undefined -Wl,dynamic_lookup
+endif
+
+all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME)
+ifeq ($(USE_SSL),1)
+all: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME)
+endif
+
+# Deps (use make dep to generate this)
+alloc.o: alloc.c fmacros.h alloc.h
+async.o: async.c fmacros.h alloc.h async.h hiredis.h read.h sds.h net.h dict.c dict.h win32.h async_private.h
+dict.o: dict.c fmacros.h alloc.h dict.h
+hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h alloc.h net.h async.h win32.h
+net.o: net.c fmacros.h net.h hiredis.h read.h sds.h alloc.h sockcompat.h win32.h
+read.o: read.c fmacros.h alloc.h read.h sds.h win32.h
+sds.o: sds.c sds.h sdsalloc.h alloc.h
+sockcompat.o: sockcompat.c sockcompat.h
+ssl.o: ssl.c hiredis.h read.h sds.h alloc.h async.h win32.h async_private.h
+test.o: test.c fmacros.h hiredis.h read.h sds.h alloc.h net.h sockcompat.h win32.h
+
+$(DYLIBNAME): $(OBJ)
+	$(DYLIB_MAKE_CMD) -o $(DYLIBNAME) $(OBJ) $(REAL_LDFLAGS)
+
+$(STLIBNAME): $(OBJ)
+	$(STLIB_MAKE_CMD) $(STLIBNAME) $(OBJ)
+
+$(SSL_DYLIBNAME): $(SSL_OBJ)
+	$(SSL_DYLIB_MAKE_CMD) $(DYLIB_PLUGIN) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(LDFLAGS) $(SSL_LDFLAGS)
+
+$(SSL_STLIBNAME): $(SSL_OBJ)
+	$(STLIB_MAKE_CMD) $(SSL_STLIBNAME) $(SSL_OBJ)
+
+dynamic: $(DYLIBNAME)
+static: $(STLIBNAME)
+ifeq ($(USE_SSL),1)
+dynamic: $(SSL_DYLIBNAME)
+static: $(SSL_STLIBNAME)
+endif
+
+# Binaries:
+hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME)
+	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(REAL_LDFLAGS)
+
+hiredis-example-libevent-ssl: examples/example-libevent-ssl.c adapters/libevent.h $(STLIBNAME) $(SSL_STLIBNAME)
+	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS)
+
+hiredis-example-libev: examples/example-libev.c adapters/libev.h $(STLIBNAME)
+	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -lev $(STLIBNAME) $(REAL_LDFLAGS)
+
+hiredis-example-glib: examples/example-glib.c adapters/glib.h $(STLIBNAME)
+	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(shell pkg-config --cflags --libs glib-2.0) $(STLIBNAME) $(REAL_LDFLAGS)
+
+hiredis-example-ivykis: examples/example-ivykis.c adapters/ivykis.h $(STLIBNAME)
+	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -livykis $(STLIBNAME) $(REAL_LDFLAGS)
+
+hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME)
+	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) $(REAL_LDFLAGS)
+
+hiredis-example-ssl: examples/example-ssl.c $(STLIBNAME) $(SSL_STLIBNAME)
+	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS)
+
+
+ifndef AE_DIR
+hiredis-example-ae:
+	@echo "Please specify AE_DIR (e.g. <redis repository>/src)"
+	@false
+else
+hiredis-example-ae: examples/example-ae.c adapters/ae.h $(STLIBNAME)
+	$(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(AE_DIR) $< $(AE_DIR)/ae.o $(AE_DIR)/zmalloc.o $(AE_DIR)/../deps/jemalloc/lib/libjemalloc.a -pthread $(STLIBNAME)
+endif
+
+ifndef LIBUV_DIR
+hiredis-example-libuv:
+	@echo "Please specify LIBUV_DIR (e.g. ../libuv/)"
+	@false
+else
+hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME)
+	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) $(REAL_LDFLAGS)
+endif
+
+ifeq ($(and $(QT_MOC),$(QT_INCLUDE_DIR),$(QT_LIBRARY_DIR)),)
+hiredis-example-qt:
+	@echo "Please specify QT_MOC, QT_INCLUDE_DIR AND QT_LIBRARY_DIR"
+	@false
+else
+hiredis-example-qt: examples/example-qt.cpp adapters/qt.h $(STLIBNAME)
+	$(QT_MOC) adapters/qt.h -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore | \
+	    $(CXX) -x c++ -o qt-adapter-moc.o -c - $(REAL_CFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore
+	$(QT_MOC) examples/example-qt.h -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore | \
+	    $(CXX) -x c++ -o qt-example-moc.o -c - $(REAL_CFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore
+	$(CXX) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore -L$(QT_LIBRARY_DIR) qt-adapter-moc.o qt-example-moc.o $< -pthread $(STLIBNAME) -lQtCore
+endif
+
+hiredis-example: examples/example.c $(STLIBNAME)
+	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS)
+
+hiredis-example-push: examples/example-push.c $(STLIBNAME)
+	$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS)
+
+examples: $(EXAMPLES)
+
+TEST_LIBS = $(STLIBNAME)
+ifeq ($(USE_SSL),1)
+    TEST_LIBS += $(SSL_STLIBNAME)
+    TEST_LDFLAGS = $(SSL_LDFLAGS) -lssl -lcrypto -lpthread
+endif
+
+hiredis-test: test.o $(TEST_LIBS)
+	$(CC) -o $@ $(REAL_CFLAGS) -I. $^ $(REAL_LDFLAGS) $(TEST_LDFLAGS)
+
+hiredis-%: %.o $(STLIBNAME)
+	$(CC) $(REAL_CFLAGS) -o $@ $< $(TEST_LIBS) $(REAL_LDFLAGS)
+
+test: hiredis-test
+	./hiredis-test
+
+check: hiredis-test
+	TEST_SSL=$(USE_SSL) ./test.sh
+
+.c.o:
+	$(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $<
+
+clean:
+	rm -rf $(DYLIBNAME) $(STLIBNAME) $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov
+
+dep:
+	$(CC) $(CPPFLAGS) $(CFLAGS) -MM *.c
+
+INSTALL?= cp -pPR
+
+$(PKGCONFNAME): hiredis.h
+	@echo "Generating $@ for pkgconfig..."
+	@echo prefix=$(PREFIX) > $@
+	@echo exec_prefix=\$${prefix} >> $@
+	@echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@
+	@echo includedir=$(PREFIX)/$(INCLUDE_PATH) >> $@
+	@echo >> $@
+	@echo Name: hiredis >> $@
+	@echo Description: Minimalistic C client library for Redis. >> $@
+	@echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@
+	@echo Libs: -L\$${libdir} -lhiredis >> $@
+	@echo Cflags: -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@
+
+$(SSL_PKGCONFNAME): hiredis_ssl.h
+	@echo "Generating $@ for pkgconfig..."
+	@echo prefix=$(PREFIX) > $@
+	@echo exec_prefix=\$${prefix} >> $@
+	@echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@
+	@echo includedir=$(PREFIX)/$(INCLUDE_PATH) >> $@
+	@echo >> $@
+	@echo Name: hiredis_ssl >> $@
+	@echo Description: SSL Support for hiredis. >> $@
+	@echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@
+	@echo Requires: hiredis >> $@
+	@echo Libs: -L\$${libdir} -lhiredis_ssl >> $@
+	@echo Libs.private: -lssl -lcrypto >> $@
+
+install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME)
+	mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL_LIBRARY_PATH)
+	$(INSTALL) hiredis.h async.h read.h sds.h alloc.h $(INSTALL_INCLUDE_PATH)
+	$(INSTALL) adapters/*.h $(INSTALL_INCLUDE_PATH)/adapters
+	$(INSTALL) $(DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(DYLIB_MINOR_NAME)
+	cd $(INSTALL_LIBRARY_PATH) && ln -sf $(DYLIB_MINOR_NAME) $(DYLIBNAME)
+	$(INSTALL) $(STLIBNAME) $(INSTALL_LIBRARY_PATH)
+	mkdir -p $(INSTALL_PKGCONF_PATH)
+	$(INSTALL) $(PKGCONFNAME) $(INSTALL_PKGCONF_PATH)
+
+ifeq ($(USE_SSL),1)
+install: install-ssl
+
+install-ssl: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME)
+	mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_LIBRARY_PATH)
+	$(INSTALL) hiredis_ssl.h $(INSTALL_INCLUDE_PATH)
+	$(INSTALL) $(SSL_DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(SSL_DYLIB_MINOR_NAME)
+	cd $(INSTALL_LIBRARY_PATH) && ln -sf $(SSL_DYLIB_MINOR_NAME) $(SSL_DYLIBNAME)
+	$(INSTALL) $(SSL_STLIBNAME) $(INSTALL_LIBRARY_PATH)
+	mkdir -p $(INSTALL_PKGCONF_PATH)
+	$(INSTALL) $(SSL_PKGCONFNAME) $(INSTALL_PKGCONF_PATH)
+endif
+
+32bit:
+	@echo ""
+	@echo "WARNING: if this fails under Linux you probably need to install libc6-dev-i386"
+	@echo ""
+	$(MAKE) CFLAGS="-m32" LDFLAGS="-m32"
+
+32bit-vars:
+	$(eval CFLAGS=-m32)
+	$(eval LDFLAGS=-m32)
+
+gprof:
+	$(MAKE) CFLAGS="-pg" LDFLAGS="-pg"
+
+gcov:
+	$(MAKE) CFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS="-fprofile-arcs"
+
+coverage: gcov
+	make check
+	mkdir -p tmp/lcov
+	lcov -d . -c -o tmp/lcov/hiredis.info
+	genhtml --legend -o tmp/lcov/report tmp/lcov/hiredis.info
+
+noopt:
+	$(MAKE) OPTIMIZATION=""
+
+.PHONY: all test check clean dep install 32bit 32bit-vars gprof gcov noopt
diff --git a/README.md b/README.md
new file mode 100644
index 000000000..3a22553ea
--- /dev/null
+++ b/README.md
@@ -0,0 +1,658 @@
+[![Build Status](https://travis-ci.org/redis/hiredis.png)](https://travis-ci.org/redis/hiredis)
+
+**This Readme reflects the latest changed in the master branch. See [v1.0.0](https://github.com/redis/hiredis/tree/v1.0.0) for the Readme and documentation for the latest release ([API/ABI history](https://abi-laboratory.pro/?view=timeline&l=hiredis)).**
+
+# HIREDIS
+
+Hiredis is a minimalistic C client library for the [Redis](http://redis.io/) database.
+
+It is minimalistic because it just adds minimal support for the protocol, but
+at the same time it uses a high level printf-alike API in order to make it
+much higher level than otherwise suggested by its minimal code base and the
+lack of explicit bindings for every Redis command.
+
+Apart from supporting sending commands and receiving replies, it comes with
+a reply parser that is decoupled from the I/O layer. It
+is a stream parser designed for easy reusability, which can for instance be used
+in higher level language bindings for efficient reply parsing.
+
+Hiredis only supports the binary-safe Redis protocol, so you can use it with any
+Redis version >= 1.2.0.
+
+The library comes with multiple APIs. There is the
+*synchronous API*, the *asynchronous API* and the *reply parsing API*.
+
+## Upgrading to `1.0.0`
+
+Version 1.0.0 marks the first stable release of Hiredis.
+It includes some minor breaking changes, mostly to make the exposed API more uniform and self-explanatory.
+It also bundles the updated `sds` library, to sync up with upstream and Redis.
+For code changes see the [Changelog](CHANGELOG.md).
+
+_Note:  As described below, a few member names have been changed but most applications should be able to upgrade with minor code changes and recompiling._
+
+## IMPORTANT:  Breaking changes from `0.14.1` -> `1.0.0`
+
+* `redisContext` has two additional members (`free_privdata`, and `privctx`).
+* `redisOptions.timeout` has been renamed to `redisOptions.connect_timeout`, and we've added `redisOptions.command_timeout`.
+* `redisReplyObjectFunctions.createArray` now takes `size_t` instead of `int` for its length parameter.
+
+## IMPORTANT:  Breaking changes when upgrading from 0.13.x -> 0.14.x
+
+Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now
+protocol errors. This is consistent with the RESP specification. On 32-bit
+platforms, the upper bound is lowered to `SIZE_MAX`.
+
+Change `redisReply.len` to `size_t`, as it denotes the the size of a string
+
+User code should compare this to `size_t` values as well.  If it was used to
+compare to other values, casting might be necessary or can be removed, if
+casting was applied before.
+
+## Upgrading from `<0.9.0`
+
+Version 0.9.0 is a major overhaul of hiredis in every aspect. However, upgrading existing
+code using hiredis should not be a big pain. The key thing to keep in mind when
+upgrading is that hiredis >= 0.9.0 uses a `redisContext*` to keep state, in contrast to
+the stateless 0.0.1 that only has a file descriptor to work with.
+
+## Synchronous API
+
+To consume the synchronous API, there are only a few function calls that need to be introduced:
+
+```c
+redisContext *redisConnect(const char *ip, int port);
+void *redisCommand(redisContext *c, const char *format, ...);
+void freeReplyObject(void *reply);
+```
+
+### Connecting
+
+The function `redisConnect` is used to create a so-called `redisContext`. The
+context is where Hiredis holds state for a connection. The `redisContext`
+struct has an integer `err` field that is non-zero when the connection is in
+an error state. The field `errstr` will contain a string with a description of
+the error. More information on errors can be found in the **Errors** section.
+After trying to connect to Redis using `redisConnect` you should
+check the `err` field to see if establishing the connection was successful:
+```c
+redisContext *c = redisConnect("127.0.0.1", 6379);
+if (c == NULL || c->err) {
+    if (c) {
+        printf("Error: %s\n", c->errstr);
+        // handle error
+    } else {
+        printf("Can't allocate redis context\n");
+    }
+}
+```
+
+*Note: A `redisContext` is not thread-safe.*
+
+### Sending commands
+
+There are several ways to issue commands to Redis. The first that will be introduced is
+`redisCommand`. This function takes a format similar to printf. In the simplest form,
+it is used like this:
+```c
+reply = redisCommand(context, "SET foo bar");
+```
+
+The specifier `%s` interpolates a string in the command, and uses `strlen` to
+determine the length of the string:
+```c
+reply = redisCommand(context, "SET foo %s", value);
+```
+When you need to pass binary safe strings in a command, the `%b` specifier can be
+used. Together with a pointer to the string, it requires a `size_t` length argument
+of the string:
+```c
+reply = redisCommand(context, "SET foo %b", value, (size_t) valuelen);
+```
+Internally, Hiredis splits the command in different arguments and will
+convert it to the protocol used to communicate with Redis.
+One or more spaces separates arguments, so you can use the specifiers
+anywhere in an argument:
+```c
+reply = redisCommand(context, "SET key:%s %s", myid, value);
+```
+
+### Using replies
+
+The return value of `redisCommand` holds a reply when the command was
+successfully executed. When an error occurs, the return value is `NULL` and
+the `err` field in the context will be set (see section on **Errors**).
+Once an error is returned the context cannot be reused and you should set up
+a new connection.
+
+The standard replies that `redisCommand` are of the type `redisReply`. The
+`type` field in the `redisReply` should be used to test what kind of reply
+was received:
+
+### RESP2
+
+* **`REDIS_REPLY_STATUS`**:
+    * The command replied with a status reply. The status string can be accessed using `reply->str`.
+      The length of this string can be accessed using `reply->len`.
+
+* **`REDIS_REPLY_ERROR`**:
+    *  The command replied with an error. The error string can be accessed identical to `REDIS_REPLY_STATUS`.
+
+* **`REDIS_REPLY_INTEGER`**:
+    * The command replied with an integer. The integer value can be accessed using the
+      `reply->integer` field of type `long long`.
+
+* **`REDIS_REPLY_NIL`**:
+    * The command replied with a **nil** object. There is no data to access.
+
+* **`REDIS_REPLY_STRING`**:
+    * A bulk (string) reply. The value of the reply can be accessed using `reply->str`.
+      The length of this string can be accessed using `reply->len`.
+
+* **`REDIS_REPLY_ARRAY`**:
+    * A multi bulk reply. The number of elements in the multi bulk reply is stored in
+      `reply->elements`. Every element in the multi bulk reply is a `redisReply` object as well
+      and can be accessed via `reply->element[..index..]`.
+      Redis may reply with nested arrays but this is fully supported.
+
+### RESP3
+
+Hiredis also supports every new `RESP3` data type which are as follows.  For more information about the protocol see the `RESP3` [specification.](https://github.com/antirez/RESP3/blob/master/spec.md)
+
+* **`REDIS_REPLY_DOUBLE`**:
+    * The command replied with a double-precision floating point number.
+      The value is stored as a string in the `str` member, and can be converted with `strtod` or similar.
+
+* **`REDIS_REPLY_BOOL`**:
+    * A boolean true/false reply.
+      The value is stored in the `integer` member and will be either `0` or `1`.
+
+* **`REDIS_REPLY_MAP`**:
+    * An array with the added invariant that there will always be an even number of elements.
+      The MAP is functionally equivelant to `REDIS_REPLY_ARRAY` except for the previously mentioned invariant.
+
+* **`REDIS_REPLY_SET`**:
+    * An array response where each entry is unique.
+      Like the MAP type, the data is identical to an array response except there are no duplicate values.
+
+* **`REDIS_REPLY_PUSH`**:
+    * An array that can be generated spontaneously by Redis.
+      This array response will always contain at least two subelements.  The first contains the type of `PUSH` message (e.g. `message`, or `invalidate`), and the second being a sub-array with the `PUSH` payload itself.
+
+* **`REDIS_REPLY_ATTR`**:
+    * An array structurally identical to a `MAP` but intended as meta-data about a reply.
+      _As of Redis 6.0.6 this reply type is not used in Redis_
+
+* **`REDIS_REPLY_BIGNUM`**:
+    * A string representing an arbitrarily large signed or unsigned integer value.
+      The number will be encoded as a string in the `str` member of `redisReply`.
+
+* **`REDIS_REPLY_VERB`**:
+    * A verbatim string, intended to be presented to the user without modification.
+      The string payload is stored in the `str` memeber, and type data is stored in the `vtype` member (e.g. `txt` for raw text or `md` for markdown).
+
+Replies should be freed using the `freeReplyObject()` function.
+Note that this function will take care of freeing sub-reply objects
+contained in arrays and nested arrays, so there is no need for the user to
+free the sub replies (it is actually harmful and will corrupt the memory).
+
+**Important:** the current version of hiredis (1.0.0) frees replies when the
+asynchronous API is used. This means you should not call `freeReplyObject` when
+you use this API. The reply is cleaned up by hiredis _after_ the callback
+returns.  We may introduce a flag to make this configurable in future versions of the library.
+
+### Cleaning up
+
+To disconnect and free the context the following function can be used:
+```c
+void redisFree(redisContext *c);
+```
+This function immediately closes the socket and then frees the allocations done in
+creating the context.
+
+### Sending commands (cont'd)
+
+Together with `redisCommand`, the function `redisCommandArgv` can be used to issue commands.
+It has the following prototype:
+```c
+void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
+```
+It takes the number of arguments `argc`, an array of strings `argv` and the lengths of the
+arguments `argvlen`. For convenience, `argvlen` may be set to `NULL` and the function will
+use `strlen(3)` on every argument to determine its length. Obviously, when any of the arguments
+need to be binary safe, the entire array of lengths `argvlen` should be provided.
+
+The return value has the same semantic as `redisCommand`.
+
+### Pipelining
+
+To explain how Hiredis supports pipelining in a blocking connection, there needs to be
+understanding of the internal execution flow.
+
+When any of the functions in the `redisCommand` family is called, Hiredis first formats the
+command according to the Redis protocol. The formatted command is then put in the output buffer
+of the context. This output buffer is dynamic, so it can hold any number of commands.
+After the command is put in the output buffer, `redisGetReply` is called. This function has the
+following two execution paths:
+
+1. The input buffer is non-empty:
+    * Try to parse a single reply from the input buffer and return it
+    * If no reply could be parsed, continue at *2*
+2. The input buffer is empty:
+    * Write the **entire** output buffer to the socket
+    * Read from the socket until a single reply could be parsed
+
+The function `redisGetReply` is exported as part of the Hiredis API and can be used when a reply
+is expected on the socket. To pipeline commands, the only things that needs to be done is
+filling up the output buffer. For this cause, two commands can be used that are identical
+to the `redisCommand` family, apart from not returning a reply:
+```c
+void redisAppendCommand(redisContext *c, const char *format, ...);
+void redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
+```
+After calling either function one or more times, `redisGetReply` can be used to receive the
+subsequent replies. The return value for this function is either `REDIS_OK` or `REDIS_ERR`, where
+the latter means an error occurred while reading a reply. Just as with the other commands,
+the `err` field in the context can be used to find out what the cause of this error is.
+
+The following examples shows a simple pipeline (resulting in only a single call to `write(2)` and
+a single call to `read(2)`):
+```c
+redisReply *reply;
+redisAppendCommand(context,"SET foo bar");
+redisAppendCommand(context,"GET foo");
+redisGetReply(context,(void *)&reply); // reply for SET
+freeReplyObject(reply);
+redisGetReply(context,(void *)&reply); // reply for GET
+freeReplyObject(reply);
+```
+This API can also be used to implement a blocking subscriber:
+```c
+reply = redisCommand(context,"SUBSCRIBE foo");
+freeReplyObject(reply);
+while(redisGetReply(context,(void *)&reply) == REDIS_OK) {
+    // consume message
+    freeReplyObject(reply);
+}
+```
+### Errors
+
+When a function call is not successful, depending on the function either `NULL` or `REDIS_ERR` is
+returned. The `err` field inside the context will be non-zero and set to one of the
+following constants:
+
+* **`REDIS_ERR_IO`**:
+    There was an I/O error while creating the connection, trying to write
+    to the socket or read from the socket. If you included `errno.h` in your
+    application, you can use the global `errno` variable to find out what is
+    wrong.
+
+* **`REDIS_ERR_EOF`**:
+    The server closed the connection which resulted in an empty read.
+
+* **`REDIS_ERR_PROTOCOL`**:
+    There was an error while parsing the protocol.
+
+* **`REDIS_ERR_OTHER`**:
+    Any other error. Currently, it is only used when a specified hostname to connect
+    to cannot be resolved.
+
+In every case, the `errstr` field in the context will be set to hold a string representation
+of the error.
+
+## Asynchronous API
+
+Hiredis comes with an asynchronous API that works easily with any event library.
+Examples are bundled that show using Hiredis with [libev](http://software.schmorp.de/pkg/libev.html)
+and [libevent](http://monkey.org/~provos/libevent/).
+
+### Connecting
+
+The function `redisAsyncConnect` can be used to establish a non-blocking connection to
+Redis. It returns a pointer to the newly created `redisAsyncContext` struct. The `err` field
+should be checked after creation to see if there were errors creating the connection.
+Because the connection that will be created is non-blocking, the kernel is not able to
+instantly return if the specified host and port is able to accept a connection.
+
+*Note: A `redisAsyncContext` is not thread-safe.*
+
+```c
+redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
+if (c->err) {
+    printf("Error: %s\n", c->errstr);
+    // handle error
+}
+```
+
+The asynchronous context can hold a disconnect callback function that is called when the
+connection is disconnected (either because of an error or per user request). This function should
+have the following prototype:
+```c
+void(const redisAsyncContext *c, int status);
+```
+On a disconnect, the `status` argument is set to `REDIS_OK` when disconnection was initiated by the
+user, or `REDIS_ERR` when the disconnection was caused by an error. When it is `REDIS_ERR`, the `err`
+field in the context can be accessed to find out the cause of the error.
+
+The context object is always freed after the disconnect callback fired. When a reconnect is needed,
+the disconnect callback is a good point to do so.
+
+Setting the disconnect callback can only be done once per context. For subsequent calls it will
+return `REDIS_ERR`. The function to set the disconnect callback has the following prototype:
+```c
+int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn);
+```
+`ac->data` may be used to pass user data to this callback, the same can be done for redisConnectCallback.
+### Sending commands and their callbacks
+
+In an asynchronous context, commands are automatically pipelined due to the nature of an event loop.
+Therefore, unlike the synchronous API, there is only a single way to send commands.
+Because commands are sent to Redis asynchronously, issuing a command requires a callback function
+that is called when the reply is received. Reply callbacks should have the following prototype:
+```c
+void(redisAsyncContext *c, void *reply, void *privdata);
+```
+The `privdata` argument can be used to curry arbitrary data to the callback from the point where
+the command is initially queued for execution.
+
+The functions that can be used to issue commands in an asynchronous context are:
+```c
+int redisAsyncCommand(
+  redisAsyncContext *ac, redisCallbackFn *fn, void *privdata,
+  const char *format, ...);
+int redisAsyncCommandArgv(
+  redisAsyncContext *ac, redisCallbackFn *fn, void *privdata,
+  int argc, const char **argv, const size_t *argvlen);
+```
+Both functions work like their blocking counterparts. The return value is `REDIS_OK` when the command
+was successfully added to the output buffer and `REDIS_ERR` otherwise. Example: when the connection
+is being disconnected per user-request, no new commands may be added to the output buffer and `REDIS_ERR` is
+returned on calls to the `redisAsyncCommand` family.
+
+If the reply for a command with a `NULL` callback is read, it is immediately freed. When the callback
+for a command is non-`NULL`, the memory is freed immediately following the callback: the reply is only
+valid for the duration of the callback.
+
+All pending callbacks are called with a `NULL` reply when the context encountered an error.
+
+### Disconnecting
+
+An asynchronous connection can be terminated using:
+```c
+void redisAsyncDisconnect(redisAsyncContext *ac);
+```
+When this function is called, the connection is **not** immediately terminated. Instead, new
+commands are no longer accepted and the connection is only terminated when all pending commands
+have been written to the socket, their respective replies have been read and their respective
+callbacks have been executed. After this, the disconnection callback is executed with the
+`REDIS_OK` status and the context object is freed.
+
+### Hooking it up to event library *X*
+
+There are a few hooks that need to be set on the context object after it is created.
+See the `adapters/` directory for bindings to *libev* and *libevent*.
+
+## Reply parsing API
+
+Hiredis comes with a reply parsing API that makes it easy for writing higher
+level language bindings.
+
+The reply parsing API consists of the following functions:
+```c
+redisReader *redisReaderCreate(void);
+void redisReaderFree(redisReader *reader);
+int redisReaderFeed(redisReader *reader, const char *buf, size_t len);
+int redisReaderGetReply(redisReader *reader, void **reply);
+```
+The same set of functions are used internally by hiredis when creating a
+normal Redis context, the above API just exposes it to the user for a direct
+usage.
+
+### Usage
+
+The function `redisReaderCreate` creates a `redisReader` structure that holds a
+buffer with unparsed data and state for the protocol parser.
+
+Incoming data -- most likely from a socket -- can be placed in the internal
+buffer of the `redisReader` using `redisReaderFeed`. This function will make a
+copy of the buffer pointed to by `buf` for `len` bytes. This data is parsed
+when `redisReaderGetReply` is called. This function returns an integer status
+and a reply object (as described above) via `void **reply`. The returned status
+can be either `REDIS_OK` or `REDIS_ERR`, where the latter means something went
+wrong (either a protocol error, or an out of memory error).
+
+The parser limits the level of nesting for multi bulk payloads to 7. If the
+multi bulk nesting level is higher than this, the parser returns an error.
+
+### Customizing replies
+
+The function `redisReaderGetReply` creates `redisReply` and makes the function
+argument `reply` point to the created `redisReply` variable. For instance, if
+the response of type `REDIS_REPLY_STATUS` then the `str` field of `redisReply`
+will hold the status as a vanilla C string. However, the functions that are
+responsible for creating instances of the `redisReply` can be customized by
+setting the `fn` field on the `redisReader` struct. This should be done
+immediately after creating the `redisReader`.
+
+For example, [hiredis-rb](https://github.com/pietern/hiredis-rb/blob/master/ext/hiredis_ext/reader.c)
+uses customized reply object functions to create Ruby objects.
+
+### Reader max buffer
+
+Both when using the Reader API directly or when using it indirectly via a
+normal Redis context, the redisReader structure uses a buffer in order to
+accumulate data from the server.
+Usually this buffer is destroyed when it is empty and is larger than 16
+KiB in order to avoid wasting memory in unused buffers
+
+However when working with very big payloads destroying the buffer may slow
+down performances considerably, so it is possible to modify the max size of
+an idle buffer changing the value of the `maxbuf` field of the reader structure
+to the desired value. The special value of 0 means that there is no maximum
+value for an idle buffer, so the buffer will never get freed.
+
+For instance if you have a normal Redis context you can set the maximum idle
+buffer to zero (unlimited) just with:
+```c
+context->reader->maxbuf = 0;
+```
+This should be done only in order to maximize performances when working with
+large payloads. The context should be set back to `REDIS_READER_MAX_BUF` again
+as soon as possible in order to prevent allocation of useless memory.
+
+### Reader max array elements
+
+By default the hiredis reply parser sets the maximum number of multi-bulk elements
+to 2^32 - 1 or 4,294,967,295 entries.  If you need to process multi-bulk replies
+with more than this many elements you can set the value higher or to zero, meaning
+unlimited with:
+```c
+context->reader->maxelements = 0;
+```
+
+## SSL/TLS Support
+
+### Building
+
+SSL/TLS support is not built by default and requires an explicit flag:
+
+    make USE_SSL=1
+
+This requires OpenSSL development package (e.g. including header files to be
+available.
+
+When enabled, SSL/TLS support is built into extra `libhiredis_ssl.a` and
+`libhiredis_ssl.so` static/dynamic libraries. This leaves the original libraries
+unaffected so no additional dependencies are introduced.
+
+### Using it
+
+First, you'll need to make sure you include the SSL header file:
+
+```c
+#include "hiredis.h"
+#include "hiredis_ssl.h"
+```
+
+You will also need to link against `libhiredis_ssl`, **in addition** to
+`libhiredis` and add `-lssl -lcrypto` to satisfy its dependencies.
+
+Hiredis implements SSL/TLS on top of its normal `redisContext` or
+`redisAsyncContext`, so you will need to establish a connection first and then
+initiate an SSL/TLS handshake.
+
+#### Hiredis OpenSSL Wrappers
+
+Before Hiredis can negotiate an SSL/TLS connection, it is necessary to
+initialize OpenSSL and create a context. You can do that in two ways:
+
+1. Work directly with the OpenSSL API to initialize the library's global context
+   and create `SSL_CTX *` and `SSL *` contexts. With an `SSL *` object you can
+   call `redisInitiateSSL()`.
+2. Work with a set of Hiredis-provided wrappers around OpenSSL, create a
+   `redisSSLContext` object to hold configuration and use
+   `redisInitiateSSLWithContext()` to initiate the SSL/TLS handshake.
+
+```c
+/* An Hiredis SSL context. It holds SSL configuration and can be reused across
+ * many contexts.
+ */
+redisSSLContext *ssl;
+
+/* An error variable to indicate what went wrong, if the context fails to
+ * initialize.
+ */
+redisSSLContextError ssl_error;
+
+/* Initialize global OpenSSL state.
+ *
+ * You should call this only once when your app initializes, and only if
+ * you don't explicitly or implicitly initialize OpenSSL it elsewhere.
+ */
+redisInitOpenSSL();
+
+/* Create SSL context */
+ssl = redisCreateSSLContext(
+    "cacertbundle.crt",     /* File name of trusted CA/ca bundle file, optional */
+    "/path/to/certs",       /* Path of trusted certificates, optional */
+    "client_cert.pem",      /* File name of client certificate file, optional */
+    "client_key.pem",       /* File name of client private key, optional */
+    "redis.mydomain.com",   /* Server name to request (SNI), optional */
+    &ssl_error
+    ) != REDIS_OK) {
+        printf("SSL error: %s\n", redisSSLContextGetError(ssl_error);
+        /* Abort... */
+    }
+
+/* Create Redis context and establish connection */
+c = redisConnect("localhost", 6443);
+if (c == NULL || c->err) {
+    /* Handle error and abort... */
+}
+
+/* Negotiate SSL/TLS */
+if (redisInitiateSSLWithContext(c, ssl) != REDIS_OK) {
+    /* Handle error, in c->err / c->errstr */
+}
+```
+
+## RESP3 PUSH replies
+Redis 6.0 introduced PUSH replies with the reply-type `>`.  These messages are generated spontaneously and can arrive at any time, so must be handled using callbacks.
+
+### Default behavior
+Hiredis installs handlers on `redisContext` and `redisAsyncContext` by default, which will intercept and free any PUSH replies detected.  This means existing code will work as-is after upgrading to Redis 6 and switching to `RESP3`.
+
+### Custom PUSH handler prototypes
+The callback prototypes differ between `redisContext` and `redisAsyncContext`.
+
+#### redisContext
+```c
+void my_push_handler(void *privdata, void *reply) {
+    /* Handle the reply */
+
+    /* Note: We need to free the reply in our custom handler for
+             blocking contexts.  This lets us keep the reply if
+             we want. */
+    freeReplyObject(reply);
+}
+```
+
+#### redisAsyncContext
+```c
+void my_async_push_handler(redisAsyncContext *ac, void *reply) {
+    /* Handle the reply */
+
+    /* Note:  Because async hiredis always frees replies, you should
+              not call freeReplyObject in an async push callback. */
+}
+```
+
+### Installing a custom handler
+There are two ways to set your own PUSH handlers.
+
+1. Set `push_cb` or `async_push_cb` in the `redisOptions` struct and connect with `redisConnectWithOptions` or `redisAsyncConnectWithOptions`.
+    ```c
+    redisOptions = {0};
+    REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379);
+    options->push_cb = my_push_handler;
+    redisContext *context = redisConnectWithOptions(&options);
+    ```
+2.  Call `redisSetPushCallback` or `redisAsyncSetPushCallback` on a connected context.
+    ```c
+    redisContext *context = redisConnect("127.0.0.1", 6379);
+    redisSetPushCallback(context, my_push_handler);
+    ```
+
+    _Note `redisSetPushCallback` and `redisAsyncSetPushCallback` both return any currently configured handler,  making it easy to override and then return to the old value._
+
+### Specifying no handler
+If you have a unique use-case where you don't want hiredis to automatically intercept and free PUSH replies, you will want to configure no handler at all.  This can be done in two ways.
+1.  Set the `REDIS_OPT_NO_PUSH_AUTOFREE` flag in `redisOptions` and leave the callback function pointer `NULL`.
+    ```c
+    redisOptions = {0};
+    REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379);
+    options->options |= REDIS_OPT_NO_PUSH_AUTOFREE;
+    redisContext *context = redisConnectWithOptions(&options);
+    ```
+3.  Call `redisSetPushCallback` with `NULL` once connected.
+    ```c
+    redisContext *context = redisConnect("127.0.0.1", 6379);
+    redisSetPushCallback(context, NULL);
+    ```
+
+    _Note:  With no handler configured, calls to `redisCommand` may generate more than one reply, so this strategy is only applicable when there's some kind of blocking`redisGetReply()` loop (e.g. `MONITOR` or `SUBSCRIBE` workloads)._
+
+## Allocator injection
+
+Hiredis uses a pass-thru structure of function pointers defined in [alloc.h](https://github.com/redis/hiredis/blob/f5d25850/alloc.h#L41) that contain the currently configured allocation and deallocation functions.  By default they just point to libc (`malloc`, `calloc`, `realloc`, etc).
+
+### Overriding
+
+One can override the allocators like so:
+
+```c
+hiredisAllocFuncs myfuncs = {
+    .mallocFn = my_malloc,
+    .callocFn = my_calloc,
+    .reallocFn = my_realloc,
+    .strdupFn = my_strdup,
+    .freeFn = my_free,
+};
+
+// Override allocators (function returns current allocators if needed)
+hiredisAllocFuncs orig = hiredisSetAllocators(&myfuncs);
+```
+
+To reset the allocators to their default libc function simply call:
+
+```c
+hiredisResetAllocators();
+```
+
+## AUTHORS
+
+Salvatore Sanfilippo (antirez at gmail),\
+Pieter Noordhuis (pcnoordhuis at gmail)\
+Michael Grunder (michael dot grunder at gmail)
+
+_Hiredis is released under the BSD license._
diff --git a/adapters/ae.h b/adapters/ae.h
new file mode 100644
index 000000000..660d82eb0
--- /dev/null
+++ b/adapters/ae.h
@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_AE_H__
+#define __HIREDIS_AE_H__
+#include <sys/types.h>
+#include <ae.h>
+#include "../hiredis.h"
+#include "../async.h"
+
+typedef struct redisAeEvents {
+    redisAsyncContext *context;
+    aeEventLoop *loop;
+    int fd;
+    int reading, writing;
+} redisAeEvents;
+
+static void redisAeReadEvent(aeEventLoop *el, int fd, void *privdata, int mask) {
+    ((void)el); ((void)fd); ((void)mask);
+
+    redisAeEvents *e = (redisAeEvents*)privdata;
+    redisAsyncHandleRead(e->context);
+}
+
+static void redisAeWriteEvent(aeEventLoop *el, int fd, void *privdata, int mask) {
+    ((void)el); ((void)fd); ((void)mask);
+
+    redisAeEvents *e = (redisAeEvents*)privdata;
+    redisAsyncHandleWrite(e->context);
+}
+
+static void redisAeAddRead(void *privdata) {
+    redisAeEvents *e = (redisAeEvents*)privdata;
+    aeEventLoop *loop = e->loop;
+    if (!e->reading) {
+        e->reading = 1;
+        aeCreateFileEvent(loop,e->fd,AE_READABLE,redisAeReadEvent,e);
+    }
+}
+
+static void redisAeDelRead(void *privdata) {
+    redisAeEvents *e = (redisAeEvents*)privdata;
+    aeEventLoop *loop = e->loop;
+    if (e->reading) {
+        e->reading = 0;
+        aeDeleteFileEvent(loop,e->fd,AE_READABLE);
+    }
+}
+
+static void redisAeAddWrite(void *privdata) {
+    redisAeEvents *e = (redisAeEvents*)privdata;
+    aeEventLoop *loop = e->loop;
+    if (!e->writing) {
+        e->writing = 1;
+        aeCreateFileEvent(loop,e->fd,AE_WRITABLE,redisAeWriteEvent,e);
+    }
+}
+
+static void redisAeDelWrite(void *privdata) {
+    redisAeEvents *e = (redisAeEvents*)privdata;
+    aeEventLoop *loop = e->loop;
+    if (e->writing) {
+        e->writing = 0;
+        aeDeleteFileEvent(loop,e->fd,AE_WRITABLE);
+    }
+}
+
+static void redisAeCleanup(void *privdata) {
+    redisAeEvents *e = (redisAeEvents*)privdata;
+    redisAeDelRead(privdata);
+    redisAeDelWrite(privdata);
+    hi_free(e);
+}
+
+static int redisAeAttach(aeEventLoop *loop, redisAsyncContext *ac) {
+    redisContext *c = &(ac->c);
+    redisAeEvents *e;
+
+    /* Nothing should be attached when something is already attached */
+    if (ac->ev.data != NULL)
+        return REDIS_ERR;
+
+    /* Create container for context and r/w events */
+    e = (redisAeEvents*)hi_malloc(sizeof(*e));
+    if (e == NULL)
+        return REDIS_ERR;
+
+    e->context = ac;
+    e->loop = loop;
+    e->fd = c->fd;
+    e->reading = e->writing = 0;
+
+    /* Register functions to start/stop listening for events */
+    ac->ev.addRead = redisAeAddRead;
+    ac->ev.delRead = redisAeDelRead;
+    ac->ev.addWrite = redisAeAddWrite;
+    ac->ev.delWrite = redisAeDelWrite;
+    ac->ev.cleanup = redisAeCleanup;
+    ac->ev.data = e;
+
+    return REDIS_OK;
+}
+#endif
diff --git a/adapters/glib.h b/adapters/glib.h
new file mode 100644
index 000000000..ad59dd142
--- /dev/null
+++ b/adapters/glib.h
@@ -0,0 +1,156 @@
+#ifndef __HIREDIS_GLIB_H__
+#define __HIREDIS_GLIB_H__
+
+#include <glib.h>
+
+#include "../hiredis.h"
+#include "../async.h"
+
+typedef struct
+{
+    GSource source;
+    redisAsyncContext *ac;
+    GPollFD poll_fd;
+} RedisSource;
+
+static void
+redis_source_add_read (gpointer data)
+{
+    RedisSource *source = (RedisSource *)data;
+    g_return_if_fail(source);
+    source->poll_fd.events |= G_IO_IN;
+    g_main_context_wakeup(g_source_get_context((GSource *)data));
+}
+
+static void
+redis_source_del_read (gpointer data)
+{
+    RedisSource *source = (RedisSource *)data;
+    g_return_if_fail(source);
+    source->poll_fd.events &= ~G_IO_IN;
+    g_main_context_wakeup(g_source_get_context((GSource *)data));
+}
+
+static void
+redis_source_add_write (gpointer data)
+{
+    RedisSource *source = (RedisSource *)data;
+    g_return_if_fail(source);
+    source->poll_fd.events |= G_IO_OUT;
+    g_main_context_wakeup(g_source_get_context((GSource *)data));
+}
+
+static void
+redis_source_del_write (gpointer data)
+{
+    RedisSource *source = (RedisSource *)data;
+    g_return_if_fail(source);
+    source->poll_fd.events &= ~G_IO_OUT;
+    g_main_context_wakeup(g_source_get_context((GSource *)data));
+}
+
+static void
+redis_source_cleanup (gpointer data)
+{
+    RedisSource *source = (RedisSource *)data;
+
+    g_return_if_fail(source);
+
+    redis_source_del_read(source);
+    redis_source_del_write(source);
+    /*
+     * It is not our responsibility to remove ourself from the
+     * current main loop. However, we will remove the GPollFD.
+     */
+    if (source->poll_fd.fd >= 0) {
+        g_source_remove_poll((GSource *)data, &source->poll_fd);
+        source->poll_fd.fd = -1;
+    }
+}
+
+static gboolean
+redis_source_prepare (GSource *source,
+                      gint    *timeout_)
+{
+    RedisSource *redis = (RedisSource *)source;
+    *timeout_ = -1;
+    return !!(redis->poll_fd.events & redis->poll_fd.revents);
+}
+
+static gboolean
+redis_source_check (GSource *source)
+{
+    RedisSource *redis = (RedisSource *)source;
+    return !!(redis->poll_fd.events & redis->poll_fd.revents);
+}
+
+static gboolean
+redis_source_dispatch (GSource      *source,
+                       GSourceFunc   callback,
+                       gpointer      user_data)
+{
+    RedisSource *redis = (RedisSource *)source;
+
+    if ((redis->poll_fd.revents & G_IO_OUT)) {
+        redisAsyncHandleWrite(redis->ac);
+        redis->poll_fd.revents &= ~G_IO_OUT;
+    }
+
+    if ((redis->poll_fd.revents & G_IO_IN)) {
+        redisAsyncHandleRead(redis->ac);
+        redis->poll_fd.revents &= ~G_IO_IN;
+    }
+
+    if (callback) {
+        return callback(user_data);
+    }
+
+    return TRUE;
+}
+
+static void
+redis_source_finalize (GSource *source)
+{
+    RedisSource *redis = (RedisSource *)source;
+
+    if (redis->poll_fd.fd >= 0) {
+        g_source_remove_poll(source, &redis->poll_fd);
+        redis->poll_fd.fd = -1;
+    }
+}
+
+static GSource *
+redis_source_new (redisAsyncContext *ac)
+{
+    static GSourceFuncs source_funcs = {
+        .prepare  = redis_source_prepare,
+        .check     = redis_source_check,
+        .dispatch = redis_source_dispatch,
+        .finalize = redis_source_finalize,
+    };
+    redisContext *c = &ac->c;
+    RedisSource *source;
+
+    g_return_val_if_fail(ac != NULL, NULL);
+
+    source = (RedisSource *)g_source_new(&source_funcs, sizeof *source);
+    if (source == NULL)
+        return NULL;
+
+    source->ac = ac;
+    source->poll_fd.fd = c->fd;
+    source->poll_fd.events = 0;
+    source->poll_fd.revents = 0;
+    g_source_add_poll((GSource *)source, &source->poll_fd);
+
+    ac->ev.addRead = redis_source_add_read;
+    ac->ev.delRead = redis_source_del_read;
+    ac->ev.addWrite = redis_source_add_write;
+    ac->ev.delWrite = redis_source_del_write;
+    ac->ev.cleanup = redis_source_cleanup;
+    ac->ev.data = source;
+
+    return (GSource *)source;
+}
+
+#endif /* __HIREDIS_GLIB_H__ */
diff --git a/adapters/ivykis.h b/adapters/ivykis.h
new file mode 100644
index 000000000..179f6ab52
--- /dev/null
+++ b/adapters/ivykis.h
@@ -0,0 +1,84 @@
+#ifndef __HIREDIS_IVYKIS_H__
+#define __HIREDIS_IVYKIS_H__
+#include <iv.h>
+#include "../hiredis.h"
+#include "../async.h"
+
+typedef struct redisIvykisEvents {
+    redisAsyncContext *context;
+    struct iv_fd fd;
+} redisIvykisEvents;
+
+static void redisIvykisReadEvent(void *arg) {
+    redisAsyncContext *context = (redisAsyncContext *)arg;
+    redisAsyncHandleRead(context);
+}
+
+static void redisIvykisWriteEvent(void *arg) {
+    redisAsyncContext *context = (redisAsyncContext *)arg;
+    redisAsyncHandleWrite(context);
+}
+
+static void redisIvykisAddRead(void *privdata) {
+    redisIvykisEvents *e = (redisIvykisEvents*)privdata;
+    iv_fd_set_handler_in(&e->fd, redisIvykisReadEvent);
+}
+
+static void redisIvykisDelRead(void *privdata) {
+    redisIvykisEvents *e = (redisIvykisEvents*)privdata;
+    iv_fd_set_handler_in(&e->fd, NULL);
+}
+
+static void redisIvykisAddWrite(void *privdata) {
+    redisIvykisEvents *e = (redisIvykisEvents*)privdata;
+    iv_fd_set_handler_out(&e->fd, redisIvykisWriteEvent);
+}
+
+static void redisIvykisDelWrite(void *privdata) {
+    redisIvykisEvents *e = (redisIvykisEvents*)privdata;
+    iv_fd_set_handler_out(&e->fd, NULL);
+}
+
+static void redisIvykisCleanup(void *privdata) {
+    redisIvykisEvents *e = (redisIvykisEvents*)privdata;
+
+    iv_fd_unregister(&e->fd);
+    hi_free(e);
+}
+
+static int redisIvykisAttach(redisAsyncContext *ac) {
+    redisContext *c = &(ac->c);
+    redisIvykisEvents *e;
+
+    /* Nothing should be attached when something is already attached */
+    if (ac->ev.data != NULL)
+        return REDIS_ERR;
+
+    /* Create container for context and r/w events */
+    e = (redisIvykisEvents*)hi_malloc(sizeof(*e));
+    if (e == NULL)
+        return REDIS_ERR;
+
+    e->context = ac;
+
+    /* Register functions to start/stop listening for events */
+    ac->ev.addRead = redisIvykisAddRead;
+    ac->ev.delRead = redisIvykisDelRead;
+    ac->ev.addWrite = redisIvykisAddWrite;
+    ac->ev.delWrite = redisIvykisDelWrite;
+    ac->ev.cleanup = redisIvykisCleanup;
+    ac->ev.data = e;
+
+    /* Initialize and install read/write events */
+    IV_FD_INIT(&e->fd);
+    e->fd.fd = c->fd;
+    e->fd.handler_in = redisIvykisReadEvent;
+    e->fd.handler_out = redisIvykisWriteEvent;
+    e->fd.handler_err = NULL;
+    e->fd.cookie = e->context;
+
+    iv_fd_register(&e->fd);
+
+    return REDIS_OK;
+}
+#endif
diff --git a/adapters/libev.h b/adapters/libev.h
new file mode 100644
index 000000000..e1e7bbd99
--- /dev/null
+++ b/adapters/libev.h
@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_LIBEV_H__
+#define __HIREDIS_LIBEV_H__
+#include <stdlib.h>
+#include <sys/types.h>
+#include <ev.h>
+#include "../hiredis.h"
+#include "../async.h"
+
+typedef struct redisLibevEvents {
+    redisAsyncContext *context;
+    struct ev_loop *loop;
+    int reading, writing;
+    ev_io rev, wev;
+    ev_timer timer;
+} redisLibevEvents;
+
+static void redisLibevReadEvent(EV_P_ ev_io *watcher, int revents) {
+#if EV_MULTIPLICITY
+    ((void)loop);
+#endif
+    ((void)revents);
+
+    redisLibevEvents *e = (redisLibevEvents*)watcher->data;
+    redisAsyncHandleRead(e->context);
+}
+
+static void redisLibevWriteEvent(EV_P_ ev_io *watcher, int revents) {
+#if EV_MULTIPLICITY
+    ((void)loop);
+#endif
+    ((void)revents);
+
+    redisLibevEvents *e = (redisLibevEvents*)watcher->data;
+    redisAsyncHandleWrite(e->context);
+}
+
+static void redisLibevAddRead(void *privdata) {
+    redisLibevEvents *e = (redisLibevEvents*)privdata;
+    struct ev_loop *loop = e->loop;
+    ((void)loop);
+    if (!e->reading) {
+        e->reading = 1;
+        ev_io_start(EV_A_ &e->rev);
+    }
+}
+
+static void redisLibevDelRead(void *privdata) {
+    redisLibevEvents *e = (redisLibevEvents*)privdata;
+    struct ev_loop *loop = e->loop;
+    ((void)loop);
+    if (e->reading) {
+        e->reading = 0;
+        ev_io_stop(EV_A_ &e->rev);
+    }
+}
+
+static void redisLibevAddWrite(void *privdata) {
+    redisLibevEvents *e = (redisLibevEvents*)privdata;
+    struct ev_loop *loop = e->loop;
+    ((void)loop);
+    if (!e->writing) {
+        e->writing = 1;
+        ev_io_start(EV_A_ &e->wev);
+    }
+}
+
+static void redisLibevDelWrite(void *privdata) {
+    redisLibevEvents *e = (redisLibevEvents*)privdata;
+    struct ev_loop *loop = e->loop;
+    ((void)loop);
+    if (e->writing) {
+        e->writing = 0;
+        ev_io_stop(EV_A_ &e->wev);
+    }
+}
+
+static void redisLibevStopTimer(void *privdata) {
+    redisLibevEvents *e = (redisLibevEvents*)privdata;
+    struct ev_loop *loop = e->loop;
+    ((void)loop);
+    ev_timer_stop(EV_A_ &e->timer);
+}
+
+static void redisLibevCleanup(void *privdata) {
+    redisLibevEvents *e = (redisLibevEvents*)privdata;
+    redisLibevDelRead(privdata);
+    redisLibevDelWrite(privdata);
+    redisLibevStopTimer(privdata);
+    hi_free(e);
+}
+
+static void redisLibevTimeout(EV_P_ ev_timer *timer, int revents) {
+    ((void)revents);
+    redisLibevEvents *e = (redisLibevEvents*)timer->data;
+    redisAsyncHandleTimeout(e->context);
+}
+
+static void redisLibevSetTimeout(void *privdata, struct timeval tv) {
+    redisLibevEvents *e = (redisLibevEvents*)privdata;
+    struct ev_loop *loop = e->loop;
+    ((void)loop);
+
+    if (!ev_is_active(&e->timer)) {
+        ev_init(&e->timer, redisLibevTimeout);
+        e->timer.data = e;
+    }
+
+    e->timer.repeat = tv.tv_sec + tv.tv_usec / 1000000.00;
+    ev_timer_again(EV_A_ &e->timer);
+}
+
+static int redisLibevAttach(EV_P_ redisAsyncContext *ac) {
+    redisContext *c = &(ac->c);
+    redisLibevEvents *e;
+
+    /* Nothing should be attached when something is already attached */
+    if (ac->ev.data != NULL)
+        return REDIS_ERR;
+
+    /* Create container for context and r/w events */
+    e = (redisLibevEvents*)hi_calloc(1, sizeof(*e));
+    if (e == NULL)
+        return REDIS_ERR;
+
+    e->context = ac;
+#if EV_MULTIPLICITY
+    e->loop = loop;
+#else
+    e->loop = NULL;
+#endif
+    e->rev.data = e;
+    e->wev.data = e;
+
+    /* Register functions to start/stop listening for events */
+    ac->ev.addRead = redisLibevAddRead;
+    ac->ev.delRead = redisLibevDelRead;
+    ac->ev.addWrite = redisLibevAddWrite;
+    ac->ev.delWrite = redisLibevDelWrite;
+    ac->ev.cleanup = redisLibevCleanup;
+    ac->ev.scheduleTimer = redisLibevSetTimeout;
+    ac->ev.data = e;
+
+    /* Initialize read/write events */
+    ev_io_init(&e->rev,redisLibevReadEvent,c->fd,EV_READ);
+    ev_io_init(&e->wev,redisLibevWriteEvent,c->fd,EV_WRITE);
+    return REDIS_OK;
+}
+
+#endif
diff --git a/adapters/libevent.h b/adapters/libevent.h
new file mode 100644
index 000000000..9150979bc
--- /dev/null
+++ b/adapters/libevent.h
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_LIBEVENT_H__
+#define __HIREDIS_LIBEVENT_H__
+#include <event2/event.h>
+#include "../hiredis.h"
+#include "../async.h"
+
+#define REDIS_LIBEVENT_DELETED 0x01
+#define REDIS_LIBEVENT_ENTERED 0x02
+
+typedef struct redisLibeventEvents {
+    redisAsyncContext *context;
+    struct event *ev;
+    struct event_base *base;
+    struct timeval tv;
+    short flags;
+    short state;
+} redisLibeventEvents;
+
+static void redisLibeventDestroy(redisLibeventEvents *e) {
+    hi_free(e);
+}
+
+static void redisLibeventHandler(int fd, short event, void *arg) {
+    ((void)fd);
+    redisLibeventEvents *e = (redisLibeventEvents*)arg;
+    e->state |= REDIS_LIBEVENT_ENTERED;
+
+    #define CHECK_DELETED() if (e->state & REDIS_LIBEVENT_DELETED) {\
+        redisLibeventDestroy(e);\
+        return; \
+    }
+
+    if ((event & EV_TIMEOUT) && (e->state & REDIS_LIBEVENT_DELETED) == 0) {
+        redisAsyncHandleTimeout(e->context);
+        CHECK_DELETED();
+    }
+
+    if ((event & EV_READ) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) {
+        redisAsyncHandleRead(e->context);
+        CHECK_DELETED();
+    }
+
+    if ((event & EV_WRITE) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) {
+        redisAsyncHandleWrite(e->context);
+        CHECK_DELETED();
+    }
+
+    e->state &= ~REDIS_LIBEVENT_ENTERED;
+    #undef CHECK_DELETED
+}
+
+static void redisLibeventUpdate(void *privdata, short flag, int isRemove) {
+    redisLibeventEvents *e = (redisLibeventEvents *)privdata;
+    const struct timeval *tv = e->tv.tv_sec || e->tv.tv_usec ? &e->tv : NULL;
+
+    if (isRemove) {
+        if ((e->flags & flag) == 0) {
+            return;
+        } else {
+            e->flags &= ~flag;
+        }
+    } else {
+        if (e->flags & flag) {
+            return;
+        } else {
+            e->flags |= flag;
+        }
+    }
+
+    event_del(e->ev);
+    event_assign(e->ev, e->base, e->context->c.fd, e->flags | EV_PERSIST,
+                 redisLibeventHandler, privdata);
+    event_add(e->ev, tv);
+}
+
+static void redisLibeventAddRead(void *privdata) {
+    redisLibeventUpdate(privdata, EV_READ, 0);
+}
+
+static void redisLibeventDelRead(void *privdata) {
+    redisLibeventUpdate(privdata, EV_READ, 1);
+}
+
+static void redisLibeventAddWrite(void *privdata) {
+    redisLibeventUpdate(privdata, EV_WRITE, 0);
+}
+
+static void redisLibeventDelWrite(void *privdata) {
+    redisLibeventUpdate(privdata, EV_WRITE, 1);
+}
+
+static void redisLibeventCleanup(void *privdata) {
+    redisLibeventEvents *e = (redisLibeventEvents*)privdata;
+    if (!e) {
+        return;
+    }
+    event_del(e->ev);
+    event_free(e->ev);
+    e->ev = NULL;
+
+    if (e->state & REDIS_LIBEVENT_ENTERED) {
+        e->state |= REDIS_LIBEVENT_DELETED;
+    } else {
+        redisLibeventDestroy(e);
+    }
+}
+
+static void redisLibeventSetTimeout(void *privdata, struct timeval tv) {
+    redisLibeventEvents *e = (redisLibeventEvents *)privdata;
+    short flags = e->flags;
+    e->flags = 0;
+    e->tv = tv;
+    redisLibeventUpdate(e, flags, 0);
+}
+
+static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) {
+    redisContext *c = &(ac->c);
+    redisLibeventEvents *e;
+
+    /* Nothing should be attached when something is already attached */
+    if (ac->ev.data != NULL)
+        return REDIS_ERR;
+
+    /* Create container for context and r/w events */
+    e = (redisLibeventEvents*)hi_calloc(1, sizeof(*e));
+    if (e == NULL)
+        return REDIS_ERR;
+
+    e->context = ac;
+
+    /* Register functions to start/stop listening for events */
+    ac->ev.addRead = redisLibeventAddRead;
+    ac->ev.delRead = redisLibeventDelRead;
+    ac->ev.addWrite = redisLibeventAddWrite;
+    ac->ev.delWrite = redisLibeventDelWrite;
+    ac->ev.cleanup = redisLibeventCleanup;
+    ac->ev.scheduleTimer = redisLibeventSetTimeout;
+    ac->ev.data = e;
+
+    /* Initialize and install read/write events */
+    e->ev = event_new(base, c->fd, EV_READ | EV_WRITE, redisLibeventHandler, e);
+    e->base = base;
+    return REDIS_OK;
+}
+#endif
diff --git a/adapters/libuv.h b/adapters/libuv.h
new file mode 100644
index 000000000..c120b1b39
--- /dev/null
+++ b/adapters/libuv.h
@@ -0,0 +1,117 @@
+#ifndef __HIREDIS_LIBUV_H__
+#define __HIREDIS_LIBUV_H__
+#include <stdlib.h>
+#include <uv.h>
+#include "../hiredis.h"
+#include "../async.h"
+#include <string.h>
+
+typedef struct redisLibuvEvents {
+  redisAsyncContext* context;
+  uv_poll_t          handle;
+  int                events;
+} redisLibuvEvents;
+
+
+static void redisLibuvPoll(uv_poll_t* handle, int status, int events) {
+  redisLibuvEvents* p = (redisLibuvEvents*)handle->data;
+  int ev = (status ? p->events : events);
+
+  if (p->context != NULL && (ev & UV_READABLE)) {
+    redisAsyncHandleRead(p->context);
+  }
+  if (p->context != NULL && (ev & UV_WRITABLE)) {
+    redisAsyncHandleWrite(p->context);
+  }
+}
+
+
+static void redisLibuvAddRead(void *privdata) {
+  redisLibuvEvents* p = (redisLibuvEvents*)privdata;
+
+  p->events |= UV_READABLE;
+
+  uv_poll_start(&p->handle, p->events, redisLibuvPoll);
+}
+
+
+static void redisLibuvDelRead(void *privdata) {
+  redisLibuvEvents* p = (redisLibuvEvents*)privdata;
+
+  p->events &= ~UV_READABLE;
+
+  if (p->events) {
+    uv_poll_start(&p->handle, p->events, redisLibuvPoll);
+  } else {
+    uv_poll_stop(&p->handle);
+  }
+}
+
+
+static void redisLibuvAddWrite(void *privdata) {
+  redisLibuvEvents* p = (redisLibuvEvents*)privdata;
+
+  p->events |= UV_WRITABLE;
+
+  uv_poll_start(&p->handle, p->events, redisLibuvPoll);
+}
+
+
+static void redisLibuvDelWrite(void *privdata) {
+  redisLibuvEvents* p = (redisLibuvEvents*)privdata;
+
+  p->events &= ~UV_WRITABLE;
+
+  if (p->events) {
+    uv_poll_start(&p->handle, p->events, redisLibuvPoll);
+  } else {
+    uv_poll_stop(&p->handle);
+  }
+}
+
+
+static void on_close(uv_handle_t* handle) {
+  redisLibuvEvents* p = (redisLibuvEvents*)handle->data;
+
+  hi_free(p);
+}
+
+
+static void redisLibuvCleanup(void *privdata) {
+  redisLibuvEvents* p = (redisLibuvEvents*)privdata;
+
+  p->context = NULL; // indicate that context might no longer exist
+  uv_close((uv_handle_t*)&p->handle, on_close);
+}
+
+
+static int redisLibuvAttach(redisAsyncContext* ac, uv_loop_t* loop) {
+  redisContext *c = &(ac->c);
+
+  if (ac->ev.data != NULL) {
+    return REDIS_ERR;
+  }
+
+  ac->ev.addRead  = redisLibuvAddRead;
+  ac->ev.delRead  = redisLibuvDelRead;
+  ac->ev.addWrite = redisLibuvAddWrite;
+  ac->ev.delWrite = redisLibuvDelWrite;
+  ac->ev.cleanup  = redisLibuvCleanup;
+
+  redisLibuvEvents* p = (redisLibuvEvents*)hi_malloc(sizeof(*p));
+  if (p == NULL)
+      return REDIS_ERR;
+
+  memset(p, 0, sizeof(*p));
+
+  if (uv_poll_init_socket(loop, &p->handle, c->fd) != 0) {
+    return REDIS_ERR;
+  }
+
+  ac->ev.data    = p;
+  p->handle.data = p;
+  p->context     = ac;
+
+  return REDIS_OK;
+}
+#endif
diff --git a/adapters/macosx.h b/adapters/macosx.h
new file mode 100644
index 000000000..3c87f1b2f
--- /dev/null
+++ b/adapters/macosx.h
@@ -0,0 +1,115 @@
+//
+//  Created by Дмитрий Бахвалов on 13.07.15.
+//  Copyright (c) 2015 Dmitry Bakhvalov. All rights reserved.
+//
+
+#ifndef __HIREDIS_MACOSX_H__
+#define __HIREDIS_MACOSX_H__
+
+#include <CoreFoundation/CoreFoundation.h>
+
+#include "../hiredis.h"
+#include "../async.h"
+
+typedef struct {
+    redisAsyncContext *context;
+    CFSocketRef socketRef;
+    CFRunLoopSourceRef sourceRef;
+} RedisRunLoop;
+
+static int freeRedisRunLoop(RedisRunLoop* redisRunLoop) {
+    if( redisRunLoop != NULL ) {
+        if( redisRunLoop->sourceRef != NULL ) {
+            CFRunLoopSourceInvalidate(redisRunLoop->sourceRef);
+            CFRelease(redisRunLoop->sourceRef);
+        }
+        if( redisRunLoop->socketRef != NULL ) {
+            CFSocketInvalidate(redisRunLoop->socketRef);
+            CFRelease(redisRunLoop->socketRef);
+        }
+        hi_free(redisRunLoop);
+    }
+    return REDIS_ERR;
+}
+
+static void redisMacOSAddRead(void *privdata) {
+    RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
+    CFSocketEnableCallBacks(redisRunLoop->socketRef, kCFSocketReadCallBack);
+}
+
+static void redisMacOSDelRead(void *privdata) {
+    RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
+    CFSocketDisableCallBacks(redisRunLoop->socketRef, kCFSocketReadCallBack);
+}
+
+static void redisMacOSAddWrite(void *privdata) {
+    RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
+    CFSocketEnableCallBacks(redisRunLoop->socketRef, kCFSocketWriteCallBack);
+}
+
+static void redisMacOSDelWrite(void *privdata) {
+    RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
+    CFSocketDisableCallBacks(redisRunLoop->socketRef, kCFSocketWriteCallBack);
+}
+
+static void redisMacOSCleanup(void *privdata) {
+    RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
+    freeRedisRunLoop(redisRunLoop);
+}
+
+static void redisMacOSAsyncCallback(CFSocketRef __unused s, CFSocketCallBackType callbackType, CFDataRef __unused address, const void __unused *data, void *info) {
+    redisAsyncContext* context = (redisAsyncContext*) info;
+
+    switch (callbackType) {
+        case kCFSocketReadCallBack:
+            redisAsyncHandleRead(context);
+            break;
+
+        case kCFSocketWriteCallBack:
+            redisAsyncHandleWrite(context);
+            break;
+
+        default:
+            break;
+    }
+}
+
+static int redisMacOSAttach(redisAsyncContext *redisAsyncCtx, CFRunLoopRef runLoop) {
+    redisContext *redisCtx = &(redisAsyncCtx->c);
+
+    /* Nothing should be attached when something is already attached */
+    if( redisAsyncCtx->ev.data != NULL ) return REDIS_ERR;
+
+    RedisRunLoop* redisRunLoop = (RedisRunLoop*) hi_calloc(1, sizeof(RedisRunLoop));
+    if (redisRunLoop == NULL)
+        return REDIS_ERR;
+
+    /* Setup redis stuff */
+    redisRunLoop->context = redisAsyncCtx;
+
+    redisAsyncCtx->ev.addRead  = redisMacOSAddRead;
+    redisAsyncCtx->ev.delRead  = redisMacOSDelRead;
+    redisAsyncCtx->ev.addWrite = redisMacOSAddWrite;
+    redisAsyncCtx->ev.delWrite = redisMacOSDelWrite;
+    redisAsyncCtx->ev.cleanup  = redisMacOSCleanup;
+    redisAsyncCtx->ev.data     = redisRunLoop;
+
+    /* Initialize and install read/write events */
+    CFSocketContext socketCtx = { 0, redisAsyncCtx, NULL, NULL, NULL };
+
+    redisRunLoop->socketRef = CFSocketCreateWithNative(NULL, redisCtx->fd,
+                                                       kCFSocketReadCallBack | kCFSocketWriteCallBack,
+                                                       redisMacOSAsyncCallback,
+                                                       &socketCtx);
+    if( !redisRunLoop->socketRef ) return freeRedisRunLoop(redisRunLoop);
+
+    redisRunLoop->sourceRef = CFSocketCreateRunLoopSource(NULL, redisRunLoop->socketRef, 0);
+    if( !redisRunLoop->sourceRef ) return freeRedisRunLoop(redisRunLoop);
+
+    CFRunLoopAddSource(runLoop, redisRunLoop->sourceRef, kCFRunLoopDefaultMode);
+
+    return REDIS_OK;
+}
+
+#endif
+
diff --git a/adapters/qt.h b/adapters/qt.h
new file mode 100644
index 000000000..5cc02e6ce
--- /dev/null
+++ b/adapters/qt.h
@@ -0,0 +1,135 @@
+/*-
+ * Copyright (C) 2014 Pietro Cerutti <gahr@gahr.ch>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_QT_H__
+#define __HIREDIS_QT_H__
+#include <QSocketNotifier>
+#include "../async.h"
+
+static void RedisQtAddRead(void *);
+static void RedisQtDelRead(void *);
+static void RedisQtAddWrite(void *);
+static void RedisQtDelWrite(void *);
+static void RedisQtCleanup(void *);
+
+class RedisQtAdapter : public QObject {
+
+    Q_OBJECT
+
+    friend
+    void RedisQtAddRead(void * adapter) {
+        RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
+        a->addRead();
+    }
+
+    friend
+    void RedisQtDelRead(void * adapter) {
+        RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
+        a->delRead();
+    }
+
+    friend
+    void RedisQtAddWrite(void * adapter) {
+        RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
+        a->addWrite();
+    }
+
+    friend
+    void RedisQtDelWrite(void * adapter) {
+        RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
+        a->delWrite();
+    }
+
+    friend
+    void RedisQtCleanup(void * adapter) {
+        RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
+        a->cleanup();
+    }
+
+    public:
+        RedisQtAdapter(QObject * parent = 0)
+            : QObject(parent), m_ctx(0), m_read(0), m_write(0) { }
+
+        ~RedisQtAdapter() {
+            if (m_ctx != 0) {
+                m_ctx->ev.data = NULL;
+            }
+        }
+
+        int setContext(redisAsyncContext * ac) {
+            if (ac->ev.data != NULL) {
+                return REDIS_ERR;
+            }
+            m_ctx = ac;
+            m_ctx->ev.data = this;
+            m_ctx->ev.addRead = RedisQtAddRead;
+            m_ctx->ev.delRead = RedisQtDelRead;
+            m_ctx->ev.addWrite = RedisQtAddWrite;
+            m_ctx->ev.delWrite = RedisQtDelWrite;
+            m_ctx->ev.cleanup = RedisQtCleanup;
+            return REDIS_OK;
+        }
+
+    private:
+        void addRead() {
+            if (m_read) return;
+            m_read = new QSocketNotifier(m_ctx->c.fd, QSocketNotifier::Read, 0);
+            connect(m_read, SIGNAL(activated(int)), this, SLOT(read()));
+        }
+
+        void delRead() {
+            if (!m_read) return;
+            delete m_read;
+            m_read = 0;
+        }
+
+        void addWrite() {
+            if (m_write) return;
+            m_write = new QSocketNotifier(m_ctx->c.fd, QSocketNotifier::Write, 0);
+            connect(m_write, SIGNAL(activated(int)), this, SLOT(write()));
+        }
+
+        void delWrite() {
+            if (!m_write) return;
+            delete m_write;
+            m_write = 0;
+        }
+
+        void cleanup() {
+            delRead();
+            delWrite();
+        }
+
+    private slots:
+        void read() { redisAsyncHandleRead(m_ctx); }
+        void write() { redisAsyncHandleWrite(m_ctx); }
+
+    private:
+        redisAsyncContext * m_ctx;
+        QSocketNotifier * m_read;
+        QSocketNotifier * m_write;
+};
+
+#endif /* !__HIREDIS_QT_H__ */
diff --git a/alloc.c b/alloc.c
new file mode 100644
index 000000000..7fb6b35e7
--- /dev/null
+++ b/alloc.c
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2020, Michael Grunder <michael dot grunder at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fmacros.h"
+#include "alloc.h"
+#include <string.h>
+#include <stdlib.h>
+
+hiredisAllocFuncs hiredisAllocFns = {
+    .mallocFn = malloc,
+    .callocFn = calloc,
+    .reallocFn = realloc,
+    .strdupFn = strdup,
+    .freeFn = free,
+};
+
+/* Override hiredis' allocators with ones supplied by the user */
+hiredisAllocFuncs hiredisSetAllocators(hiredisAllocFuncs *override) {
+    hiredisAllocFuncs orig = hiredisAllocFns;
+
+    hiredisAllocFns = *override;
+
+    return orig;
+}
+
+/* Reset allocators to use libc defaults */
+void hiredisResetAllocators(void) {
+    hiredisAllocFns = (hiredisAllocFuncs) {
+        .mallocFn = malloc,
+        .callocFn = calloc,
+        .reallocFn = realloc,
+        .strdupFn = strdup,
+        .freeFn = free,
+    };
+}
+
+#ifdef _WIN32
+
+void *hi_malloc(size_t size) {
+    return hiredisAllocFns.mallocFn(size);
+}
+
+void *hi_calloc(size_t nmemb, size_t size) {
+    return hiredisAllocFns.callocFn(nmemb, size);
+}
+
+void *hi_realloc(void *ptr, size_t size) {
+    return hiredisAllocFns.reallocFn(ptr, size);
+}
+
+char *hi_strdup(const char *str) {
+    return hiredisAllocFns.strdupFn(str);
+}
+
+void hi_free(void *ptr) {
+    hiredisAllocFns.freeFn(ptr);
+}
+
+#endif
diff --git a/alloc.h b/alloc.h
new file mode 100644
index 000000000..34a05f49f
--- /dev/null
+++ b/alloc.h
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2020, Michael Grunder <michael dot grunder at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef HIREDIS_ALLOC_H
+#define HIREDIS_ALLOC_H
+
+#include <stddef.h> /* for size_t */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Structure pointing to our actually configured allocators */
+typedef struct hiredisAllocFuncs {
+    void *(*mallocFn)(size_t);
+    void *(*callocFn)(size_t,size_t);
+    void *(*reallocFn)(void*,size_t);
+    char *(*strdupFn)(const char*);
+    void (*freeFn)(void*);
+} hiredisAllocFuncs;
+
+hiredisAllocFuncs hiredisSetAllocators(hiredisAllocFuncs *ha);
+void hiredisResetAllocators(void);
+
+#ifndef _WIN32
+
+/* Hiredis' configured allocator function pointer struct */
+extern hiredisAllocFuncs hiredisAllocFns;
+
+static inline void *hi_malloc(size_t size) {
+    return hiredisAllocFns.mallocFn(size);
+}
+
+static inline void *hi_calloc(size_t nmemb, size_t size) {
+    return hiredisAllocFns.callocFn(nmemb, size);
+}
+
+static inline void *hi_realloc(void *ptr, size_t size) {
+    return hiredisAllocFns.reallocFn(ptr, size);
+}
+
+static inline char *hi_strdup(const char *str) {
+    return hiredisAllocFns.strdupFn(str);
+}
+
+static inline void hi_free(void *ptr) {
+    hiredisAllocFns.freeFn(ptr);
+}
+
+#else
+
+void *hi_malloc(size_t size);
+void *hi_calloc(size_t nmemb, size_t size);
+void *hi_realloc(void *ptr, size_t size);
+char *hi_strdup(const char *str);
+void hi_free(void *ptr);
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* HIREDIS_ALLOC_H */
diff --git a/appveyor.yml b/appveyor.yml
new file mode 100644
index 000000000..5b43fdbeb
--- /dev/null
+++ b/appveyor.yml
@@ -0,0 +1,24 @@
+# Appveyor configuration file for CI build of hiredis on Windows (under Cygwin)
+environment:
+    matrix:
+        - CYG_BASH: C:\cygwin64\bin\bash
+          CC: gcc
+        - CYG_BASH: C:\cygwin\bin\bash
+          CC: gcc
+          CFLAGS: -m32
+          CXXFLAGS: -m32
+          LDFLAGS: -m32
+
+clone_depth: 1
+
+# Attempt to ensure we don't try to convert line endings to Win32 CRLF as this will cause build to fail
+init:
+    - git config --global core.autocrlf input
+
+# Install needed build dependencies
+install:
+    - '%CYG_BASH% -lc "cygcheck -dc cygwin"'
+
+build_script:
+    - 'echo building...'
+    - '%CYG_BASH% -lc "cd $APPVEYOR_BUILD_FOLDER; exec 0</dev/null; mkdir build && cd build && cmake .. -G \"Unix Makefiles\" && make VERBOSE=1"'
diff --git a/async.c b/async.c
new file mode 100644
index 000000000..020cb09b9
--- /dev/null
+++ b/async.c
@@ -0,0 +1,887 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fmacros.h"
+#include "alloc.h"
+#include <stdlib.h>
+#include <string.h>
+#ifndef _MSC_VER
+#include <strings.h>
+#endif
+#include <assert.h>
+#include <ctype.h>
+#include <errno.h>
+#include "async.h"
+#include "net.h"
+#include "dict.c"
+#include "sds.h"
+#include "win32.h"
+
+#include "async_private.h"
+
+/* Forward declarations of hiredis.c functions */
+int __redisAppendCommand(redisContext *c, const char *cmd, size_t len);
+void __redisSetError(redisContext *c, int type, const char *str);
+
+/* Functions managing dictionary of callbacks for pub/sub. */
+static unsigned int callbackHash(const void *key) {
+    return dictGenHashFunction((const unsigned char *)key,
+                               sdslen((const sds)key));
+}
+
+static void *callbackValDup(void *privdata, const void *src) {
+    ((void) privdata);
+    redisCallback *dup;
+
+    dup = hi_malloc(sizeof(*dup));
+    if (dup == NULL)
+        return NULL;
+
+    memcpy(dup,src,sizeof(*dup));
+    return dup;
+}
+
+static int callbackKeyCompare(void *privdata, const void *key1, const void *key2) {
+    int l1, l2;
+    ((void) privdata);
+
+    l1 = sdslen((const sds)key1);
+    l2 = sdslen((const sds)key2);
+    if (l1 != l2) return 0;
+    return memcmp(key1,key2,l1) == 0;
+}
+
+static void callbackKeyDestructor(void *privdata, void *key) {
+    ((void) privdata);
+    sdsfree((sds)key);
+}
+
+static void callbackValDestructor(void *privdata, void *val) {
+    ((void) privdata);
+    hi_free(val);
+}
+
+static dictType callbackDict = {
+    callbackHash,
+    NULL,
+    callbackValDup,
+    callbackKeyCompare,
+    callbackKeyDestructor,
+    callbackValDestructor
+};
+
+static redisAsyncContext *redisAsyncInitialize(redisContext *c) {
+    redisAsyncContext *ac;
+    dict *channels = NULL, *patterns = NULL;
+
+    channels = dictCreate(&callbackDict,NULL);
+    if (channels == NULL)
+        goto oom;
+
+    patterns = dictCreate(&callbackDict,NULL);
+    if (patterns == NULL)
+        goto oom;
+
+    ac = hi_realloc(c,sizeof(redisAsyncContext));
+    if (ac == NULL)
+        goto oom;
+
+    c = &(ac->c);
+
+    /* The regular connect functions will always set the flag REDIS_CONNECTED.
+     * For the async API, we want to wait until the first write event is
+     * received up before setting this flag, so reset it here. */
+    c->flags &= ~REDIS_CONNECTED;
+
+    ac->err = 0;
+    ac->errstr = NULL;
+    ac->data = NULL;
+    ac->dataCleanup = NULL;
+
+    ac->ev.data = NULL;
+    ac->ev.addRead = NULL;
+    ac->ev.delRead = NULL;
+    ac->ev.addWrite = NULL;
+    ac->ev.delWrite = NULL;
+    ac->ev.cleanup = NULL;
+    ac->ev.scheduleTimer = NULL;
+
+    ac->onConnect = NULL;
+    ac->onDisconnect = NULL;
+
+    ac->replies.head = NULL;
+    ac->replies.tail = NULL;
+    ac->sub.invalid.head = NULL;
+    ac->sub.invalid.tail = NULL;
+    ac->sub.channels = channels;
+    ac->sub.patterns = patterns;
+
+    return ac;
+oom:
+    if (channels) dictRelease(channels);
+    if (patterns) dictRelease(patterns);
+    return NULL;
+}
+
+/* We want the error field to be accessible directly instead of requiring
+ * an indirection to the redisContext struct. */
+static void __redisAsyncCopyError(redisAsyncContext *ac) {
+    if (!ac)
+        return;
+
+    redisContext *c = &(ac->c);
+    ac->err = c->err;
+    ac->errstr = c->errstr;
+}
+
+redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options) {
+    redisOptions myOptions = *options;
+    redisContext *c;
+    redisAsyncContext *ac;
+
+    /* Clear any erroneously set sync callback and flag that we don't want to
+     * use freeReplyObject by default. */
+    myOptions.push_cb = NULL;
+    myOptions.options |= REDIS_OPT_NO_PUSH_AUTOFREE;
+
+    myOptions.options |= REDIS_OPT_NONBLOCK;
+    c = redisConnectWithOptions(&myOptions);
+    if (c == NULL) {
+        return NULL;
+    }
+
+    ac = redisAsyncInitialize(c);
+    if (ac == NULL) {
+        redisFree(c);
+        return NULL;
+    }
+
+    /* Set any configured async push handler */
+    redisAsyncSetPushCallback(ac, myOptions.async_push_cb);
+
+    __redisAsyncCopyError(ac);
+    return ac;
+}
+
+redisAsyncContext *redisAsyncConnect(const char *ip, int port) {
+    redisOptions options = {0};
+    REDIS_OPTIONS_SET_TCP(&options, ip, port);
+    return redisAsyncConnectWithOptions(&options);
+}
+
+redisAsyncContext *redisAsyncConnectBind(const char *ip, int port,
+                                         const char *source_addr) {
+    redisOptions options = {0};
+    REDIS_OPTIONS_SET_TCP(&options, ip, port);
+    options.endpoint.tcp.source_addr = source_addr;
+    return redisAsyncConnectWithOptions(&options);
+}
+
+redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port,
+                                                  const char *source_addr) {
+    redisOptions options = {0};
+    REDIS_OPTIONS_SET_TCP(&options, ip, port);
+    options.options |= REDIS_OPT_REUSEADDR;
+    options.endpoint.tcp.source_addr = source_addr;
+    return redisAsyncConnectWithOptions(&options);
+}
+
+redisAsyncContext *redisAsyncConnectUnix(const char *path) {
+    redisOptions options = {0};
+    REDIS_OPTIONS_SET_UNIX(&options, path);
+    return redisAsyncConnectWithOptions(&options);
+}
+
+int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn) {
+    if (ac->onConnect == NULL) {
+        ac->onConnect = fn;
+
+        /* The common way to detect an established connection is to wait for
+         * the first write event to be fired. This assumes the related event
+         * library functions are already set. */
+        _EL_ADD_WRITE(ac);
+        return REDIS_OK;
+    }
+    return REDIS_ERR;
+}
+
+int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn) {
+    if (ac->onDisconnect == NULL) {
+        ac->onDisconnect = fn;
+        return REDIS_OK;
+    }
+    return REDIS_ERR;
+}
+
+/* Helper functions to push/shift callbacks */
+static int __redisPushCallback(redisCallbackList *list, redisCallback *source) {
+    redisCallback *cb;
+
+    /* Copy callback from stack to heap */
+    cb = hi_malloc(sizeof(*cb));
+    if (cb == NULL)
+        return REDIS_ERR_OOM;
+
+    if (source != NULL) {
+        memcpy(cb,source,sizeof(*cb));
+        cb->next = NULL;
+    }
+
+    /* Store callback in list */
+    if (list->head == NULL)
+        list->head = cb;
+    if (list->tail != NULL)
+        list->tail->next = cb;
+    list->tail = cb;
+    return REDIS_OK;
+}
+
+static int __redisShiftCallback(redisCallbackList *list, redisCallback *target) {
+    redisCallback *cb = list->head;
+    if (cb != NULL) {
+        list->head = cb->next;
+        if (cb == list->tail)
+            list->tail = NULL;
+
+        /* Copy callback from heap to stack */
+        if (target != NULL)
+            memcpy(target,cb,sizeof(*cb));
+        hi_free(cb);
+        return REDIS_OK;
+    }
+    return REDIS_ERR;
+}
+
+static void __redisRunCallback(redisAsyncContext *ac, redisCallback *cb, redisReply *reply) {
+    redisContext *c = &(ac->c);
+    if (cb->fn != NULL) {
+        c->flags |= REDIS_IN_CALLBACK;
+        cb->fn(ac,reply,cb->privdata);
+        c->flags &= ~REDIS_IN_CALLBACK;
+    }
+}
+
+static void __redisRunPushCallback(redisAsyncContext *ac, redisReply *reply) {
+    if (ac->push_cb != NULL) {
+        ac->c.flags |= REDIS_IN_CALLBACK;
+        ac->push_cb(ac, reply);
+        ac->c.flags &= ~REDIS_IN_CALLBACK;
+    }
+}
+
+/* Helper function to free the context. */
+static void __redisAsyncFree(redisAsyncContext *ac) {
+    redisContext *c = &(ac->c);
+    redisCallback cb;
+    dictIterator *it;
+    dictEntry *de;
+
+    /* Execute pending callbacks with NULL reply. */
+    while (__redisShiftCallback(&ac->replies,&cb) == REDIS_OK)
+        __redisRunCallback(ac,&cb,NULL);
+
+    /* Execute callbacks for invalid commands */
+    while (__redisShiftCallback(&ac->sub.invalid,&cb) == REDIS_OK)
+        __redisRunCallback(ac,&cb,NULL);
+
+    /* Run subscription callbacks with NULL reply */
+    if (ac->sub.channels) {
+        it = dictGetIterator(ac->sub.channels);
+        if (it != NULL) {
+            while ((de = dictNext(it)) != NULL)
+                __redisRunCallback(ac,dictGetEntryVal(de),NULL);
+            dictReleaseIterator(it);
+        }
+
+        dictRelease(ac->sub.channels);
+    }
+
+    if (ac->sub.patterns) {
+        it = dictGetIterator(ac->sub.patterns);
+        if (it != NULL) {
+            while ((de = dictNext(it)) != NULL)
+                __redisRunCallback(ac,dictGetEntryVal(de),NULL);
+            dictReleaseIterator(it);
+        }
+
+        dictRelease(ac->sub.patterns);
+    }
+
+    /* Signal event lib to clean up */
+    _EL_CLEANUP(ac);
+
+    /* Execute disconnect callback. When redisAsyncFree() initiated destroying
+     * this context, the status will always be REDIS_OK. */
+    if (ac->onDisconnect && (c->flags & REDIS_CONNECTED)) {
+        if (c->flags & REDIS_FREEING) {
+            ac->onDisconnect(ac,REDIS_OK);
+        } else {
+            ac->onDisconnect(ac,(ac->err == 0) ? REDIS_OK : REDIS_ERR);
+        }
+    }
+
+    if (ac->dataCleanup) {
+        ac->dataCleanup(ac->data);
+    }
+
+    /* Cleanup self */
+    redisFree(c);
+}
+
+/* Free the async context. When this function is called from a callback,
+ * control needs to be returned to redisProcessCallbacks() before actual
+ * free'ing. To do so, a flag is set on the context which is picked up by
+ * redisProcessCallbacks(). Otherwise, the context is immediately free'd. */
+void redisAsyncFree(redisAsyncContext *ac) {
+    redisContext *c = &(ac->c);
+    c->flags |= REDIS_FREEING;
+    if (!(c->flags & REDIS_IN_CALLBACK))
+        __redisAsyncFree(ac);
+}
+
+/* Helper function to make the disconnect happen and clean up. */
+void __redisAsyncDisconnect(redisAsyncContext *ac) {
+    redisContext *c = &(ac->c);
+
+    /* Make sure error is accessible if there is any */
+    __redisAsyncCopyError(ac);
+
+    if (ac->err == 0) {
+        /* For clean disconnects, there should be no pending callbacks. */
+        int ret = __redisShiftCallback(&ac->replies,NULL);
+        assert(ret == REDIS_ERR);
+    } else {
+        /* Disconnection is caused by an error, make sure that pending
+         * callbacks cannot call new commands. */
+        c->flags |= REDIS_DISCONNECTING;
+    }
+
+    /* cleanup event library on disconnect.
+     * this is safe to call multiple times */
+    _EL_CLEANUP(ac);
+
+    /* For non-clean disconnects, __redisAsyncFree() will execute pending
+     * callbacks with a NULL-reply. */
+    if (!(c->flags & REDIS_NO_AUTO_FREE)) {
+      __redisAsyncFree(ac);
+    }
+}
+
+/* Tries to do a clean disconnect from Redis, meaning it stops new commands
+ * from being issued, but tries to flush the output buffer and execute
+ * callbacks for all remaining replies. When this function is called from a
+ * callback, there might be more replies and we can safely defer disconnecting
+ * to redisProcessCallbacks(). Otherwise, we can only disconnect immediately
+ * when there are no pending callbacks. */
+void redisAsyncDisconnect(redisAsyncContext *ac) {
+    redisContext *c = &(ac->c);
+    c->flags |= REDIS_DISCONNECTING;
+
+    /** unset the auto-free flag here, because disconnect undoes this */
+    c->flags &= ~REDIS_NO_AUTO_FREE;
+    if (!(c->flags & REDIS_IN_CALLBACK) && ac->replies.head == NULL)
+        __redisAsyncDisconnect(ac);
+}
+
+static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, redisCallback *dstcb) {
+    redisContext *c = &(ac->c);
+    dict *callbacks;
+    redisCallback *cb;
+    dictEntry *de;
+    int pvariant;
+    char *stype;
+    sds sname;
+
+    /* Custom reply functions are not supported for pub/sub. This will fail
+     * very hard when they are used... */
+    if (reply->type == REDIS_REPLY_ARRAY || reply->type == REDIS_REPLY_PUSH) {
+        assert(reply->elements >= 2);
+        assert(reply->element[0]->type == REDIS_REPLY_STRING);
+        stype = reply->element[0]->str;
+        pvariant = (tolower(stype[0]) == 'p') ? 1 : 0;
+
+        if (pvariant)
+            callbacks = ac->sub.patterns;
+        else
+            callbacks = ac->sub.channels;
+
+        /* Locate the right callback */
+        assert(reply->element[1]->type == REDIS_REPLY_STRING);
+        sname = sdsnewlen(reply->element[1]->str,reply->element[1]->len);
+        if (sname == NULL)
+            goto oom;
+
+        de = dictFind(callbacks,sname);
+        if (de != NULL) {
+            cb = dictGetEntryVal(de);
+
+            /* If this is an subscribe reply decrease pending counter. */
+            if (strcasecmp(stype+pvariant,"subscribe") == 0) {
+                cb->pending_subs -= 1;
+            }
+
+            memcpy(dstcb,cb,sizeof(*dstcb));
+
+            /* If this is an unsubscribe message, remove it. */
+            if (strcasecmp(stype+pvariant,"unsubscribe") == 0) {
+                if (cb->pending_subs == 0)
+                    dictDelete(callbacks,sname);
+
+                /* If this was the last unsubscribe message, revert to
+                 * non-subscribe mode. */
+                assert(reply->element[2]->type == REDIS_REPLY_INTEGER);
+
+                /* Unset subscribed flag only when no pipelined pending subscribe. */
+                if (reply->element[2]->integer == 0
+                    && dictSize(ac->sub.channels) == 0
+                    && dictSize(ac->sub.patterns) == 0)
+                    c->flags &= ~REDIS_SUBSCRIBED;
+            }
+        }
+        sdsfree(sname);
+    } else {
+        /* Shift callback for invalid commands. */
+        __redisShiftCallback(&ac->sub.invalid,dstcb);
+    }
+    return REDIS_OK;
+oom:
+    __redisSetError(&(ac->c), REDIS_ERR_OOM, "Out of memory");
+    return REDIS_ERR;
+}
+
+#define redisIsSpontaneousPushReply(r) \
+    (redisIsPushReply(r) && !redisIsSubscribeReply(r))
+
+static int redisIsSubscribeReply(redisReply *reply) {
+    char *str;
+    size_t len, off;
+
+    /* We will always have at least one string with the subscribe/message type */
+    if (reply->elements < 1 || reply->element[0]->type != REDIS_REPLY_STRING ||
+        reply->element[0]->len < sizeof("message") - 1)
+    {
+        return 0;
+    }
+
+    /* Get the string/len moving past 'p' if needed */
+    off = tolower(reply->element[0]->str[0]) == 'p';
+    str = reply->element[0]->str + off;
+    len = reply->element[0]->len - off;
+
+    return !strncasecmp(str, "subscribe", len) ||
+           !strncasecmp(str, "message", len);
+
+}
+
+void redisProcessCallbacks(redisAsyncContext *ac) {
+    redisContext *c = &(ac->c);
+    redisCallback cb = {NULL, NULL, 0, NULL};
+    void *reply = NULL;
+    int status;
+
+    while((status = redisGetReply(c,&reply)) == REDIS_OK) {
+        if (reply == NULL) {
+            /* When the connection is being disconnected and there are
+             * no more replies, this is the cue to really disconnect. */
+            if (c->flags & REDIS_DISCONNECTING && sdslen(c->obuf) == 0
+                && ac->replies.head == NULL) {
+                __redisAsyncDisconnect(ac);
+                return;
+            }
+
+            /* If monitor mode, repush callback */
+            if(c->flags & REDIS_MONITORING) {
+                __redisPushCallback(&ac->replies,&cb);
+            }
+
+            /* When the connection is not being disconnected, simply stop
+             * trying to get replies and wait for the next loop tick. */
+            break;
+        }
+
+        /* Send any non-subscribe related PUSH messages to our PUSH handler
+         * while allowing subscribe related PUSH messages to pass through.
+         * This allows existing code to be backward compatible and work in
+         * either RESP2 or RESP3 mode. */
+        if (redisIsSpontaneousPushReply(reply)) {
+            __redisRunPushCallback(ac, reply);
+            c->reader->fn->freeObject(reply);
+            continue;
+        }
+
+        /* Even if the context is subscribed, pending regular
+         * callbacks will get a reply before pub/sub messages arrive. */
+        if (__redisShiftCallback(&ac->replies,&cb) != REDIS_OK) {
+            /*
+             * A spontaneous reply in a not-subscribed context can be the error
+             * reply that is sent when a new connection exceeds the maximum
+             * number of allowed connections on the server side.
+             *
+             * This is seen as an error instead of a regular reply because the
+             * server closes the connection after sending it.
+             *
+             * To prevent the error from being overwritten by an EOF error the
+             * connection is closed here. See issue #43.
+             *
+             * Another possibility is that the server is loading its dataset.
+             * In this case we also want to close the connection, and have the
+             * user wait until the server is ready to take our request.
+             */
+            if (((redisReply*)reply)->type == REDIS_REPLY_ERROR) {
+                c->err = REDIS_ERR_OTHER;
+                snprintf(c->errstr,sizeof(c->errstr),"%s",((redisReply*)reply)->str);
+                c->reader->fn->freeObject(reply);
+                __redisAsyncDisconnect(ac);
+                return;
+            }
+            /* No more regular callbacks and no errors, the context *must* be subscribed or monitoring. */
+            assert((c->flags & REDIS_SUBSCRIBED || c->flags & REDIS_MONITORING));
+            if(c->flags & REDIS_SUBSCRIBED)
+                __redisGetSubscribeCallback(ac,reply,&cb);
+        }
+
+        if (cb.fn != NULL) {
+            __redisRunCallback(ac,&cb,reply);
+            c->reader->fn->freeObject(reply);
+
+            /* Proceed with free'ing when redisAsyncFree() was called. */
+            if (c->flags & REDIS_FREEING) {
+                __redisAsyncFree(ac);
+                return;
+            }
+        } else {
+            /* No callback for this reply. This can either be a NULL callback,
+             * or there were no callbacks to begin with. Either way, don't
+             * abort with an error, but simply ignore it because the client
+             * doesn't know what the server will spit out over the wire. */
+            c->reader->fn->freeObject(reply);
+        }
+    }
+
+    /* Disconnect when there was an error reading the reply */
+    if (status != REDIS_OK)
+        __redisAsyncDisconnect(ac);
+}
+
+static void __redisAsyncHandleConnectFailure(redisAsyncContext *ac) {
+    if (ac->onConnect) ac->onConnect(ac, REDIS_ERR);
+    __redisAsyncDisconnect(ac);
+}
+
+/* Internal helper function to detect socket status the first time a read or
+ * write event fires. When connecting was not successful, the connect callback
+ * is called with a REDIS_ERR status and the context is free'd. */
+static int __redisAsyncHandleConnect(redisAsyncContext *ac) {
+    int completed = 0;
+    redisContext *c = &(ac->c);
+
+    if (redisCheckConnectDone(c, &completed) == REDIS_ERR) {
+        /* Error! */
+        redisCheckSocketError(c);
+        __redisAsyncHandleConnectFailure(ac);
+        return REDIS_ERR;
+    } else if (completed == 1) {
+        /* connected! */
+        if (c->connection_type == REDIS_CONN_TCP &&
+            redisSetTcpNoDelay(c) == REDIS_ERR) {
+            __redisAsyncHandleConnectFailure(ac);
+            return REDIS_ERR;
+        }
+
+        if (ac->onConnect) ac->onConnect(ac, REDIS_OK);
+        c->flags |= REDIS_CONNECTED;
+        return REDIS_OK;
+    } else {
+        return REDIS_OK;
+    }
+}
+
+void redisAsyncRead(redisAsyncContext *ac) {
+    redisContext *c = &(ac->c);
+
+    if (redisBufferRead(c) == REDIS_ERR) {
+        __redisAsyncDisconnect(ac);
+    } else {
+        /* Always re-schedule reads */
+        _EL_ADD_READ(ac);
+        redisProcessCallbacks(ac);
+    }
+}
+
+/* This function should be called when the socket is readable.
+ * It processes all replies that can be read and executes their callbacks.
+ */
+void redisAsyncHandleRead(redisAsyncContext *ac) {
+    redisContext *c = &(ac->c);
+
+    if (!(c->flags & REDIS_CONNECTED)) {
+        /* Abort connect was not successful. */
+        if (__redisAsyncHandleConnect(ac) != REDIS_OK)
+            return;
+        /* Try again later when the context is still not connected. */
+        if (!(c->flags & REDIS_CONNECTED))
+            return;
+    }
+
+    c->funcs->async_read(ac);
+}
+
+void redisAsyncWrite(redisAsyncContext *ac) {
+    redisContext *c = &(ac->c);
+    int done = 0;
+
+    if (redisBufferWrite(c,&done) == REDIS_ERR) {
+        __redisAsyncDisconnect(ac);
+    } else {
+        /* Continue writing when not done, stop writing otherwise */
+        if (!done)
+            _EL_ADD_WRITE(ac);
+        else
+            _EL_DEL_WRITE(ac);
+
+        /* Always schedule reads after writes */
+        _EL_ADD_READ(ac);
+    }
+}
+
+void redisAsyncHandleWrite(redisAsyncContext *ac) {
+    redisContext *c = &(ac->c);
+
+    if (!(c->flags & REDIS_CONNECTED)) {
+        /* Abort connect was not successful. */
+        if (__redisAsyncHandleConnect(ac) != REDIS_OK)
+            return;
+        /* Try again later when the context is still not connected. */
+        if (!(c->flags & REDIS_CONNECTED))
+            return;
+    }
+
+    c->funcs->async_write(ac);
+}
+
+void redisAsyncHandleTimeout(redisAsyncContext *ac) {
+    redisContext *c = &(ac->c);
+    redisCallback cb;
+
+    if ((c->flags & REDIS_CONNECTED) && ac->replies.head == NULL) {
+        /* Nothing to do - just an idle timeout */
+        return;
+    }
+
+    if (!c->err) {
+        __redisSetError(c, REDIS_ERR_TIMEOUT, "Timeout");
+    }
+
+    if (!(c->flags & REDIS_CONNECTED) && ac->onConnect) {
+        ac->onConnect(ac, REDIS_ERR);
+    }
+
+    while (__redisShiftCallback(&ac->replies, &cb) == REDIS_OK) {
+        __redisRunCallback(ac, &cb, NULL);
+    }
+
+    /**
+     * TODO: Don't automatically sever the connection,
+     * rather, allow to ignore <x> responses before the queue is clear
+     */
+    __redisAsyncDisconnect(ac);
+}
+
+/* Sets a pointer to the first argument and its length starting at p. Returns
+ * the number of bytes to skip to get to the following argument. */
+static const char *nextArgument(const char *start, const char **str, size_t *len) {
+    const char *p = start;
+    if (p[0] != '$') {
+        p = strchr(p,'$');
+        if (p == NULL) return NULL;
+    }
+
+    *len = (int)strtol(p+1,NULL,10);
+    p = strchr(p,'\r');
+    assert(p);
+    *str = p+2;
+    return p+2+(*len)+2;
+}
+
+/* Helper function for the redisAsyncCommand* family of functions. Writes a
+ * formatted command to the output buffer and registers the provided callback
+ * function with the context. */
+static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) {
+    redisContext *c = &(ac->c);
+    redisCallback cb;
+    struct dict *cbdict;
+    dictEntry *de;
+    redisCallback *existcb;
+    int pvariant, hasnext;
+    const char *cstr, *astr;
+    size_t clen, alen;
+    const char *p;
+    sds sname;
+    int ret;
+
+    /* Don't accept new commands when the connection is about to be closed. */
+    if (c->flags & (REDIS_DISCONNECTING | REDIS_FREEING)) return REDIS_ERR;
+
+    /* Setup callback */
+    cb.fn = fn;
+    cb.privdata = privdata;
+    cb.pending_subs = 1;
+
+    /* Find out which command will be appended. */
+    p = nextArgument(cmd,&cstr,&clen);
+    assert(p != NULL);
+    hasnext = (p[0] == '$');
+    pvariant = (tolower(cstr[0]) == 'p') ? 1 : 0;
+    cstr += pvariant;
+    clen -= pvariant;
+
+    if (hasnext && strncasecmp(cstr,"subscribe\r\n",11) == 0) {
+        c->flags |= REDIS_SUBSCRIBED;
+
+        /* Add every channel/pattern to the list of subscription callbacks. */
+        while ((p = nextArgument(p,&astr,&alen)) != NULL) {
+            sname = sdsnewlen(astr,alen);
+            if (sname == NULL)
+                goto oom;
+
+            if (pvariant)
+                cbdict = ac->sub.patterns;
+            else
+                cbdict = ac->sub.channels;
+
+            de = dictFind(cbdict,sname);
+
+            if (de != NULL) {
+                existcb = dictGetEntryVal(de);
+                cb.pending_subs = existcb->pending_subs + 1;
+            }
+
+            ret = dictReplace(cbdict,sname,&cb);
+
+            if (ret == 0) sdsfree(sname);
+        }
+    } else if (strncasecmp(cstr,"unsubscribe\r\n",13) == 0) {
+        /* It is only useful to call (P)UNSUBSCRIBE when the context is
+         * subscribed to one or more channels or patterns. */
+        if (!(c->flags & REDIS_SUBSCRIBED)) return REDIS_ERR;
+
+        /* (P)UNSUBSCRIBE does not have its own response: every channel or
+         * pattern that is unsubscribed will receive a message. This means we
+         * should not append a callback function for this command. */
+     } else if(strncasecmp(cstr,"monitor\r\n",9) == 0) {
+         /* Set monitor flag and push callback */
+         c->flags |= REDIS_MONITORING;
+         __redisPushCallback(&ac->replies,&cb);
+    } else {
+        if (c->flags & REDIS_SUBSCRIBED)
+            /* This will likely result in an error reply, but it needs to be
+             * received and passed to the callback. */
+            __redisPushCallback(&ac->sub.invalid,&cb);
+        else
+            __redisPushCallback(&ac->replies,&cb);
+    }
+
+    __redisAppendCommand(c,cmd,len);
+
+    /* Always schedule a write when the write buffer is non-empty */
+    _EL_ADD_WRITE(ac);
+
+    return REDIS_OK;
+oom:
+    __redisSetError(&(ac->c), REDIS_ERR_OOM, "Out of memory");
+    return REDIS_ERR;
+}
+
+int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap) {
+    char *cmd;
+    int len;
+    int status;
+    len = redisvFormatCommand(&cmd,format,ap);
+
+    /* We don't want to pass -1 or -2 to future functions as a length. */
+    if (len < 0)
+        return REDIS_ERR;
+
+    status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
+    hi_free(cmd);
+    return status;
+}
+
+int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...) {
+    va_list ap;
+    int status;
+    va_start(ap,format);
+    status = redisvAsyncCommand(ac,fn,privdata,format,ap);
+    va_end(ap);
+    return status;
+}
+
+int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen) {
+    sds cmd;
+    int len;
+    int status;
+    len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen);
+    if (len < 0)
+        return REDIS_ERR;
+    status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
+    sdsfree(cmd);
+    return status;
+}
+
+int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) {
+    int status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
+    return status;
+}
+
+redisAsyncPushFn *redisAsyncSetPushCallback(redisAsyncContext *ac, redisAsyncPushFn *fn) {
+    redisAsyncPushFn *old = ac->push_cb;
+    ac->push_cb = fn;
+    return old;
+}
+
+int redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv) {
+    if (!ac->c.command_timeout) {
+        ac->c.command_timeout = hi_calloc(1, sizeof(tv));
+        if (ac->c.command_timeout == NULL) {
+            __redisSetError(&ac->c, REDIS_ERR_OOM, "Out of memory");
+            __redisAsyncCopyError(ac);
+            return REDIS_ERR;
+        }
+    }
+
+    if (tv.tv_sec != ac->c.command_timeout->tv_sec ||
+        tv.tv_usec != ac->c.command_timeout->tv_usec)
+    {
+        *ac->c.command_timeout = tv;
+    }
+
+    return REDIS_OK;
+}
diff --git a/async.h b/async.h
new file mode 100644
index 000000000..b1d2cb263
--- /dev/null
+++ b/async.h
@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_ASYNC_H
+#define __HIREDIS_ASYNC_H
+#include "hiredis.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct redisAsyncContext; /* need forward declaration of redisAsyncContext */
+struct dict; /* dictionary header is included in async.c */
+
+/* Reply callback prototype and container */
+typedef void (redisCallbackFn)(struct redisAsyncContext*, void*, void*);
+typedef struct redisCallback {
+    struct redisCallback *next; /* simple singly linked list */
+    redisCallbackFn *fn;
+    int pending_subs;
+    void *privdata;
+} redisCallback;
+
+/* List of callbacks for either regular replies or pub/sub */
+typedef struct redisCallbackList {
+    redisCallback *head, *tail;
+} redisCallbackList;
+
+/* Connection callback prototypes */
+typedef void (redisDisconnectCallback)(const struct redisAsyncContext*, int status);
+typedef void (redisConnectCallback)(const struct redisAsyncContext*, int status);
+typedef void(redisTimerCallback)(void *timer, void *privdata);
+
+/* Context for an async connection to Redis */
+typedef struct redisAsyncContext {
+    /* Hold the regular context, so it can be realloc'ed. */
+    redisContext c;
+
+    /* Setup error flags so they can be used directly. */
+    int err;
+    char *errstr;
+
+    /* Not used by hiredis */
+    void *data;
+    void (*dataCleanup)(void *privdata);
+
+    /* Event library data and hooks */
+    struct {
+        void *data;
+
+        /* Hooks that are called when the library expects to start
+         * reading/writing. These functions should be idempotent. */
+        void (*addRead)(void *privdata);
+        void (*delRead)(void *privdata);
+        void (*addWrite)(void *privdata);
+        void (*delWrite)(void *privdata);
+        void (*cleanup)(void *privdata);
+        void (*scheduleTimer)(void *privdata, struct timeval tv);
+    } ev;
+
+    /* Called when either the connection is terminated due to an error or per
+     * user request. The status is set accordingly (REDIS_OK, REDIS_ERR). */
+    redisDisconnectCallback *onDisconnect;
+
+    /* Called when the first write event was received. */
+    redisConnectCallback *onConnect;
+
+    /* Regular command callbacks */
+    redisCallbackList replies;
+
+    /* Address used for connect() */
+    struct sockaddr *saddr;
+    size_t addrlen;
+
+    /* Subscription callbacks */
+    struct {
+        redisCallbackList invalid;
+        struct dict *channels;
+        struct dict *patterns;
+    } sub;
+
+    /* Any configured RESP3 PUSH handler */
+    redisAsyncPushFn *push_cb;
+} redisAsyncContext;
+
+/* Functions that proxy to hiredis */
+redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options);
+redisAsyncContext *redisAsyncConnect(const char *ip, int port);
+redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr);
+redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port,
+                                                  const char *source_addr);
+redisAsyncContext *redisAsyncConnectUnix(const char *path);
+int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn);
+int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn);
+
+redisAsyncPushFn *redisAsyncSetPushCallback(redisAsyncContext *ac, redisAsyncPushFn *fn);
+int redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv);
+void redisAsyncDisconnect(redisAsyncContext *ac);
+void redisAsyncFree(redisAsyncContext *ac);
+
+/* Handle read/write events */
+void redisAsyncHandleRead(redisAsyncContext *ac);
+void redisAsyncHandleWrite(redisAsyncContext *ac);
+void redisAsyncHandleTimeout(redisAsyncContext *ac);
+void redisAsyncRead(redisAsyncContext *ac);
+void redisAsyncWrite(redisAsyncContext *ac);
+
+/* Command functions for an async context. Write the command to the
+ * output buffer and register the provided callback. */
+int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap);
+int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...);
+int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen);
+int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/async_private.h b/async_private.h
new file mode 100644
index 000000000..b9d23fffd
--- /dev/null
+++ b/async_private.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_ASYNC_PRIVATE_H
+#define __HIREDIS_ASYNC_PRIVATE_H
+
+#define _EL_ADD_READ(ctx)                                         \
+    do {                                                          \
+        refreshTimeout(ctx);                                      \
+        if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \
+    } while (0)
+#define _EL_DEL_READ(ctx) do { \
+        if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \
+    } while(0)
+#define _EL_ADD_WRITE(ctx)                                          \
+    do {                                                            \
+        refreshTimeout(ctx);                                        \
+        if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \
+    } while (0)
+#define _EL_DEL_WRITE(ctx) do { \
+        if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \
+    } while(0)
+#define _EL_CLEANUP(ctx) do { \
+        if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \
+        ctx->ev.cleanup = NULL; \
+    } while(0);
+
+static inline void refreshTimeout(redisAsyncContext *ctx) {
+    #define REDIS_TIMER_ISSET(tvp) \
+        (tvp && ((tvp)->tv_sec || (tvp)->tv_usec))
+
+    #define REDIS_EL_TIMER(ac, tvp) \
+        if ((ac)->ev.scheduleTimer && REDIS_TIMER_ISSET(tvp)) { \
+            (ac)->ev.scheduleTimer((ac)->ev.data, *(tvp)); \
+        }
+
+    if (ctx->c.flags & REDIS_CONNECTED) {
+        REDIS_EL_TIMER(ctx, ctx->c.command_timeout);
+    } else {
+        REDIS_EL_TIMER(ctx, ctx->c.connect_timeout);
+    }
+}
+
+void __redisAsyncDisconnect(redisAsyncContext *ac);
+void redisProcessCallbacks(redisAsyncContext *ac);
+
+#endif  /* __HIREDIS_ASYNC_PRIVATE_H */
diff --git a/dict.c b/dict.c
new file mode 100644
index 000000000..34a33ead9
--- /dev/null
+++ b/dict.c
@@ -0,0 +1,352 @@
+/* Hash table implementation.
+ *
+ * This file implements in memory hash tables with insert/del/replace/find/
+ * get-random-element operations. Hash tables will auto resize if needed
+ * tables of power of two in size are used, collisions are handled by
+ * chaining. See the source code for more information... :)
+ *
+ * Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fmacros.h"
+#include "alloc.h"
+#include <stdlib.h>
+#include <assert.h>
+#include <limits.h>
+#include "dict.h"
+
+/* -------------------------- private prototypes ---------------------------- */
+
+static int _dictExpandIfNeeded(dict *ht);
+static unsigned long _dictNextPower(unsigned long size);
+static int _dictKeyIndex(dict *ht, const void *key);
+static int _dictInit(dict *ht, dictType *type, void *privDataPtr);
+
+/* -------------------------- hash functions -------------------------------- */
+
+/* Generic hash function (a popular one from Bernstein).
+ * I tested a few and this was the best. */
+static unsigned int dictGenHashFunction(const unsigned char *buf, int len) {
+    unsigned int hash = 5381;
+
+    while (len--)
+        hash = ((hash << 5) + hash) + (*buf++); /* hash * 33 + c */
+    return hash;
+}
+
+/* ----------------------------- API implementation ------------------------- */
+
+/* Reset an hashtable already initialized with ht_init().
+ * NOTE: This function should only called by ht_destroy(). */
+static void _dictReset(dict *ht) {
+    ht->table = NULL;
+    ht->size = 0;
+    ht->sizemask = 0;
+    ht->used = 0;
+}
+
+/* Create a new hash table */
+static dict *dictCreate(dictType *type, void *privDataPtr) {
+    dict *ht = hi_malloc(sizeof(*ht));
+    if (ht == NULL)
+        return NULL;
+
+    _dictInit(ht,type,privDataPtr);
+    return ht;
+}
+
+/* Initialize the hash table */
+static int _dictInit(dict *ht, dictType *type, void *privDataPtr) {
+    _dictReset(ht);
+    ht->type = type;
+    ht->privdata = privDataPtr;
+    return DICT_OK;
+}
+
+/* Expand or create the hashtable */
+static int dictExpand(dict *ht, unsigned long size) {
+    dict n; /* the new hashtable */
+    unsigned long realsize = _dictNextPower(size), i;
+
+    /* the size is invalid if it is smaller than the number of
+     * elements already inside the hashtable */
+    if (ht->used > size)
+        return DICT_ERR;
+
+    _dictInit(&n, ht->type, ht->privdata);
+    n.size = realsize;
+    n.sizemask = realsize-1;
+    n.table = hi_calloc(realsize,sizeof(dictEntry*));
+    if (n.table == NULL)
+        return DICT_ERR;
+
+    /* Copy all the elements from the old to the new table:
+     * note that if the old hash table is empty ht->size is zero,
+     * so dictExpand just creates an hash table. */
+    n.used = ht->used;
+    for (i = 0; i < ht->size && ht->used > 0; i++) {
+        dictEntry *he, *nextHe;
+
+        if (ht->table[i] == NULL) continue;
+
+        /* For each hash entry on this slot... */
+        he = ht->table[i];
+        while(he) {
+            unsigned int h;
+
+            nextHe = he->next;
+            /* Get the new element index */
+            h = dictHashKey(ht, he->key) & n.sizemask;
+            he->next = n.table[h];
+            n.table[h] = he;
+            ht->used--;
+            /* Pass to the next element */
+            he = nextHe;
+        }
+    }
+    assert(ht->used == 0);
+    hi_free(ht->table);
+
+    /* Remap the new hashtable in the old */
+    *ht = n;
+    return DICT_OK;
+}
+
+/* Add an element to the target hash table */
+static int dictAdd(dict *ht, void *key, void *val) {
+    int index;
+    dictEntry *entry;
+
+    /* Get the index of the new element, or -1 if
+     * the element already exists. */
+    if ((index = _dictKeyIndex(ht, key)) == -1)
+        return DICT_ERR;
+
+    /* Allocates the memory and stores key */
+    entry = hi_malloc(sizeof(*entry));
+    if (entry == NULL)
+        return DICT_ERR;
+
+    entry->next = ht->table[index];
+    ht->table[index] = entry;
+
+    /* Set the hash entry fields. */
+    dictSetHashKey(ht, entry, key);
+    dictSetHashVal(ht, entry, val);
+    ht->used++;
+    return DICT_OK;
+}
+
+/* Add an element, discarding the old if the key already exists.
+ * Return 1 if the key was added from scratch, 0 if there was already an
+ * element with such key and dictReplace() just performed a value update
+ * operation. */
+static int dictReplace(dict *ht, void *key, void *val) {
+    dictEntry *entry, auxentry;
+
+    /* Try to add the element. If the key
+     * does not exists dictAdd will succeed. */
+    if (dictAdd(ht, key, val) == DICT_OK)
+        return 1;
+    /* It already exists, get the entry */
+    entry = dictFind(ht, key);
+    if (entry == NULL)
+        return 0;
+
+    /* Free the old value and set the new one */
+    /* Set the new value and free the old one. Note that it is important
+     * to do that in this order, as the value may just be exactly the same
+     * as the previous one. In this context, think to reference counting,
+     * you want to increment (set), and then decrement (free), and not the
+     * reverse. */
+    auxentry = *entry;
+    dictSetHashVal(ht, entry, val);
+    dictFreeEntryVal(ht, &auxentry);
+    return 0;
+}
+
+/* Search and remove an element */
+static int dictDelete(dict *ht, const void *key) {
+    unsigned int h;
+    dictEntry *de, *prevde;
+
+    if (ht->size == 0)
+        return DICT_ERR;
+    h = dictHashKey(ht, key) & ht->sizemask;
+    de = ht->table[h];
+
+    prevde = NULL;
+    while(de) {
+        if (dictCompareHashKeys(ht,key,de->key)) {
+            /* Unlink the element from the list */
+            if (prevde)
+                prevde->next = de->next;
+            else
+                ht->table[h] = de->next;
+
+            dictFreeEntryKey(ht,de);
+            dictFreeEntryVal(ht,de);
+            hi_free(de);
+            ht->used--;
+            return DICT_OK;
+        }
+        prevde = de;
+        de = de->next;
+    }
+    return DICT_ERR; /* not found */
+}
+
+/* Destroy an entire hash table */
+static int _dictClear(dict *ht) {
+    unsigned long i;
+
+    /* Free all the elements */
+    for (i = 0; i < ht->size && ht->used > 0; i++) {
+        dictEntry *he, *nextHe;
+
+        if ((he = ht->table[i]) == NULL) continue;
+        while(he) {
+            nextHe = he->next;
+            dictFreeEntryKey(ht, he);
+            dictFreeEntryVal(ht, he);
+            hi_free(he);
+            ht->used--;
+            he = nextHe;
+        }
+    }
+    /* Free the table and the allocated cache structure */
+    hi_free(ht->table);
+    /* Re-initialize the table */
+    _dictReset(ht);
+    return DICT_OK; /* never fails */
+}
+
+/* Clear & Release the hash table */
+static void dictRelease(dict *ht) {
+    _dictClear(ht);
+    hi_free(ht);
+}
+
+static dictEntry *dictFind(dict *ht, const void *key) {
+    dictEntry *he;
+    unsigned int h;
+
+    if (ht->size == 0) return NULL;
+    h = dictHashKey(ht, key) & ht->sizemask;
+    he = ht->table[h];
+    while(he) {
+        if (dictCompareHashKeys(ht, key, he->key))
+            return he;
+        he = he->next;
+    }
+    return NULL;
+}
+
+static dictIterator *dictGetIterator(dict *ht) {
+    dictIterator *iter = hi_malloc(sizeof(*iter));
+    if (iter == NULL)
+        return NULL;
+
+    iter->ht = ht;
+    iter->index = -1;
+    iter->entry = NULL;
+    iter->nextEntry = NULL;
+    return iter;
+}
+
+static dictEntry *dictNext(dictIterator *iter) {
+    while (1) {
+        if (iter->entry == NULL) {
+            iter->index++;
+            if (iter->index >=
+                    (signed)iter->ht->size) break;
+            iter->entry = iter->ht->table[iter->index];
+        } else {
+            iter->entry = iter->nextEntry;
+        }
+        if (iter->entry) {
+            /* We need to save the 'next' here, the iterator user
+             * may delete the entry we are returning. */
+            iter->nextEntry = iter->entry->next;
+            return iter->entry;
+        }
+    }
+    return NULL;
+}
+
+static void dictReleaseIterator(dictIterator *iter) {
+    hi_free(iter);
+}
+
+/* ------------------------- private functions ------------------------------ */
+
+/* Expand the hash table if needed */
+static int _dictExpandIfNeeded(dict *ht) {
+    /* If the hash table is empty expand it to the initial size,
+     * if the table is "full" double its size. */
+    if (ht->size == 0)
+        return dictExpand(ht, DICT_HT_INITIAL_SIZE);
+    if (ht->used == ht->size)
+        return dictExpand(ht, ht->size*2);
+    return DICT_OK;
+}
+
+/* Our hash table capability is a power of two */
+static unsigned long _dictNextPower(unsigned long size) {
+    unsigned long i = DICT_HT_INITIAL_SIZE;
+
+    if (size >= LONG_MAX) return LONG_MAX;
+    while(1) {
+        if (i >= size)
+            return i;
+        i *= 2;
+    }
+}
+
+/* Returns the index of a free slot that can be populated with
+ * an hash entry for the given 'key'.
+ * If the key already exists, -1 is returned. */
+static int _dictKeyIndex(dict *ht, const void *key) {
+    unsigned int h;
+    dictEntry *he;
+
+    /* Expand the hashtable if needed */
+    if (_dictExpandIfNeeded(ht) == DICT_ERR)
+        return -1;
+    /* Compute the key hash value */
+    h = dictHashKey(ht, key) & ht->sizemask;
+    /* Search if this slot does not already contain the given key */
+    he = ht->table[h];
+    while(he) {
+        if (dictCompareHashKeys(ht, key, he->key))
+            return -1;
+        he = he->next;
+    }
+    return h;
+}
+
diff --git a/dict.h b/dict.h
new file mode 100644
index 000000000..95fcd280e
--- /dev/null
+++ b/dict.h
@@ -0,0 +1,126 @@
+/* Hash table implementation.
+ *
+ * This file implements in memory hash tables with insert/del/replace/find/
+ * get-random-element operations. Hash tables will auto resize if needed
+ * tables of power of two in size are used, collisions are handled by
+ * chaining. See the source code for more information... :)
+ *
+ * Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __DICT_H
+#define __DICT_H
+
+#define DICT_OK 0
+#define DICT_ERR 1
+
+/* Unused arguments generate annoying warnings... */
+#define DICT_NOTUSED(V) ((void) V)
+
+typedef struct dictEntry {
+    void *key;
+    void *val;
+    struct dictEntry *next;
+} dictEntry;
+
+typedef struct dictType {
+    unsigned int (*hashFunction)(const void *key);
+    void *(*keyDup)(void *privdata, const void *key);
+    void *(*valDup)(void *privdata, const void *obj);
+    int (*keyCompare)(void *privdata, const void *key1, const void *key2);
+    void (*keyDestructor)(void *privdata, void *key);
+    void (*valDestructor)(void *privdata, void *obj);
+} dictType;
+
+typedef struct dict {
+    dictEntry **table;
+    dictType *type;
+    unsigned long size;
+    unsigned long sizemask;
+    unsigned long used;
+    void *privdata;
+} dict;
+
+typedef struct dictIterator {
+    dict *ht;
+    int index;
+    dictEntry *entry, *nextEntry;
+} dictIterator;
+
+/* This is the initial size of every hash table */
+#define DICT_HT_INITIAL_SIZE     4
+
+/* ------------------------------- Macros ------------------------------------*/
+#define dictFreeEntryVal(ht, entry) \
+    if ((ht)->type->valDestructor) \
+        (ht)->type->valDestructor((ht)->privdata, (entry)->val)
+
+#define dictSetHashVal(ht, entry, _val_) do { \
+    if ((ht)->type->valDup) \
+        entry->val = (ht)->type->valDup((ht)->privdata, _val_); \
+    else \
+        entry->val = (_val_); \
+} while(0)
+
+#define dictFreeEntryKey(ht, entry) \
+    if ((ht)->type->keyDestructor) \
+        (ht)->type->keyDestructor((ht)->privdata, (entry)->key)
+
+#define dictSetHashKey(ht, entry, _key_) do { \
+    if ((ht)->type->keyDup) \
+        entry->key = (ht)->type->keyDup((ht)->privdata, _key_); \
+    else \
+        entry->key = (_key_); \
+} while(0)
+
+#define dictCompareHashKeys(ht, key1, key2) \
+    (((ht)->type->keyCompare) ? \
+        (ht)->type->keyCompare((ht)->privdata, key1, key2) : \
+        (key1) == (key2))
+
+#define dictHashKey(ht, key) (ht)->type->hashFunction(key)
+
+#define dictGetEntryKey(he) ((he)->key)
+#define dictGetEntryVal(he) ((he)->val)
+#define dictSlots(ht) ((ht)->size)
+#define dictSize(ht) ((ht)->used)
+
+/* API */
+static unsigned int dictGenHashFunction(const unsigned char *buf, int len);
+static dict *dictCreate(dictType *type, void *privDataPtr);
+static int dictExpand(dict *ht, unsigned long size);
+static int dictAdd(dict *ht, void *key, void *val);
+static int dictReplace(dict *ht, void *key, void *val);
+static int dictDelete(dict *ht, const void *key);
+static void dictRelease(dict *ht);
+static dictEntry * dictFind(dict *ht, const void *key);
+static dictIterator *dictGetIterator(dict *ht);
+static dictEntry *dictNext(dictIterator *iter);
+static void dictReleaseIterator(dictIterator *iter);
+
+#endif /* __DICT_H */
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
new file mode 100644
index 000000000..1d5bc56e0
--- /dev/null
+++ b/examples/CMakeLists.txt
@@ -0,0 +1,49 @@
+INCLUDE(FindPkgConfig)
+# Check for GLib
+
+PKG_CHECK_MODULES(GLIB2 glib-2.0)
+if (GLIB2_FOUND)
+    INCLUDE_DIRECTORIES(${GLIB2_INCLUDE_DIRS})
+    LINK_DIRECTORIES(${GLIB2_LIBRARY_DIRS})
+    ADD_EXECUTABLE(example-glib example-glib.c)
+    TARGET_LINK_LIBRARIES(example-glib hiredis ${GLIB2_LIBRARIES})
+ENDIF(GLIB2_FOUND)
+
+FIND_PATH(LIBEV ev.h
+    HINTS /usr/local /usr/opt/local
+    ENV LIBEV_INCLUDE_DIR)
+
+if (LIBEV)
+    # Just compile and link with libev
+    ADD_EXECUTABLE(example-libev example-libev.c)
+    TARGET_LINK_LIBRARIES(example-libev hiredis ev)
+ENDIF()
+
+FIND_PATH(LIBEVENT event.h)
+if (LIBEVENT)
+    ADD_EXECUTABLE(example-libevent example-libevent)
+    TARGET_LINK_LIBRARIES(example-libevent hiredis event)
+ENDIF()
+
+FIND_PATH(LIBUV uv.h)
+IF (LIBUV)
+    ADD_EXECUTABLE(example-libuv example-libuv.c)
+    TARGET_LINK_LIBRARIES(example-libuv hiredis uv)
+ENDIF()
+
+IF (APPLE)
+    FIND_LIBRARY(CF CoreFoundation)
+    ADD_EXECUTABLE(example-macosx example-macosx.c)
+    TARGET_LINK_LIBRARIES(example-macosx hiredis ${CF})
+ENDIF()
+
+IF (ENABLE_SSL)
+    ADD_EXECUTABLE(example-ssl example-ssl.c)
+    TARGET_LINK_LIBRARIES(example-ssl hiredis hiredis_ssl)
+ENDIF()
+
+ADD_EXECUTABLE(example example.c)
+TARGET_LINK_LIBRARIES(example hiredis)
+
+ADD_EXECUTABLE(example-push example-push.c)
+TARGET_LINK_LIBRARIES(example-push hiredis)
diff --git a/examples/example-ae.c b/examples/example-ae.c
new file mode 100644
index 000000000..8efa7306a
--- /dev/null
+++ b/examples/example-ae.c
@@ -0,0 +1,62 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+
+#include <hiredis.h>
+#include <async.h>
+#include <adapters/ae.h>
+
+/* Put event loop in the global scope, so it can be explicitly stopped */
+static aeEventLoop *loop;
+
+void getCallback(redisAsyncContext *c, void *r, void *privdata) {
+    redisReply *reply = r;
+    if (reply == NULL) return;
+    printf("argv[%s]: %s\n", (char*)privdata, reply->str);
+
+    /* Disconnect after receiving the reply to GET */
+    redisAsyncDisconnect(c);
+}
+
+void connectCallback(const redisAsyncContext *c, int status) {
+    if (status != REDIS_OK) {
+        printf("Error: %s\n", c->errstr);
+        aeStop(loop);
+        return;
+    }
+
+    printf("Connected...\n");
+}
+
+void disconnectCallback(const redisAsyncContext *c, int status) {
+    if (status != REDIS_OK) {
+        printf("Error: %s\n", c->errstr);
+        aeStop(loop);
+        return;
+    }
+
+    printf("Disconnected...\n");
+    aeStop(loop);
+}
+
+int main (int argc, char **argv) {
+    signal(SIGPIPE, SIG_IGN);
+
+    redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
+    if (c->err) {
+        /* Let *c leak for now... */
+        printf("Error: %s\n", c->errstr);
+        return 1;
+    }
+
+    loop = aeCreateEventLoop(64);
+    redisAeAttach(loop, c);
+    redisAsyncSetConnectCallback(c,connectCallback);
+    redisAsyncSetDisconnectCallback(c,disconnectCallback);
+    redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
+    redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
+    aeMain(loop);
+    return 0;
+}
+
diff --git a/examples/example-glib.c b/examples/example-glib.c
new file mode 100644
index 000000000..d6e10f8e8
--- /dev/null
+++ b/examples/example-glib.c
@@ -0,0 +1,73 @@
+#include <stdlib.h>
+
+#include <hiredis.h>
+#include <async.h>
+#include <adapters/glib.h>
+
+static GMainLoop *mainloop;
+
+static void
+connect_cb (const redisAsyncContext *ac G_GNUC_UNUSED,
+            int status)
+{
+    if (status != REDIS_OK) {
+        g_printerr("Failed to connect: %s\n", ac->errstr);
+        g_main_loop_quit(mainloop);
+    } else {
+        g_printerr("Connected...\n");
+    }
+}
+
+static void
+disconnect_cb (const redisAsyncContext *ac G_GNUC_UNUSED,
+               int status)
+{
+    if (status != REDIS_OK) {
+        g_error("Failed to disconnect: %s", ac->errstr);
+    } else {
+        g_printerr("Disconnected...\n");
+        g_main_loop_quit(mainloop);
+    }
+}
+
+static void
+command_cb(redisAsyncContext *ac,
+           gpointer r,
+           gpointer user_data G_GNUC_UNUSED)
+{
+    redisReply *reply = r;
+
+    if (reply) {
+        g_print("REPLY: %s\n", reply->str);
+    }
+
+    redisAsyncDisconnect(ac);
+}
+
+gint
+main (gint argc     G_GNUC_UNUSED,
+      gchar *argv[] G_GNUC_UNUSED)
+{
+    redisAsyncContext *ac;
+    GMainContext *context = NULL;
+    GSource *source;
+
+    ac = redisAsyncConnect("127.0.0.1", 6379);
+    if (ac->err) {
+        g_printerr("%s\n", ac->errstr);
+        exit(EXIT_FAILURE);
+    }
+
+    source = redis_source_new(ac);
+    mainloop = g_main_loop_new(context, FALSE);
+    g_source_attach(source, context);
+
+    redisAsyncSetConnectCallback(ac, connect_cb);
+    redisAsyncSetDisconnectCallback(ac, disconnect_cb);
+    redisAsyncCommand(ac, command_cb, NULL, "SET key 1234");
+    redisAsyncCommand(ac, command_cb, NULL, "GET key");
+
+    g_main_loop_run(mainloop);
+
+    return EXIT_SUCCESS;
+}
diff --git a/examples/example-ivykis.c b/examples/example-ivykis.c
new file mode 100644
index 000000000..f57dc3887
--- /dev/null
+++ b/examples/example-ivykis.c
@@ -0,0 +1,60 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+
+#include <hiredis.h>
+#include <async.h>
+#include <adapters/ivykis.h>
+
+void getCallback(redisAsyncContext *c, void *r, void *privdata) {
+    redisReply *reply = r;
+    if (reply == NULL) return;
+    printf("argv[%s]: %s\n", (char*)privdata, reply->str);
+
+    /* Disconnect after receiving the reply to GET */
+    redisAsyncDisconnect(c);
+}
+
+void connectCallback(const redisAsyncContext *c, int status) {
+    if (status != REDIS_OK) {
+        printf("Error: %s\n", c->errstr);
+        return;
+    }
+    printf("Connected...\n");
+}
+
+void disconnectCallback(const redisAsyncContext *c, int status) {
+    if (status != REDIS_OK) {
+        printf("Error: %s\n", c->errstr);
+        return;
+    }
+    printf("Disconnected...\n");
+}
+
+int main (int argc, char **argv) {
+#ifndef _WIN32
+    signal(SIGPIPE, SIG_IGN);
+#endif
+
+    iv_init();
+
+    redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
+    if (c->err) {
+        /* Let *c leak for now... */
+        printf("Error: %s\n", c->errstr);
+        return 1;
+    }
+
+    redisIvykisAttach(c);
+    redisAsyncSetConnectCallback(c,connectCallback);
+    redisAsyncSetDisconnectCallback(c,disconnectCallback);
+    redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
+    redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
+
+    iv_main();
+
+    iv_deinit();
+
+    return 0;
+}
diff --git a/examples/example-libev.c b/examples/example-libev.c
new file mode 100644
index 000000000..ec474306b
--- /dev/null
+++ b/examples/example-libev.c
@@ -0,0 +1,54 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+
+#include <hiredis.h>
+#include <async.h>
+#include <adapters/libev.h>
+
+void getCallback(redisAsyncContext *c, void *r, void *privdata) {
+    redisReply *reply = r;
+    if (reply == NULL) return;
+    printf("argv[%s]: %s\n", (char*)privdata, reply->str);
+
+    /* Disconnect after receiving the reply to GET */
+    redisAsyncDisconnect(c);
+}
+
+void connectCallback(const redisAsyncContext *c, int status) {
+    if (status != REDIS_OK) {
+        printf("Error: %s\n", c->errstr);
+        return;
+    }
+    printf("Connected...\n");
+}
+
+void disconnectCallback(const redisAsyncContext *c, int status) {
+    if (status != REDIS_OK) {
+        printf("Error: %s\n", c->errstr);
+        return;
+    }
+    printf("Disconnected...\n");
+}
+
+int main (int argc, char **argv) {
+#ifndef _WIN32
+    signal(SIGPIPE, SIG_IGN);
+#endif
+
+    redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
+    if (c->err) {
+        /* Let *c leak for now... */
+        printf("Error: %s\n", c->errstr);
+        return 1;
+    }
+
+    redisLibevAttach(EV_DEFAULT_ c);
+    redisAsyncSetConnectCallback(c,connectCallback);
+    redisAsyncSetDisconnectCallback(c,disconnectCallback);
+    redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
+    redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
+    ev_loop(EV_DEFAULT_ 0);
+    return 0;
+}
diff --git a/examples/example-libevent-ssl.c b/examples/example-libevent-ssl.c
new file mode 100644
index 000000000..7d99af1ba
--- /dev/null
+++ b/examples/example-libevent-ssl.c
@@ -0,0 +1,90 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+
+#include <hiredis.h>
+#include <hiredis_ssl.h>
+#include <async.h>
+#include <adapters/libevent.h>
+
+void getCallback(redisAsyncContext *c, void *r, void *privdata) {
+    redisReply *reply = r;
+    if (reply == NULL) return;
+    printf("argv[%s]: %s\n", (char*)privdata, reply->str);
+
+    /* Disconnect after receiving the reply to GET */
+    redisAsyncDisconnect(c);
+}
+
+void connectCallback(const redisAsyncContext *c, int status) {
+    if (status != REDIS_OK) {
+        printf("Error: %s\n", c->errstr);
+        return;
+    }
+    printf("Connected...\n");
+}
+
+void disconnectCallback(const redisAsyncContext *c, int status) {
+    if (status != REDIS_OK) {
+        printf("Error: %s\n", c->errstr);
+        return;
+    }
+    printf("Disconnected...\n");
+}
+
+int main (int argc, char **argv) {
+#ifndef _WIN32
+    signal(SIGPIPE, SIG_IGN);
+#endif
+
+    struct event_base *base = event_base_new();
+    if (argc < 5) {
+        fprintf(stderr,
+                "Usage: %s <key> <host> <port> <cert> <certKey> [ca]\n", argv[0]);
+        exit(1);
+    }
+
+    const char *value = argv[1];
+    size_t nvalue = strlen(value);
+
+    const char *hostname = argv[2];
+    int port = atoi(argv[3]);
+
+    const char *cert = argv[4];
+    const char *certKey = argv[5];
+    const char *caCert = argc > 5 ? argv[6] : NULL;
+
+    redisSSLContext *ssl;
+    redisSSLContextError ssl_error;
+
+    redisInitOpenSSL();
+
+    ssl = redisCreateSSLContext(caCert, NULL,
+            cert, certKey, NULL, &ssl_error);
+    if (!ssl) {
+        printf("Error: %s\n", redisSSLContextGetError(ssl_error));
+        return 1;
+    }
+
+    redisAsyncContext *c = redisAsyncConnect(hostname, port);
+    if (c->err) {
+        /* Let *c leak for now... */
+        printf("Error: %s\n", c->errstr);
+        return 1;
+    }
+    if (redisInitiateSSLWithContext(&c->c, ssl) != REDIS_OK) {
+        printf("SSL Error!\n");
+        exit(1);
+    }
+
+    redisLibeventAttach(c,base);
+    redisAsyncSetConnectCallback(c,connectCallback);
+    redisAsyncSetDisconnectCallback(c,disconnectCallback);
+    redisAsyncCommand(c, NULL, NULL, "SET key %b", value, nvalue);
+    redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
+    event_base_dispatch(base);
+
+    redisFreeSSLContext(ssl);
+    return 0;
+}
diff --git a/examples/example-libevent.c b/examples/example-libevent.c
new file mode 100644
index 000000000..49bddd0c2
--- /dev/null
+++ b/examples/example-libevent.c
@@ -0,0 +1,67 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+
+#include <hiredis.h>
+#include <async.h>
+#include <adapters/libevent.h>
+
+void getCallback(redisAsyncContext *c, void *r, void *privdata) {
+    redisReply *reply = r;
+    if (reply == NULL) {
+        if (c->errstr) {
+            printf("errstr: %s\n", c->errstr);
+        }
+        return;
+    }
+    printf("argv[%s]: %s\n", (char*)privdata, reply->str);
+
+    /* Disconnect after receiving the reply to GET */
+    redisAsyncDisconnect(c);
+}
+
+void connectCallback(const redisAsyncContext *c, int status) {
+    if (status != REDIS_OK) {
+        printf("Error: %s\n", c->errstr);
+        return;
+    }
+    printf("Connected...\n");
+}
+
+void disconnectCallback(const redisAsyncContext *c, int status) {
+    if (status != REDIS_OK) {
+        printf("Error: %s\n", c->errstr);
+        return;
+    }
+    printf("Disconnected...\n");
+}
+
+int main (int argc, char **argv) {
+#ifndef _WIN32
+    signal(SIGPIPE, SIG_IGN);
+#endif
+
+    struct event_base *base = event_base_new();
+    redisOptions options = {0};
+    REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379);
+    struct timeval tv = {0};
+    tv.tv_sec = 1;
+    options.connect_timeout = &tv;
+
+
+    redisAsyncContext *c = redisAsyncConnectWithOptions(&options);
+    if (c->err) {
+        /* Let *c leak for now... */
+        printf("Error: %s\n", c->errstr);
+        return 1;
+    }
+
+    redisLibeventAttach(c,base);
+    redisAsyncSetConnectCallback(c,connectCallback);
+    redisAsyncSetDisconnectCallback(c,disconnectCallback);
+    redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
+    redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
+    event_base_dispatch(base);
+    return 0;
+}
diff --git a/examples/example-libuv.c b/examples/example-libuv.c
new file mode 100644
index 000000000..cbde452b9
--- /dev/null
+++ b/examples/example-libuv.c
@@ -0,0 +1,56 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+
+#include <hiredis.h>
+#include <async.h>
+#include <adapters/libuv.h>
+
+void getCallback(redisAsyncContext *c, void *r, void *privdata) {
+    redisReply *reply = r;
+    if (reply == NULL) return;
+    printf("argv[%s]: %s\n", (char*)privdata, reply->str);
+
+    /* Disconnect after receiving the reply to GET */
+    redisAsyncDisconnect(c);
+}
+
+void connectCallback(const redisAsyncContext *c, int status) {
+    if (status != REDIS_OK) {
+        printf("Error: %s\n", c->errstr);
+        return;
+    }
+    printf("Connected...\n");
+}
+
+void disconnectCallback(const redisAsyncContext *c, int status) {
+    if (status != REDIS_OK) {
+        printf("Error: %s\n", c->errstr);
+        return;
+    }
+    printf("Disconnected...\n");
+}
+
+int main (int argc, char **argv) {
+#ifndef _WIN32
+    signal(SIGPIPE, SIG_IGN);
+#endif
+
+    uv_loop_t* loop = uv_default_loop();
+
+    redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
+    if (c->err) {
+        /* Let *c leak for now... */
+        printf("Error: %s\n", c->errstr);
+        return 1;
+    }
+
+    redisLibuvAttach(c,loop);
+    redisAsyncSetConnectCallback(c,connectCallback);
+    redisAsyncSetDisconnectCallback(c,disconnectCallback);
+    redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
+    redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
+    uv_run(loop, UV_RUN_DEFAULT);
+    return 0;
+}
diff --git a/examples/example-macosx.c b/examples/example-macosx.c
new file mode 100644
index 000000000..bc84ed5ba
--- /dev/null
+++ b/examples/example-macosx.c
@@ -0,0 +1,66 @@
+//
+//  Created by Дмитрий Бахвалов on 13.07.15.
+//  Copyright (c) 2015 Dmitry Bakhvalov. All rights reserved.
+//
+
+#include <stdio.h>
+
+#include <hiredis.h>
+#include <async.h>
+#include <adapters/macosx.h>
+
+void getCallback(redisAsyncContext *c, void *r, void *privdata) {
+    redisReply *reply = r;
+    if (reply == NULL) return;
+    printf("argv[%s]: %s\n", (char*)privdata, reply->str);
+
+    /* Disconnect after receiving the reply to GET */
+    redisAsyncDisconnect(c);
+}
+
+void connectCallback(const redisAsyncContext *c, int status) {
+    if (status != REDIS_OK) {
+        printf("Error: %s\n", c->errstr);
+        return;
+    }
+    printf("Connected...\n");
+}
+
+void disconnectCallback(const redisAsyncContext *c, int status) {
+    if (status != REDIS_OK) {
+        printf("Error: %s\n", c->errstr);
+        return;
+    }
+    CFRunLoopStop(CFRunLoopGetCurrent());
+    printf("Disconnected...\n");
+}
+
+int main (int argc, char **argv) {
+    signal(SIGPIPE, SIG_IGN);
+
+    CFRunLoopRef loop = CFRunLoopGetCurrent();
+    if( !loop ) {
+        printf("Error: Cannot get current run loop\n");
+        return 1;
+    }
+
+    redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
+    if (c->err) {
+        /* Let *c leak for now... */
+        printf("Error: %s\n", c->errstr);
+        return 1;
+    }
+
+    redisMacOSAttach(c, loop);
+
+    redisAsyncSetConnectCallback(c,connectCallback);
+    redisAsyncSetDisconnectCallback(c,disconnectCallback);
+
+    redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
+    redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
+
+    CFRunLoopRun();
+
+    return 0;
+}
+
diff --git a/examples/example-push.c b/examples/example-push.c
new file mode 100644
index 000000000..2d4ab4dc0
--- /dev/null
+++ b/examples/example-push.c
@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2020, Michael Grunder <michael dot grunder at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <hiredis.h>
+#include <win32.h>
+
+#define KEY_COUNT 5
+
+#define panicAbort(fmt, ...) \
+    do { \
+        fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, __LINE__, __func__, __VA_ARGS__); \
+        exit(-1); \
+    } while (0)
+
+static void assertReplyAndFree(redisContext *context, redisReply *reply, int type) {
+    if (reply == NULL)
+        panicAbort("NULL reply from server (error: %s)", context->errstr);
+
+    if (reply->type != type) {
+        if (reply->type == REDIS_REPLY_ERROR)
+            fprintf(stderr, "Redis Error: %s\n", reply->str);
+
+        panicAbort("Expected reply type %d but got type %d", type, reply->type);
+    }
+
+    freeReplyObject(reply);
+}
+
+/* Switch to the RESP3 protocol and enable client tracking */
+static void enableClientTracking(redisContext *c) {
+    redisReply *reply = redisCommand(c, "HELLO 3");
+    if (reply == NULL || c->err) {
+        panicAbort("NULL reply or server error (error: %s)", c->errstr);
+    }
+
+    if (reply->type != REDIS_REPLY_MAP) {
+        fprintf(stderr, "Error: Can't send HELLO 3 command.  Are you sure you're ");
+        fprintf(stderr, "connected to redis-server >= 6.0.0?\nRedis error: %s\n",
+                        reply->type == REDIS_REPLY_ERROR ? reply->str : "(unknown)");
+        exit(-1);
+    }
+
+    freeReplyObject(reply);
+
+    /* Enable client tracking */
+    reply = redisCommand(c, "CLIENT TRACKING ON");
+    assertReplyAndFree(c, reply, REDIS_REPLY_STATUS);
+}
+
+void pushReplyHandler(void *privdata, void *r) {
+    redisReply *reply = r;
+    int *invalidations = privdata;
+
+    /* Sanity check on the invalidation reply */
+    if (reply->type != REDIS_REPLY_PUSH || reply->elements != 2 ||
+        reply->element[1]->type != REDIS_REPLY_ARRAY ||
+        reply->element[1]->element[0]->type != REDIS_REPLY_STRING)
+    {
+        panicAbort("%s", "Can't parse PUSH message!");
+    }
+
+    /* Increment our invalidation count */
+    *invalidations += 1;
+
+    printf("pushReplyHandler(): INVALIDATE '%s' (invalidation count: %d)\n",
+           reply->element[1]->element[0]->str, *invalidations);
+
+    freeReplyObject(reply);
+}
+
+/* We aren't actually freeing anything here, but it is included to show that we can
+ * have hiredis call our data destructor when freeing the context */
+void privdata_dtor(void *privdata) {
+    unsigned int *icount = privdata;
+    printf("privdata_dtor():  In context privdata dtor (invalidations: %u)\n", *icount);
+}
+
+int main(int argc, char **argv) {
+    unsigned int j, invalidations = 0;
+    redisContext *c;
+    redisReply *reply;
+
+    const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1";
+    int port = (argc > 2) ? atoi(argv[2]) : 6379;
+
+    redisOptions o = {0};
+    REDIS_OPTIONS_SET_TCP(&o, hostname, port);
+
+    /* Set our context privdata to the address of our invalidation counter.  Each
+     * time our PUSH handler is called, hiredis will pass the privdata for context.
+     *
+     * This could also be done after we create the context like so:
+     *
+     *    c->privdata = &invalidations;
+     *    c->free_privdata = privdata_dtor;
+     */
+    REDIS_OPTIONS_SET_PRIVDATA(&o, &invalidations, privdata_dtor);
+
+    /* Set our custom PUSH message handler */
+    o.push_cb = pushReplyHandler;
+
+    c = redisConnectWithOptions(&o);
+    if (c == NULL || c->err)
+        panicAbort("Connection error:  %s", c ? c->errstr : "OOM");
+
+    /* Enable RESP3 and turn on client tracking */
+    enableClientTracking(c);
+
+    /* Set some keys and then read them back.  Once we do that, Redis will deliver
+     * invalidation push messages whenever the key is modified */
+    for (j = 0; j < KEY_COUNT; j++) {
+        reply = redisCommand(c, "SET key:%d initial:%d", j, j);
+        assertReplyAndFree(c, reply, REDIS_REPLY_STATUS);
+
+        reply = redisCommand(c, "GET key:%d", j);
+        assertReplyAndFree(c, reply, REDIS_REPLY_STRING);
+    }
+
+    /* Trigger invalidation messages by updating keys we just read */
+    for (j = 0; j < KEY_COUNT; j++) {
+        printf("            main(): SET key:%d update:%d\n", j, j);
+        reply = redisCommand(c, "SET key:%d update:%d", j, j);
+        assertReplyAndFree(c, reply, REDIS_REPLY_STATUS);
+        printf("            main(): SET REPLY OK\n");
+    }
+
+    printf("\nTotal detected invalidations: %d, expected: %d\n", invalidations, KEY_COUNT);
+
+    /* PING server */
+    redisFree(c);
+}
diff --git a/examples/example-qt.cpp b/examples/example-qt.cpp
new file mode 100644
index 000000000..f524c3f3d
--- /dev/null
+++ b/examples/example-qt.cpp
@@ -0,0 +1,46 @@
+#include <iostream>
+using namespace std;
+
+#include <QCoreApplication>
+#include <QTimer>
+
+#include "example-qt.h"
+
+void getCallback(redisAsyncContext *, void * r, void * privdata) {
+
+    redisReply * reply = static_cast<redisReply *>(r);
+    ExampleQt * ex = static_cast<ExampleQt *>(privdata);
+    if (reply == nullptr || ex == nullptr) return;
+
+    cout << "key: " << reply->str << endl;
+
+    ex->finish();
+}
+
+void ExampleQt::run() {
+
+    m_ctx = redisAsyncConnect("localhost", 6379);
+
+    if (m_ctx->err) {
+        cerr << "Error: " << m_ctx->errstr << endl;
+        redisAsyncFree(m_ctx);
+        emit finished();
+    }
+
+    m_adapter.setContext(m_ctx);
+
+    redisAsyncCommand(m_ctx, NULL, NULL, "SET key %s", m_value);
+    redisAsyncCommand(m_ctx, getCallback, this, "GET key");
+}
+
+int main (int argc, char **argv) {
+
+    QCoreApplication app(argc, argv);
+
+    ExampleQt example(argv[argc-1]);
+
+    QObject::connect(&example, SIGNAL(finished()), &app, SLOT(quit()));
+    QTimer::singleShot(0, &example, SLOT(run()));
+
+    return app.exec();
+}
diff --git a/examples/example-qt.h b/examples/example-qt.h
new file mode 100644
index 000000000..374f47666
--- /dev/null
+++ b/examples/example-qt.h
@@ -0,0 +1,32 @@
+#ifndef __HIREDIS_EXAMPLE_QT_H
+#define __HIREDIS_EXAMPLE_QT_H
+
+#include <adapters/qt.h>
+
+class ExampleQt : public QObject {
+
+    Q_OBJECT
+
+    public:
+        ExampleQt(const char * value, QObject * parent = 0)
+            : QObject(parent), m_value(value) {}
+
+    signals:
+        void finished();
+
+    public slots:
+        void run();
+
+    private:
+        void finish() { emit finished(); }
+
+    private:
+        const char * m_value;
+        redisAsyncContext * m_ctx;
+        RedisQtAdapter m_adapter;
+
+    friend
+    void getCallback(redisAsyncContext *, void *, void *);
+};
+
+#endif /* !__HIREDIS_EXAMPLE_QT_H */
diff --git a/examples/example-ssl.c b/examples/example-ssl.c
new file mode 100644
index 000000000..c754177cf
--- /dev/null
+++ b/examples/example-ssl.c
@@ -0,0 +1,110 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <hiredis.h>
+#include <hiredis_ssl.h>
+#include <win32.h>
+
+int main(int argc, char **argv) {
+    unsigned int j;
+    redisSSLContext *ssl;
+    redisSSLContextError ssl_error;
+    redisContext *c;
+    redisReply *reply;
+    if (argc < 4) {
+        printf("Usage: %s <host> <port> <cert> <key> [ca]\n", argv[0]);
+        exit(1);
+    }
+    const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1";
+    int port = atoi(argv[2]);
+    const char *cert = argv[3];
+    const char *key = argv[4];
+    const char *ca = argc > 4 ? argv[5] : NULL;
+
+    redisInitOpenSSL();
+    ssl = redisCreateSSLContext(ca, NULL, cert, key, NULL, &ssl_error);
+    if (!ssl) {
+        printf("SSL Context error: %s\n",
+                redisSSLContextGetError(ssl_error));
+        exit(1);
+    }
+
+    struct timeval tv = { 1, 500000 }; // 1.5 seconds
+    redisOptions options = {0};
+    REDIS_OPTIONS_SET_TCP(&options, hostname, port);
+    options.connect_timeout = &tv;
+    c = redisConnectWithOptions(&options);
+
+    if (c == NULL || c->err) {
+        if (c) {
+            printf("Connection error: %s\n", c->errstr);
+            redisFree(c);
+        } else {
+            printf("Connection error: can't allocate redis context\n");
+        }
+        exit(1);
+    }
+
+    if (redisInitiateSSLWithContext(c, ssl) != REDIS_OK) {
+        printf("Couldn't initialize SSL!\n");
+        printf("Error: %s\n", c->errstr);
+        redisFree(c);
+        exit(1);
+    }
+
+    /* PING server */
+    reply = redisCommand(c,"PING");
+    printf("PING: %s\n", reply->str);
+    freeReplyObject(reply);
+
+    /* Set a key */
+    reply = redisCommand(c,"SET %s %s", "foo", "hello world");
+    printf("SET: %s\n", reply->str);
+    freeReplyObject(reply);
+
+    /* Set a key using binary safe API */
+    reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5);
+    printf("SET (binary API): %s\n", reply->str);
+    freeReplyObject(reply);
+
+    /* Try a GET and two INCR */
+    reply = redisCommand(c,"GET foo");
+    printf("GET foo: %s\n", reply->str);
+    freeReplyObject(reply);
+
+    reply = redisCommand(c,"INCR counter");
+    printf("INCR counter: %lld\n", reply->integer);
+    freeReplyObject(reply);
+    /* again ... */
+    reply = redisCommand(c,"INCR counter");
+    printf("INCR counter: %lld\n", reply->integer);
+    freeReplyObject(reply);
+
+    /* Create a list of numbers, from 0 to 9 */
+    reply = redisCommand(c,"DEL mylist");
+    freeReplyObject(reply);
+    for (j = 0; j < 10; j++) {
+        char buf[64];
+
+        snprintf(buf,64,"%u",j);
+        reply = redisCommand(c,"LPUSH mylist element-%s", buf);
+        freeReplyObject(reply);
+    }
+
+    /* Let's check what we have inside the list */
+    reply = redisCommand(c,"LRANGE mylist 0 -1");
+    if (reply->type == REDIS_REPLY_ARRAY) {
+        for (j = 0; j < reply->elements; j++) {
+            printf("%u) %s\n", j, reply->element[j]->str);
+        }
+    }
+    freeReplyObject(reply);
+
+    /* Disconnects and frees the context */
+    redisFree(c);
+
+    redisFreeSSLContext(ssl);
+
+    return 0;
+}
diff --git a/examples/example.c b/examples/example.c
new file mode 100644
index 000000000..15dacbd18
--- /dev/null
+++ b/examples/example.c
@@ -0,0 +1,91 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <hiredis.h>
+#include <win32.h>
+
+int main(int argc, char **argv) {
+    unsigned int j, isunix = 0;
+    redisContext *c;
+    redisReply *reply;
+    const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1";
+
+    if (argc > 2) {
+        if (*argv[2] == 'u' || *argv[2] == 'U') {
+            isunix = 1;
+            /* in this case, host is the path to the unix socket */
+            printf("Will connect to unix socket @%s\n", hostname);
+        }
+    }
+
+    int port = (argc > 2) ? atoi(argv[2]) : 6379;
+
+    struct timeval timeout = { 1, 500000 }; // 1.5 seconds
+    if (isunix) {
+        c = redisConnectUnixWithTimeout(hostname, timeout);
+    } else {
+        c = redisConnectWithTimeout(hostname, port, timeout);
+    }
+    if (c == NULL || c->err) {
+        if (c) {
+            printf("Connection error: %s\n", c->errstr);
+            redisFree(c);
+        } else {
+            printf("Connection error: can't allocate redis context\n");
+        }
+        exit(1);
+    }
+
+    /* PING server */
+    reply = redisCommand(c,"PING");
+    printf("PING: %s\n", reply->str);
+    freeReplyObject(reply);
+
+    /* Set a key */
+    reply = redisCommand(c,"SET %s %s", "foo", "hello world");
+    printf("SET: %s\n", reply->str);
+    freeReplyObject(reply);
+
+    /* Set a key using binary safe API */
+    reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5);
+    printf("SET (binary API): %s\n", reply->str);
+    freeReplyObject(reply);
+
+    /* Try a GET and two INCR */
+    reply = redisCommand(c,"GET foo");
+    printf("GET foo: %s\n", reply->str);
+    freeReplyObject(reply);
+
+    reply = redisCommand(c,"INCR counter");
+    printf("INCR counter: %lld\n", reply->integer);
+    freeReplyObject(reply);
+    /* again ... */
+    reply = redisCommand(c,"INCR counter");
+    printf("INCR counter: %lld\n", reply->integer);
+    freeReplyObject(reply);
+
+    /* Create a list of numbers, from 0 to 9 */
+    reply = redisCommand(c,"DEL mylist");
+    freeReplyObject(reply);
+    for (j = 0; j < 10; j++) {
+        char buf[64];
+
+        snprintf(buf,64,"%u",j);
+        reply = redisCommand(c,"LPUSH mylist element-%s", buf);
+        freeReplyObject(reply);
+    }
+
+    /* Let's check what we have inside the list */
+    reply = redisCommand(c,"LRANGE mylist 0 -1");
+    if (reply->type == REDIS_REPLY_ARRAY) {
+        for (j = 0; j < reply->elements; j++) {
+            printf("%u) %s\n", j, reply->element[j]->str);
+        }
+    }
+    freeReplyObject(reply);
+
+    /* Disconnects and frees the context */
+    redisFree(c);
+
+    return 0;
+}
diff --git a/fmacros.h b/fmacros.h
new file mode 100644
index 000000000..3227faafd
--- /dev/null
+++ b/fmacros.h
@@ -0,0 +1,12 @@
+#ifndef __HIREDIS_FMACRO_H
+#define __HIREDIS_FMACRO_H
+
+#define _XOPEN_SOURCE 600
+#define _POSIX_C_SOURCE 200112L
+
+#if defined(__APPLE__) && defined(__MACH__)
+/* Enable TCP_KEEPALIVE */
+#define _DARWIN_C_SOURCE
+#endif
+
+#endif
diff --git a/hiredis-config.cmake.in b/hiredis-config.cmake.in
new file mode 100644
index 000000000..98851dcee
--- /dev/null
+++ b/hiredis-config.cmake.in
@@ -0,0 +1,13 @@
+@PACKAGE_INIT@
+
+set_and_check(hiredis_INCLUDEDIR "@PACKAGE_INCLUDE_INSTALL_DIR@")
+
+IF (NOT TARGET hiredis::hiredis)
+	INCLUDE(${CMAKE_CURRENT_LIST_DIR}/hiredis-targets.cmake)
+ENDIF()
+
+SET(hiredis_LIBRARIES hiredis::hiredis)
+SET(hiredis_INCLUDE_DIRS ${hiredis_INCLUDEDIR})
+
+check_required_components(hiredis)
+
diff --git a/hiredis.c b/hiredis.c
new file mode 100644
index 000000000..a7fbf486f
--- /dev/null
+++ b/hiredis.c
@@ -0,0 +1,1173 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
+ *                     Jan-Erik Rediger <janerik at fnordig dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fmacros.h"
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <errno.h>
+#include <ctype.h>
+
+#include "hiredis.h"
+#include "net.h"
+#include "sds.h"
+#include "async.h"
+#include "win32.h"
+
+extern int redisContextUpdateConnectTimeout(redisContext *c, const struct timeval *timeout);
+extern int redisContextUpdateCommandTimeout(redisContext *c, const struct timeval *timeout);
+
+static redisContextFuncs redisContextDefaultFuncs = {
+    .free_privctx = NULL,
+    .async_read = redisAsyncRead,
+    .async_write = redisAsyncWrite,
+    .read = redisNetRead,
+    .write = redisNetWrite
+};
+
+static redisReply *createReplyObject(int type);
+static void *createStringObject(const redisReadTask *task, char *str, size_t len);
+static void *createArrayObject(const redisReadTask *task, size_t elements);
+static void *createIntegerObject(const redisReadTask *task, long long value);
+static void *createDoubleObject(const redisReadTask *task, double value, char *str, size_t len);
+static void *createNilObject(const redisReadTask *task);
+static void *createBoolObject(const redisReadTask *task, int bval);
+
+/* Default set of functions to build the reply. Keep in mind that such a
+ * function returning NULL is interpreted as OOM. */
+static redisReplyObjectFunctions defaultFunctions = {
+    createStringObject,
+    createArrayObject,
+    createIntegerObject,
+    createDoubleObject,
+    createNilObject,
+    createBoolObject,
+    freeReplyObject
+};
+
+/* Create a reply object */
+static redisReply *createReplyObject(int type) {
+    redisReply *r = hi_calloc(1,sizeof(*r));
+
+    if (r == NULL)
+        return NULL;
+
+    r->type = type;
+    return r;
+}
+
+/* Free a reply object */
+void freeReplyObject(void *reply) {
+    redisReply *r = reply;
+    size_t j;
+
+    if (r == NULL)
+        return;
+
+    switch(r->type) {
+    case REDIS_REPLY_INTEGER:
+        break; /* Nothing to free */
+    case REDIS_REPLY_ARRAY:
+    case REDIS_REPLY_MAP:
+    case REDIS_REPLY_SET:
+    case REDIS_REPLY_PUSH:
+        if (r->element != NULL) {
+            for (j = 0; j < r->elements; j++)
+                freeReplyObject(r->element[j]);
+            hi_free(r->element);
+        }
+        break;
+    case REDIS_REPLY_ERROR:
+    case REDIS_REPLY_STATUS:
+    case REDIS_REPLY_STRING:
+    case REDIS_REPLY_DOUBLE:
+    case REDIS_REPLY_VERB:
+        hi_free(r->str);
+        break;
+    }
+    hi_free(r);
+}
+
+static void *createStringObject(const redisReadTask *task, char *str, size_t len) {
+    redisReply *r, *parent;
+    char *buf;
+
+    r = createReplyObject(task->type);
+    if (r == NULL)
+        return NULL;
+
+    assert(task->type == REDIS_REPLY_ERROR  ||
+           task->type == REDIS_REPLY_STATUS ||
+           task->type == REDIS_REPLY_STRING ||
+           task->type == REDIS_REPLY_VERB);
+
+    /* Copy string value */
+    if (task->type == REDIS_REPLY_VERB) {
+        buf = hi_malloc(len-4+1); /* Skip 4 bytes of verbatim type header. */
+        if (buf == NULL) goto oom;
+
+        memcpy(r->vtype,str,3);
+        r->vtype[3] = '\0';
+        memcpy(buf,str+4,len-4);
+        buf[len-4] = '\0';
+        r->len = len - 4;
+    } else {
+        buf = hi_malloc(len+1);
+        if (buf == NULL) goto oom;
+
+        memcpy(buf,str,len);
+        buf[len] = '\0';
+        r->len = len;
+    }
+    r->str = buf;
+
+    if (task->parent) {
+        parent = task->parent->obj;
+        assert(parent->type == REDIS_REPLY_ARRAY ||
+               parent->type == REDIS_REPLY_MAP ||
+               parent->type == REDIS_REPLY_SET ||
+               parent->type == REDIS_REPLY_PUSH);
+        parent->element[task->idx] = r;
+    }
+    return r;
+
+oom:
+    freeReplyObject(r);
+    return NULL;
+}
+
+static void *createArrayObject(const redisReadTask *task, size_t elements) {
+    redisReply *r, *parent;
+
+    r = createReplyObject(task->type);
+    if (r == NULL)
+        return NULL;
+
+    if (elements > 0) {
+        r->element = hi_calloc(elements,sizeof(redisReply*));
+        if (r->element == NULL) {
+            freeReplyObject(r);
+            return NULL;
+        }
+    }
+
+    r->elements = elements;
+
+    if (task->parent) {
+        parent = task->parent->obj;
+        assert(parent->type == REDIS_REPLY_ARRAY ||
+               parent->type == REDIS_REPLY_MAP ||
+               parent->type == REDIS_REPLY_SET ||
+               parent->type == REDIS_REPLY_PUSH);
+        parent->element[task->idx] = r;
+    }
+    return r;
+}
+
+static void *createIntegerObject(const redisReadTask *task, long long value) {
+    redisReply *r, *parent;
+
+    r = createReplyObject(REDIS_REPLY_INTEGER);
+    if (r == NULL)
+        return NULL;
+
+    r->integer = value;
+
+    if (task->parent) {
+        parent = task->parent->obj;
+        assert(parent->type == REDIS_REPLY_ARRAY ||
+               parent->type == REDIS_REPLY_MAP ||
+               parent->type == REDIS_REPLY_SET ||
+               parent->type == REDIS_REPLY_PUSH);
+        parent->element[task->idx] = r;
+    }
+    return r;
+}
+
+static void *createDoubleObject(const redisReadTask *task, double value, char *str, size_t len) {
+    redisReply *r, *parent;
+
+    r = createReplyObject(REDIS_REPLY_DOUBLE);
+    if (r == NULL)
+        return NULL;
+
+    r->dval = value;
+    r->str = hi_malloc(len+1);
+    if (r->str == NULL) {
+        freeReplyObject(r);
+        return NULL;
+    }
+
+    /* The double reply also has the original protocol string representing a
+     * double as a null terminated string. This way the caller does not need
+     * to format back for string conversion, especially since Redis does efforts
+     * to make the string more human readable avoiding the calssical double
+     * decimal string conversion artifacts. */
+    memcpy(r->str, str, len);
+    r->str[len] = '\0';
+
+    if (task->parent) {
+        parent = task->parent->obj;
+        assert(parent->type == REDIS_REPLY_ARRAY ||
+               parent->type == REDIS_REPLY_MAP ||
+               parent->type == REDIS_REPLY_SET);
+        parent->element[task->idx] = r;
+    }
+    return r;
+}
+
+static void *createNilObject(const redisReadTask *task) {
+    redisReply *r, *parent;
+
+    r = createReplyObject(REDIS_REPLY_NIL);
+    if (r == NULL)
+        return NULL;
+
+    if (task->parent) {
+        parent = task->parent->obj;
+        assert(parent->type == REDIS_REPLY_ARRAY ||
+               parent->type == REDIS_REPLY_MAP ||
+               parent->type == REDIS_REPLY_SET);
+        parent->element[task->idx] = r;
+    }
+    return r;
+}
+
+static void *createBoolObject(const redisReadTask *task, int bval) {
+    redisReply *r, *parent;
+
+    r = createReplyObject(REDIS_REPLY_BOOL);
+    if (r == NULL)
+        return NULL;
+
+    r->integer = bval != 0;
+
+    if (task->parent) {
+        parent = task->parent->obj;
+        assert(parent->type == REDIS_REPLY_ARRAY ||
+               parent->type == REDIS_REPLY_MAP ||
+               parent->type == REDIS_REPLY_SET);
+        parent->element[task->idx] = r;
+    }
+    return r;
+}
+
+/* Return the number of digits of 'v' when converted to string in radix 10.
+ * Implementation borrowed from link in redis/src/util.c:string2ll(). */
+static uint32_t countDigits(uint64_t v) {
+  uint32_t result = 1;
+  for (;;) {
+    if (v < 10) return result;
+    if (v < 100) return result + 1;
+    if (v < 1000) return result + 2;
+    if (v < 10000) return result + 3;
+    v /= 10000U;
+    result += 4;
+  }
+}
+
+/* Helper that calculates the bulk length given a certain string length. */
+static size_t bulklen(size_t len) {
+    return 1+countDigits(len)+2+len+2;
+}
+
+int redisvFormatCommand(char **target, const char *format, va_list ap) {
+    const char *c = format;
+    char *cmd = NULL; /* final command */
+    int pos; /* position in final command */
+    sds curarg, newarg; /* current argument */
+    int touched = 0; /* was the current argument touched? */
+    char **curargv = NULL, **newargv = NULL;
+    int argc = 0;
+    int totlen = 0;
+    int error_type = 0; /* 0 = no error; -1 = memory error; -2 = format error */
+    int j;
+
+    /* Abort if there is not target to set */
+    if (target == NULL)
+        return -1;
+
+    /* Build the command string accordingly to protocol */
+    curarg = sdsempty();
+    if (curarg == NULL)
+        return -1;
+
+    while(*c != '\0') {
+        if (*c != '%' || c[1] == '\0') {
+            if (*c == ' ') {
+                if (touched) {
+                    newargv = hi_realloc(curargv,sizeof(char*)*(argc+1));
+                    if (newargv == NULL) goto memory_err;
+                    curargv = newargv;
+                    curargv[argc++] = curarg;
+                    totlen += bulklen(sdslen(curarg));
+
+                    /* curarg is put in argv so it can be overwritten. */
+                    curarg = sdsempty();
+                    if (curarg == NULL) goto memory_err;
+                    touched = 0;
+                }
+            } else {
+                newarg = sdscatlen(curarg,c,1);
+                if (newarg == NULL) goto memory_err;
+                curarg = newarg;
+                touched = 1;
+            }
+        } else {
+            char *arg;
+            size_t size;
+
+            /* Set newarg so it can be checked even if it is not touched. */
+            newarg = curarg;
+
+            switch(c[1]) {
+            case 's':
+                arg = va_arg(ap,char*);
+                size = strlen(arg);
+                if (size > 0)
+                    newarg = sdscatlen(curarg,arg,size);
+                break;
+            case 'b':
+                arg = va_arg(ap,char*);
+                size = va_arg(ap,size_t);
+                if (size > 0)
+                    newarg = sdscatlen(curarg,arg,size);
+                break;
+            case '%':
+                newarg = sdscat(curarg,"%");
+                break;
+            default:
+                /* Try to detect printf format */
+                {
+                    static const char intfmts[] = "diouxX";
+                    static const char flags[] = "#0-+ ";
+                    char _format[16];
+                    const char *_p = c+1;
+                    size_t _l = 0;
+                    va_list _cpy;
+
+                    /* Flags */
+                    while (*_p != '\0' && strchr(flags,*_p) != NULL) _p++;
+
+                    /* Field width */
+                    while (*_p != '\0' && isdigit(*_p)) _p++;
+
+                    /* Precision */
+                    if (*_p == '.') {
+                        _p++;
+                        while (*_p != '\0' && isdigit(*_p)) _p++;
+                    }
+
+                    /* Copy va_list before consuming with va_arg */
+                    va_copy(_cpy,ap);
+
+                    /* Integer conversion (without modifiers) */
+                    if (strchr(intfmts,*_p) != NULL) {
+                        va_arg(ap,int);
+                        goto fmt_valid;
+                    }
+
+                    /* Double conversion (without modifiers) */
+                    if (strchr("eEfFgGaA",*_p) != NULL) {
+                        va_arg(ap,double);
+                        goto fmt_valid;
+                    }
+
+                    /* Size: char */
+                    if (_p[0] == 'h' && _p[1] == 'h') {
+                        _p += 2;
+                        if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
+                            va_arg(ap,int); /* char gets promoted to int */
+                            goto fmt_valid;
+                        }
+                        goto fmt_invalid;
+                    }
+
+                    /* Size: short */
+                    if (_p[0] == 'h') {
+                        _p += 1;
+                        if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
+                            va_arg(ap,int); /* short gets promoted to int */
+                            goto fmt_valid;
+                        }
+                        goto fmt_invalid;
+                    }
+
+                    /* Size: long long */
+                    if (_p[0] == 'l' && _p[1] == 'l') {
+                        _p += 2;
+                        if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
+                            va_arg(ap,long long);
+                            goto fmt_valid;
+                        }
+                        goto fmt_invalid;
+                    }
+
+                    /* Size: long */
+                    if (_p[0] == 'l') {
+                        _p += 1;
+                        if (*_p != '\0' && strchr(intfmts,*_p) != NULL) {
+                            va_arg(ap,long);
+                            goto fmt_valid;
+                        }
+                        goto fmt_invalid;
+                    }
+
+                fmt_invalid:
+                    va_end(_cpy);
+                    goto format_err;
+
+                fmt_valid:
+                    _l = (_p+1)-c;
+                    if (_l < sizeof(_format)-2) {
+                        memcpy(_format,c,_l);
+                        _format[_l] = '\0';
+                        newarg = sdscatvprintf(curarg,_format,_cpy);
+
+                        /* Update current position (note: outer blocks
+                         * increment c twice so compensate here) */
+                        c = _p-1;
+                    }
+
+                    va_end(_cpy);
+                    break;
+                }
+            }
+
+            if (newarg == NULL) goto memory_err;
+            curarg = newarg;
+
+            touched = 1;
+            c++;
+        }
+        c++;
+    }
+
+    /* Add the last argument if needed */
+    if (touched) {
+        newargv = hi_realloc(curargv,sizeof(char*)*(argc+1));
+        if (newargv == NULL) goto memory_err;
+        curargv = newargv;
+        curargv[argc++] = curarg;
+        totlen += bulklen(sdslen(curarg));
+    } else {
+        sdsfree(curarg);
+    }
+
+    /* Clear curarg because it was put in curargv or was free'd. */
+    curarg = NULL;
+
+    /* Add bytes needed to hold multi bulk count */
+    totlen += 1+countDigits(argc)+2;
+
+    /* Build the command at protocol level */
+    cmd = hi_malloc(totlen+1);
+    if (cmd == NULL) goto memory_err;
+
+    pos = sprintf(cmd,"*%d\r\n",argc);
+    for (j = 0; j < argc; j++) {
+        pos += sprintf(cmd+pos,"$%zu\r\n",sdslen(curargv[j]));
+        memcpy(cmd+pos,curargv[j],sdslen(curargv[j]));
+        pos += sdslen(curargv[j]);
+        sdsfree(curargv[j]);
+        cmd[pos++] = '\r';
+        cmd[pos++] = '\n';
+    }
+    assert(pos == totlen);
+    cmd[pos] = '\0';
+
+    hi_free(curargv);
+    *target = cmd;
+    return totlen;
+
+format_err:
+    error_type = -2;
+    goto cleanup;
+
+memory_err:
+    error_type = -1;
+    goto cleanup;
+
+cleanup:
+    if (curargv) {
+        while(argc--)
+            sdsfree(curargv[argc]);
+        hi_free(curargv);
+    }
+
+    sdsfree(curarg);
+    hi_free(cmd);
+
+    return error_type;
+}
+
+/* Format a command according to the Redis protocol. This function
+ * takes a format similar to printf:
+ *
+ * %s represents a C null terminated string you want to interpolate
+ * %b represents a binary safe string
+ *
+ * When using %b you need to provide both the pointer to the string
+ * and the length in bytes as a size_t. Examples:
+ *
+ * len = redisFormatCommand(target, "GET %s", mykey);
+ * len = redisFormatCommand(target, "SET %s %b", mykey, myval, myvallen);
+ */
+int redisFormatCommand(char **target, const char *format, ...) {
+    va_list ap;
+    int len;
+    va_start(ap,format);
+    len = redisvFormatCommand(target,format,ap);
+    va_end(ap);
+
+    /* The API says "-1" means bad result, but we now also return "-2" in some
+     * cases.  Force the return value to always be -1. */
+    if (len < 0)
+        len = -1;
+
+    return len;
+}
+
+/* Format a command according to the Redis protocol using an sds string and
+ * sdscatfmt for the processing of arguments. This function takes the
+ * number of arguments, an array with arguments and an array with their
+ * lengths. If the latter is set to NULL, strlen will be used to compute the
+ * argument lengths.
+ */
+int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv,
+                              const size_t *argvlen)
+{
+    sds cmd, aux;
+    unsigned long long totlen;
+    int j;
+    size_t len;
+
+    /* Abort on a NULL target */
+    if (target == NULL)
+        return -1;
+
+    /* Calculate our total size */
+    totlen = 1+countDigits(argc)+2;
+    for (j = 0; j < argc; j++) {
+        len = argvlen ? argvlen[j] : strlen(argv[j]);
+        totlen += bulklen(len);
+    }
+
+    /* Use an SDS string for command construction */
+    cmd = sdsempty();
+    if (cmd == NULL)
+        return -1;
+
+    /* We already know how much storage we need */
+    aux = sdsMakeRoomFor(cmd, totlen);
+    if (aux == NULL) {
+        sdsfree(cmd);
+        return -1;
+    }
+
+    cmd = aux;
+
+    /* Construct command */
+    cmd = sdscatfmt(cmd, "*%i\r\n", argc);
+    for (j=0; j < argc; j++) {
+        len = argvlen ? argvlen[j] : strlen(argv[j]);
+        cmd = sdscatfmt(cmd, "$%u\r\n", len);
+        cmd = sdscatlen(cmd, argv[j], len);
+        cmd = sdscatlen(cmd, "\r\n", sizeof("\r\n")-1);
+    }
+
+    assert(sdslen(cmd)==totlen);
+
+    *target = cmd;
+    return totlen;
+}
+
+void redisFreeSdsCommand(sds cmd) {
+    sdsfree(cmd);
+}
+
+/* Format a command according to the Redis protocol. This function takes the
+ * number of arguments, an array with arguments and an array with their
+ * lengths. If the latter is set to NULL, strlen will be used to compute the
+ * argument lengths.
+ */
+int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen) {
+    char *cmd = NULL; /* final command */
+    int pos; /* position in final command */
+    size_t len;
+    int totlen, j;
+
+    /* Abort on a NULL target */
+    if (target == NULL)
+        return -1;
+
+    /* Calculate number of bytes needed for the command */
+    totlen = 1+countDigits(argc)+2;
+    for (j = 0; j < argc; j++) {
+        len = argvlen ? argvlen[j] : strlen(argv[j]);
+        totlen += bulklen(len);
+    }
+
+    /* Build the command at protocol level */
+    cmd = hi_malloc(totlen+1);
+    if (cmd == NULL)
+        return -1;
+
+    pos = sprintf(cmd,"*%d\r\n",argc);
+    for (j = 0; j < argc; j++) {
+        len = argvlen ? argvlen[j] : strlen(argv[j]);
+        pos += sprintf(cmd+pos,"$%zu\r\n",len);
+        memcpy(cmd+pos,argv[j],len);
+        pos += len;
+        cmd[pos++] = '\r';
+        cmd[pos++] = '\n';
+    }
+    assert(pos == totlen);
+    cmd[pos] = '\0';
+
+    *target = cmd;
+    return totlen;
+}
+
+void redisFreeCommand(char *cmd) {
+    hi_free(cmd);
+}
+
+void __redisSetError(redisContext *c, int type, const char *str) {
+    size_t len;
+
+    c->err = type;
+    if (str != NULL) {
+        len = strlen(str);
+        len = len < (sizeof(c->errstr)-1) ? len : (sizeof(c->errstr)-1);
+        memcpy(c->errstr,str,len);
+        c->errstr[len] = '\0';
+    } else {
+        /* Only REDIS_ERR_IO may lack a description! */
+        assert(type == REDIS_ERR_IO);
+        strerror_r(errno, c->errstr, sizeof(c->errstr));
+    }
+}
+
+redisReader *redisReaderCreate(void) {
+    return redisReaderCreateWithFunctions(&defaultFunctions);
+}
+
+static void redisPushAutoFree(void *privdata, void *reply) {
+    (void)privdata;
+    freeReplyObject(reply);
+}
+
+static redisContext *redisContextInit(void) {
+    redisContext *c;
+
+    c = hi_calloc(1, sizeof(*c));
+    if (c == NULL)
+        return NULL;
+
+    c->funcs = &redisContextDefaultFuncs;
+
+    c->obuf = sdsempty();
+    c->reader = redisReaderCreate();
+    c->fd = REDIS_INVALID_FD;
+
+    if (c->obuf == NULL || c->reader == NULL) {
+        redisFree(c);
+        return NULL;
+    }
+
+    return c;
+}
+
+void redisFree(redisContext *c) {
+    if (c == NULL)
+        return;
+    redisNetClose(c);
+
+    sdsfree(c->obuf);
+    redisReaderFree(c->reader);
+    hi_free(c->tcp.host);
+    hi_free(c->tcp.source_addr);
+    hi_free(c->unix_sock.path);
+    hi_free(c->connect_timeout);
+    hi_free(c->command_timeout);
+    hi_free(c->saddr);
+
+    if (c->privdata && c->free_privdata)
+        c->free_privdata(c->privdata);
+
+    if (c->funcs->free_privctx)
+        c->funcs->free_privctx(c->privctx);
+
+    memset(c, 0xff, sizeof(*c));
+    hi_free(c);
+}
+
+redisFD redisFreeKeepFd(redisContext *c) {
+    redisFD fd = c->fd;
+    c->fd = REDIS_INVALID_FD;
+    redisFree(c);
+    return fd;
+}
+
+int redisReconnect(redisContext *c) {
+    c->err = 0;
+    memset(c->errstr, '\0', strlen(c->errstr));
+
+    if (c->privctx && c->funcs->free_privctx) {
+        c->funcs->free_privctx(c->privctx);
+        c->privctx = NULL;
+    }
+
+    redisNetClose(c);
+
+    sdsfree(c->obuf);
+    redisReaderFree(c->reader);
+
+    c->obuf = sdsempty();
+    c->reader = redisReaderCreate();
+
+    if (c->obuf == NULL || c->reader == NULL) {
+        __redisSetError(c, REDIS_ERR_OOM, "Out of memory");
+        return REDIS_ERR;
+    }
+
+    int ret = REDIS_ERR;
+    if (c->connection_type == REDIS_CONN_TCP) {
+        ret = redisContextConnectBindTcp(c, c->tcp.host, c->tcp.port,
+               c->connect_timeout, c->tcp.source_addr);
+    } else if (c->connection_type == REDIS_CONN_UNIX) {
+        ret = redisContextConnectUnix(c, c->unix_sock.path, c->connect_timeout);
+    } else {
+        /* Something bad happened here and shouldn't have. There isn't
+           enough information in the context to reconnect. */
+        __redisSetError(c,REDIS_ERR_OTHER,"Not enough information to reconnect");
+        ret = REDIS_ERR;
+    }
+
+    if (c->command_timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) {
+        redisContextSetTimeout(c, *c->command_timeout);
+    }
+
+    return ret;
+}
+
+redisContext *redisConnectWithOptions(const redisOptions *options) {
+    redisContext *c = redisContextInit();
+    if (c == NULL) {
+        return NULL;
+    }
+    if (!(options->options & REDIS_OPT_NONBLOCK)) {
+        c->flags |= REDIS_BLOCK;
+    }
+    if (options->options & REDIS_OPT_REUSEADDR) {
+        c->flags |= REDIS_REUSEADDR;
+    }
+    if (options->options & REDIS_OPT_NOAUTOFREE) {
+        c->flags |= REDIS_NO_AUTO_FREE;
+    }
+
+    /* Set any user supplied RESP3 PUSH handler or use freeReplyObject
+     * as a default unless specifically flagged that we don't want one. */
+    if (options->push_cb != NULL)
+        redisSetPushCallback(c, options->push_cb);
+    else if (!(options->options & REDIS_OPT_NO_PUSH_AUTOFREE))
+        redisSetPushCallback(c, redisPushAutoFree);
+
+    c->privdata = options->privdata;
+    c->free_privdata = options->free_privdata;
+
+    if (redisContextUpdateConnectTimeout(c, options->connect_timeout) != REDIS_OK ||
+        redisContextUpdateCommandTimeout(c, options->command_timeout) != REDIS_OK) {
+        __redisSetError(c, REDIS_ERR_OOM, "Out of memory");
+        return c;
+    }
+
+    if (options->type == REDIS_CONN_TCP) {
+        redisContextConnectBindTcp(c, options->endpoint.tcp.ip,
+                                   options->endpoint.tcp.port, options->connect_timeout,
+                                   options->endpoint.tcp.source_addr);
+    } else if (options->type == REDIS_CONN_UNIX) {
+        redisContextConnectUnix(c, options->endpoint.unix_socket,
+                                options->connect_timeout);
+    } else if (options->type == REDIS_CONN_USERFD) {
+        c->fd = options->endpoint.fd;
+        c->flags |= REDIS_CONNECTED;
+    } else {
+        // Unknown type - FIXME - FREE
+        return NULL;
+    }
+
+    if (options->command_timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) {
+        redisContextSetTimeout(c, *options->command_timeout);
+    }
+
+    return c;
+}
+
+/* Connect to a Redis instance. On error the field error in the returned
+ * context will be set to the return value of the error function.
+ * When no set of reply functions is given, the default set will be used. */
+redisContext *redisConnect(const char *ip, int port) {
+    redisOptions options = {0};
+    REDIS_OPTIONS_SET_TCP(&options, ip, port);
+    return redisConnectWithOptions(&options);
+}
+
+redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) {
+    redisOptions options = {0};
+    REDIS_OPTIONS_SET_TCP(&options, ip, port);
+    options.connect_timeout = &tv;
+    return redisConnectWithOptions(&options);
+}
+
+redisContext *redisConnectNonBlock(const char *ip, int port) {
+    redisOptions options = {0};
+    REDIS_OPTIONS_SET_TCP(&options, ip, port);
+    options.options |= REDIS_OPT_NONBLOCK;
+    return redisConnectWithOptions(&options);
+}
+
+redisContext *redisConnectBindNonBlock(const char *ip, int port,
+                                       const char *source_addr) {
+    redisOptions options = {0};
+    REDIS_OPTIONS_SET_TCP(&options, ip, port);
+    options.endpoint.tcp.source_addr = source_addr;
+    options.options |= REDIS_OPT_NONBLOCK;
+    return redisConnectWithOptions(&options);
+}
+
+redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port,
+                                                const char *source_addr) {
+    redisOptions options = {0};
+    REDIS_OPTIONS_SET_TCP(&options, ip, port);
+    options.endpoint.tcp.source_addr = source_addr;
+    options.options |= REDIS_OPT_NONBLOCK|REDIS_OPT_REUSEADDR;
+    return redisConnectWithOptions(&options);
+}
+
+redisContext *redisConnectUnix(const char *path) {
+    redisOptions options = {0};
+    REDIS_OPTIONS_SET_UNIX(&options, path);
+    return redisConnectWithOptions(&options);
+}
+
+redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) {
+    redisOptions options = {0};
+    REDIS_OPTIONS_SET_UNIX(&options, path);
+    options.connect_timeout = &tv;
+    return redisConnectWithOptions(&options);
+}
+
+redisContext *redisConnectUnixNonBlock(const char *path) {
+    redisOptions options = {0};
+    REDIS_OPTIONS_SET_UNIX(&options, path);
+    options.options |= REDIS_OPT_NONBLOCK;
+    return redisConnectWithOptions(&options);
+}
+
+redisContext *redisConnectFd(redisFD fd) {
+    redisOptions options = {0};
+    options.type = REDIS_CONN_USERFD;
+    options.endpoint.fd = fd;
+    return redisConnectWithOptions(&options);
+}
+
+/* Set read/write timeout on a blocking socket. */
+int redisSetTimeout(redisContext *c, const struct timeval tv) {
+    if (c->flags & REDIS_BLOCK)
+        return redisContextSetTimeout(c,tv);
+    return REDIS_ERR;
+}
+
+/* Enable connection KeepAlive. */
+int redisEnableKeepAlive(redisContext *c) {
+    if (redisKeepAlive(c, REDIS_KEEPALIVE_INTERVAL) != REDIS_OK)
+        return REDIS_ERR;
+    return REDIS_OK;
+}
+
+/* Set a user provided RESP3 PUSH handler and return any old one set. */
+redisPushFn *redisSetPushCallback(redisContext *c, redisPushFn *fn) {
+    redisPushFn *old = c->push_cb;
+    c->push_cb = fn;
+    return old;
+}
+
+/* Use this function to handle a read event on the descriptor. It will try
+ * and read some bytes from the socket and feed them to the reply parser.
+ *
+ * After this function is called, you may use redisGetReplyFromReader to
+ * see if there is a reply available. */
+int redisBufferRead(redisContext *c) {
+    char buf[1024*16];
+    int nread;
+
+    /* Return early when the context has seen an error. */
+    if (c->err)
+        return REDIS_ERR;
+
+    nread = c->funcs->read(c, buf, sizeof(buf));
+    if (nread > 0) {
+        if (redisReaderFeed(c->reader, buf, nread) != REDIS_OK) {
+            __redisSetError(c, c->reader->err, c->reader->errstr);
+            return REDIS_ERR;
+        } else {
+        }
+    } else if (nread < 0) {
+        return REDIS_ERR;
+    }
+    return REDIS_OK;
+}
+
+/* Write the output buffer to the socket.
+ *
+ * Returns REDIS_OK when the buffer is empty, or (a part of) the buffer was
+ * successfully written to the socket. When the buffer is empty after the
+ * write operation, "done" is set to 1 (if given).
+ *
+ * Returns REDIS_ERR if an error occurred trying to write and sets
+ * c->errstr to hold the appropriate error string.
+ */
+int redisBufferWrite(redisContext *c, int *done) {
+
+    /* Return early when the context has seen an error. */
+    if (c->err)
+        return REDIS_ERR;
+
+    if (sdslen(c->obuf) > 0) {
+        ssize_t nwritten = c->funcs->write(c);
+        if (nwritten < 0) {
+            return REDIS_ERR;
+        } else if (nwritten > 0) {
+            if (nwritten == (ssize_t)sdslen(c->obuf)) {
+                sdsfree(c->obuf);
+                c->obuf = sdsempty();
+                if (c->obuf == NULL)
+                    goto oom;
+            } else {
+                if (sdsrange(c->obuf,nwritten,-1) < 0) goto oom;
+            }
+        }
+    }
+    if (done != NULL) *done = (sdslen(c->obuf) == 0);
+    return REDIS_OK;
+
+oom:
+    __redisSetError(c, REDIS_ERR_OOM, "Out of memory");
+    return REDIS_ERR;
+}
+
+/* Internal helper function to try and get a reply from the reader,
+ * or set an error in the context otherwise. */
+int redisGetReplyFromReader(redisContext *c, void **reply) {
+    if (redisReaderGetReply(c->reader,reply) == REDIS_ERR) {
+        __redisSetError(c,c->reader->err,c->reader->errstr);
+        return REDIS_ERR;
+    }
+
+    return REDIS_OK;
+}
+
+/* Internal helper that returns 1 if the reply was a RESP3 PUSH
+ * message and we handled it with a user-provided callback. */
+static int redisHandledPushReply(redisContext *c, void *reply) {
+    if (reply && c->push_cb && redisIsPushReply(reply)) {
+        c->push_cb(c->privdata, reply);
+        return 1;
+    }
+
+    return 0;
+}
+
+int redisGetReply(redisContext *c, void **reply) {
+    int wdone = 0;
+    void *aux = NULL;
+
+    /* Try to read pending replies */
+    if (redisGetReplyFromReader(c,&aux) == REDIS_ERR)
+        return REDIS_ERR;
+
+    /* For the blocking context, flush output buffer and read reply */
+    if (aux == NULL && c->flags & REDIS_BLOCK) {
+        /* Write until done */
+        do {
+            if (redisBufferWrite(c,&wdone) == REDIS_ERR)
+                return REDIS_ERR;
+        } while (!wdone);
+
+        /* Read until there is a reply */
+        do {
+            if (redisBufferRead(c) == REDIS_ERR)
+                return REDIS_ERR;
+
+            /* We loop here in case the user has specified a RESP3
+             * PUSH handler (e.g. for client tracking). */
+            do {
+                if (redisGetReplyFromReader(c,&aux) == REDIS_ERR)
+                    return REDIS_ERR;
+            } while (redisHandledPushReply(c, aux));
+        } while (aux == NULL);
+    }
+
+    /* Set reply or free it if we were passed NULL */
+    if (reply != NULL) {
+        *reply = aux;
+    } else {
+        freeReplyObject(aux);
+    }
+
+    return REDIS_OK;
+}
+
+
+/* Helper function for the redisAppendCommand* family of functions.
+ *
+ * Write a formatted command to the output buffer. When this family
+ * is used, you need to call redisGetReply yourself to retrieve
+ * the reply (or replies in pub/sub).
+ */
+int __redisAppendCommand(redisContext *c, const char *cmd, size_t len) {
+    sds newbuf;
+
+    newbuf = sdscatlen(c->obuf,cmd,len);
+    if (newbuf == NULL) {
+        __redisSetError(c,REDIS_ERR_OOM,"Out of memory");
+        return REDIS_ERR;
+    }
+
+    c->obuf = newbuf;
+    return REDIS_OK;
+}
+
+int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len) {
+
+    if (__redisAppendCommand(c, cmd, len) != REDIS_OK) {
+        return REDIS_ERR;
+    }
+
+    return REDIS_OK;
+}
+
+int redisvAppendCommand(redisContext *c, const char *format, va_list ap) {
+    char *cmd;
+    int len;
+
+    len = redisvFormatCommand(&cmd,format,ap);
+    if (len == -1) {
+        __redisSetError(c,REDIS_ERR_OOM,"Out of memory");
+        return REDIS_ERR;
+    } else if (len == -2) {
+        __redisSetError(c,REDIS_ERR_OTHER,"Invalid format string");
+        return REDIS_ERR;
+    }
+
+    if (__redisAppendCommand(c,cmd,len) != REDIS_OK) {
+        hi_free(cmd);
+        return REDIS_ERR;
+    }
+
+    hi_free(cmd);
+    return REDIS_OK;
+}
+
+int redisAppendCommand(redisContext *c, const char *format, ...) {
+    va_list ap;
+    int ret;
+
+    va_start(ap,format);
+    ret = redisvAppendCommand(c,format,ap);
+    va_end(ap);
+    return ret;
+}
+
+int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) {
+    sds cmd;
+    int len;
+
+    len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen);
+    if (len == -1) {
+        __redisSetError(c,REDIS_ERR_OOM,"Out of memory");
+        return REDIS_ERR;
+    }
+
+    if (__redisAppendCommand(c,cmd,len) != REDIS_OK) {
+        sdsfree(cmd);
+        return REDIS_ERR;
+    }
+
+    sdsfree(cmd);
+    return REDIS_OK;
+}
+
+/* Helper function for the redisCommand* family of functions.
+ *
+ * Write a formatted command to the output buffer. If the given context is
+ * blocking, immediately read the reply into the "reply" pointer. When the
+ * context is non-blocking, the "reply" pointer will not be used and the
+ * command is simply appended to the write buffer.
+ *
+ * Returns the reply when a reply was successfully retrieved. Returns NULL
+ * otherwise. When NULL is returned in a blocking context, the error field
+ * in the context will be set.
+ */
+static void *__redisBlockForReply(redisContext *c) {
+    void *reply;
+
+    if (c->flags & REDIS_BLOCK) {
+        if (redisGetReply(c,&reply) != REDIS_OK)
+            return NULL;
+        return reply;
+    }
+    return NULL;
+}
+
+void *redisvCommand(redisContext *c, const char *format, va_list ap) {
+    if (redisvAppendCommand(c,format,ap) != REDIS_OK)
+        return NULL;
+    return __redisBlockForReply(c);
+}
+
+void *redisCommand(redisContext *c, const char *format, ...) {
+    va_list ap;
+    va_start(ap,format);
+    void *reply = redisvCommand(c,format,ap);
+    va_end(ap);
+    return reply;
+}
+
+void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) {
+    if (redisAppendCommandArgv(c,argc,argv,argvlen) != REDIS_OK)
+        return NULL;
+    return __redisBlockForReply(c);
+}
diff --git a/hiredis.h b/hiredis.h
new file mode 100644
index 000000000..0622aaab8
--- /dev/null
+++ b/hiredis.h
@@ -0,0 +1,336 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
+ *                     Jan-Erik Rediger <janerik at fnordig dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_H
+#define __HIREDIS_H
+#include "read.h"
+#include <stdarg.h> /* for va_list */
+#ifndef _MSC_VER
+#include <sys/time.h> /* for struct timeval */
+#else
+struct timeval; /* forward declaration */
+typedef long long ssize_t;
+#endif
+#include <stdint.h> /* uintXX_t, etc */
+#include "sds.h" /* for sds */
+#include "alloc.h" /* for allocation wrappers */
+
+#define HIREDIS_MAJOR 1
+#define HIREDIS_MINOR 0
+#define HIREDIS_PATCH 0
+#define HIREDIS_SONAME 1.0.0
+
+/* Connection type can be blocking or non-blocking and is set in the
+ * least significant bit of the flags field in redisContext. */
+#define REDIS_BLOCK 0x1
+
+/* Connection may be disconnected before being free'd. The second bit
+ * in the flags field is set when the context is connected. */
+#define REDIS_CONNECTED 0x2
+
+/* The async API might try to disconnect cleanly and flush the output
+ * buffer and read all subsequent replies before disconnecting.
+ * This flag means no new commands can come in and the connection
+ * should be terminated once all replies have been read. */
+#define REDIS_DISCONNECTING 0x4
+
+/* Flag specific to the async API which means that the context should be clean
+ * up as soon as possible. */
+#define REDIS_FREEING 0x8
+
+/* Flag that is set when an async callback is executed. */
+#define REDIS_IN_CALLBACK 0x10
+
+/* Flag that is set when the async context has one or more subscriptions. */
+#define REDIS_SUBSCRIBED 0x20
+
+/* Flag that is set when monitor mode is active */
+#define REDIS_MONITORING 0x40
+
+/* Flag that is set when we should set SO_REUSEADDR before calling bind() */
+#define REDIS_REUSEADDR 0x80
+
+/**
+ * Flag that indicates the user does not want the context to
+ * be automatically freed upon error
+ */
+#define REDIS_NO_AUTO_FREE 0x200
+
+#define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */
+
+/* number of times we retry to connect in the case of EADDRNOTAVAIL and
+ * SO_REUSEADDR is being used. */
+#define REDIS_CONNECT_RETRIES  10
+
+/* Forward declarations for structs defined elsewhere */
+struct redisAsyncContext;
+struct redisContext;
+
+/* RESP3 push helpers and callback prototypes */
+#define redisIsPushReply(r) (((redisReply*)(r))->type == REDIS_REPLY_PUSH)
+typedef void (redisPushFn)(void *, void *);
+typedef void (redisAsyncPushFn)(struct redisAsyncContext *, void *);
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* This is the reply object returned by redisCommand() */
+typedef struct redisReply {
+    int type; /* REDIS_REPLY_* */
+    long long integer; /* The integer when type is REDIS_REPLY_INTEGER */
+    double dval; /* The double when type is REDIS_REPLY_DOUBLE */
+    size_t len; /* Length of string */
+    char *str; /* Used for REDIS_REPLY_ERROR, REDIS_REPLY_STRING
+                  REDIS_REPLY_VERB, and REDIS_REPLY_DOUBLE (in additional to dval). */
+    char vtype[4]; /* Used for REDIS_REPLY_VERB, contains the null
+                      terminated 3 character content type, such as "txt". */
+    size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */
+    struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */
+} redisReply;
+
+redisReader *redisReaderCreate(void);
+
+/* Function to free the reply objects hiredis returns by default. */
+void freeReplyObject(void *reply);
+
+/* Functions to format a command according to the protocol. */
+int redisvFormatCommand(char **target, const char *format, va_list ap);
+int redisFormatCommand(char **target, const char *format, ...);
+int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen);
+int redisFormatSdsCommandArgv(sds *target, int argc, const char ** argv, const size_t *argvlen);
+void redisFreeCommand(char *cmd);
+void redisFreeSdsCommand(sds cmd);
+
+enum redisConnectionType {
+    REDIS_CONN_TCP,
+    REDIS_CONN_UNIX,
+    REDIS_CONN_USERFD
+};
+
+struct redisSsl;
+
+#define REDIS_OPT_NONBLOCK 0x01
+#define REDIS_OPT_REUSEADDR 0x02
+
+/**
+ * Don't automatically free the async object on a connection failure,
+ * or other implicit conditions. Only free on an explicit call to disconnect() or free()
+ */
+#define REDIS_OPT_NOAUTOFREE 0x04
+
+/* Don't automatically intercept and free RESP3 PUSH replies. */
+#define REDIS_OPT_NO_PUSH_AUTOFREE 0x08
+
+/* In Unix systems a file descriptor is a regular signed int, with -1
+ * representing an invalid descriptor. In Windows it is a SOCKET
+ * (32- or 64-bit unsigned integer depending on the architecture), where
+ * all bits set (~0) is INVALID_SOCKET.  */
+#ifndef _WIN32
+typedef int redisFD;
+#define REDIS_INVALID_FD -1
+#else
+#ifdef _WIN64
+typedef unsigned long long redisFD; /* SOCKET = 64-bit UINT_PTR */
+#else
+typedef unsigned long redisFD;      /* SOCKET = 32-bit UINT_PTR */
+#endif
+#define REDIS_INVALID_FD ((redisFD)(~0)) /* INVALID_SOCKET */
+#endif
+
+typedef struct {
+    /*
+     * the type of connection to use. This also indicates which
+     * `endpoint` member field to use
+     */
+    int type;
+    /* bit field of REDIS_OPT_xxx */
+    int options;
+    /* timeout value for connect operation. If NULL, no timeout is used */
+    const struct timeval *connect_timeout;
+    /* timeout value for commands. If NULL, no timeout is used.  This can be
+     * updated at runtime with redisSetTimeout/redisAsyncSetTimeout. */
+    const struct timeval *command_timeout;
+    union {
+        /** use this field for tcp/ip connections */
+        struct {
+            const char *source_addr;
+            const char *ip;
+            int port;
+        } tcp;
+        /** use this field for unix domain sockets */
+        const char *unix_socket;
+        /**
+         * use this field to have hiredis operate an already-open
+         * file descriptor */
+        redisFD fd;
+    } endpoint;
+
+    /* Optional user defined data/destructor */
+    void *privdata;
+    void (*free_privdata)(void *);
+
+    /* A user defined PUSH message callback */
+    redisPushFn *push_cb;
+    redisAsyncPushFn *async_push_cb;
+} redisOptions;
+
+/**
+ * Helper macros to initialize options to their specified fields.
+ */
+#define REDIS_OPTIONS_SET_TCP(opts, ip_, port_) \
+    (opts)->type = REDIS_CONN_TCP; \
+    (opts)->endpoint.tcp.ip = ip_; \
+    (opts)->endpoint.tcp.port = port_;
+
+#define REDIS_OPTIONS_SET_UNIX(opts, path) \
+    (opts)->type = REDIS_CONN_UNIX;        \
+    (opts)->endpoint.unix_socket = path;
+
+#define REDIS_OPTIONS_SET_PRIVDATA(opts, data, dtor) \
+    (opts)->privdata = data;                         \
+    (opts)->free_privdata = dtor;                    \
+
+typedef struct redisContextFuncs {
+    void (*free_privctx)(void *);
+    void (*async_read)(struct redisAsyncContext *);
+    void (*async_write)(struct redisAsyncContext *);
+    ssize_t (*read)(struct redisContext *, char *, size_t);
+    ssize_t (*write)(struct redisContext *);
+} redisContextFuncs;
+
+/* Context for a connection to Redis */
+typedef struct redisContext {
+    const redisContextFuncs *funcs;   /* Function table */
+
+    int err; /* Error flags, 0 when there is no error */
+    char errstr[128]; /* String representation of error when applicable */
+    redisFD fd;
+    int flags;
+    char *obuf; /* Write buffer */
+    redisReader *reader; /* Protocol reader */
+
+    enum redisConnectionType connection_type;
+    struct timeval *connect_timeout;
+    struct timeval *command_timeout;
+
+    struct {
+        char *host;
+        char *source_addr;
+        int port;
+    } tcp;
+
+    struct {
+        char *path;
+    } unix_sock;
+
+    /* For non-blocking connect */
+    struct sockadr *saddr;
+    size_t addrlen;
+
+    /* Optional data and corresponding destructor users can use to provide
+     * context to a given redisContext.  Not used by hiredis. */
+    void *privdata;
+    void (*free_privdata)(void *);
+
+    /* Internal context pointer presently used by hiredis to manage
+     * SSL connections. */
+    void *privctx;
+
+    /* An optional RESP3 PUSH handler */
+    redisPushFn *push_cb;
+} redisContext;
+
+redisContext *redisConnectWithOptions(const redisOptions *options);
+redisContext *redisConnect(const char *ip, int port);
+redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv);
+redisContext *redisConnectNonBlock(const char *ip, int port);
+redisContext *redisConnectBindNonBlock(const char *ip, int port,
+                                       const char *source_addr);
+redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port,
+                                                const char *source_addr);
+redisContext *redisConnectUnix(const char *path);
+redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv);
+redisContext *redisConnectUnixNonBlock(const char *path);
+redisContext *redisConnectFd(redisFD fd);
+
+/**
+ * Reconnect the given context using the saved information.
+ *
+ * This re-uses the exact same connect options as in the initial connection.
+ * host, ip (or path), timeout and bind address are reused,
+ * flags are used unmodified from the existing context.
+ *
+ * Returns REDIS_OK on successful connect or REDIS_ERR otherwise.
+ */
+int redisReconnect(redisContext *c);
+
+redisPushFn *redisSetPushCallback(redisContext *c, redisPushFn *fn);
+int redisSetTimeout(redisContext *c, const struct timeval tv);
+int redisEnableKeepAlive(redisContext *c);
+void redisFree(redisContext *c);
+redisFD redisFreeKeepFd(redisContext *c);
+int redisBufferRead(redisContext *c);
+int redisBufferWrite(redisContext *c, int *done);
+
+/* In a blocking context, this function first checks if there are unconsumed
+ * replies to return and returns one if so. Otherwise, it flushes the output
+ * buffer to the socket and reads until it has a reply. In a non-blocking
+ * context, it will return unconsumed replies until there are no more. */
+int redisGetReply(redisContext *c, void **reply);
+int redisGetReplyFromReader(redisContext *c, void **reply);
+
+/* Write a formatted command to the output buffer. Use these functions in blocking mode
+ * to get a pipeline of commands. */
+int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len);
+
+/* Write a command to the output buffer. Use these functions in blocking mode
+ * to get a pipeline of commands. */
+int redisvAppendCommand(redisContext *c, const char *format, va_list ap);
+int redisAppendCommand(redisContext *c, const char *format, ...);
+int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
+
+/* Issue a command to Redis. In a blocking context, it is identical to calling
+ * redisAppendCommand, followed by redisGetReply. The function will return
+ * NULL if there was an error in performing the request, otherwise it will
+ * return the reply. In a non-blocking context, it is identical to calling
+ * only redisAppendCommand and will always return NULL. */
+void *redisvCommand(redisContext *c, const char *format, va_list ap);
+void *redisCommand(redisContext *c, const char *format, ...);
+void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/hiredis.pc.in b/hiredis.pc.in
new file mode 100644
index 000000000..91b773183
--- /dev/null
+++ b/hiredis.pc.in
@@ -0,0 +1,12 @@
+prefix=@CMAKE_INSTALL_PREFIX@
+install_libdir=@CMAKE_INSTALL_LIBDIR@
+exec_prefix=${prefix}
+libdir=${exec_prefix}/${install_libdir}
+includedir=${prefix}/include
+pkgincludedir=${includedir}/hiredis
+
+Name: hiredis
+Description: Minimalistic C client library for Redis.
+Version: @PROJECT_VERSION@
+Libs: -L${libdir} -lhiredis
+Cflags: -I${pkgincludedir} -D_FILE_OFFSET_BITS=64
diff --git a/hiredis_ssl-config.cmake.in b/hiredis_ssl-config.cmake.in
new file mode 100644
index 000000000..9a283dfc2
--- /dev/null
+++ b/hiredis_ssl-config.cmake.in
@@ -0,0 +1,13 @@
+@PACKAGE_INIT@
+
+set_and_check(hiredis_ssl_INCLUDEDIR "@PACKAGE_INCLUDE_INSTALL_DIR@")
+
+IF (NOT TARGET hiredis::hiredis_ssl)
+	INCLUDE(${CMAKE_CURRENT_LIST_DIR}/hiredis_ssl-targets.cmake)
+ENDIF()
+
+SET(hiredis_ssl_LIBRARIES hiredis::hiredis_ssl)
+SET(hiredis_ssl_INCLUDE_DIRS ${hiredis_ssl_INCLUDEDIR})
+
+check_required_components(hiredis_ssl)
+
diff --git a/hiredis_ssl.h b/hiredis_ssl.h
new file mode 100644
index 000000000..604efe0c1
--- /dev/null
+++ b/hiredis_ssl.h
@@ -0,0 +1,127 @@
+
+/*
+ * Copyright (c) 2019, Redis Labs
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIREDIS_SSL_H
+#define __HIREDIS_SSL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* This is the underlying struct for SSL in ssl.h, which is not included to
+ * keep build dependencies short here.
+ */
+struct ssl_st;
+
+/* A wrapper around OpenSSL SSL_CTX to allow easy SSL use without directly
+ * calling OpenSSL.
+ */
+typedef struct redisSSLContext redisSSLContext;
+
+/**
+ * Initialization errors that redisCreateSSLContext() may return.
+ */
+
+typedef enum {
+    REDIS_SSL_CTX_NONE = 0,                     /* No Error */
+    REDIS_SSL_CTX_CREATE_FAILED,                /* Failed to create OpenSSL SSL_CTX */
+    REDIS_SSL_CTX_CERT_KEY_REQUIRED,            /* Client cert and key must both be specified or skipped */
+    REDIS_SSL_CTX_CA_CERT_LOAD_FAILED,          /* Failed to load CA Certificate or CA Path */
+    REDIS_SSL_CTX_CLIENT_CERT_LOAD_FAILED,      /* Failed to load client certificate */
+    REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED       /* Failed to load private key */
+} redisSSLContextError;
+
+/**
+ * Return the error message corresponding with the specified error code.
+ */
+
+const char *redisSSLContextGetError(redisSSLContextError error);
+
+/**
+ * Helper function to initialize the OpenSSL library.
+ *
+ * OpenSSL requires one-time initialization before it can be used. Callers should
+ * call this function only once, and only if OpenSSL is not directly initialized
+ * elsewhere.
+ */
+int redisInitOpenSSL(void);
+
+/**
+ * Helper function to initialize an OpenSSL context that can be used
+ * to initiate SSL connections.
+ *
+ * cacert_filename is an optional name of a CA certificate/bundle file to load
+ * and use for validation.
+ *
+ * capath is an optional directory path where trusted CA certificate files are
+ * stored in an OpenSSL-compatible structure.
+ *
+ * cert_filename and private_key_filename are optional names of a client side
+ * certificate and private key files to use for authentication. They need to
+ * be both specified or omitted.
+ *
+ * server_name is an optional and will be used as a server name indication
+ * (SNI) TLS extension.
+ *
+ * If error is non-null, it will be populated in case the context creation fails
+ * (returning a NULL).
+ */
+
+redisSSLContext *redisCreateSSLContext(const char *cacert_filename, const char *capath,
+        const char *cert_filename, const char *private_key_filename,
+        const char *server_name, redisSSLContextError *error);
+
+/**
+ * Free a previously created OpenSSL context.
+ */
+void redisFreeSSLContext(redisSSLContext *redis_ssl_ctx);
+
+/**
+ * Initiate SSL on an existing redisContext.
+ *
+ * This is similar to redisInitiateSSL() but does not require the caller
+ * to directly interact with OpenSSL, and instead uses a redisSSLContext
+ * previously created using redisCreateSSLContext().
+ */
+
+int redisInitiateSSLWithContext(redisContext *c, redisSSLContext *redis_ssl_ctx);
+
+/**
+ * Initiate SSL/TLS negotiation on a provided OpenSSL SSL object.
+ */
+
+int redisInitiateSSL(redisContext *c, struct ssl_st *ssl);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  /* __HIREDIS_SSL_H */
diff --git a/hiredis_ssl.pc.in b/hiredis_ssl.pc.in
new file mode 100644
index 000000000..588a978a5
--- /dev/null
+++ b/hiredis_ssl.pc.in
@@ -0,0 +1,12 @@
+prefix=@CMAKE_INSTALL_PREFIX@
+exec_prefix=${prefix}
+libdir=${exec_prefix}/lib
+includedir=${prefix}/include
+pkgincludedir=${includedir}/hiredis
+
+Name: hiredis_ssl
+Description: SSL Support for hiredis.
+Version: @PROJECT_VERSION@
+Requires: hiredis
+Libs: -L${libdir} -lhiredis_ssl
+Libs.private: -lssl -lcrypto
diff --git a/net.c b/net.c
new file mode 100644
index 000000000..c6b0e5d8e
--- /dev/null
+++ b/net.c
@@ -0,0 +1,612 @@
+/* Extracted from anet.c to work properly with Hiredis error reporting.
+ *
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
+ *                     Jan-Erik Rediger <janerik at fnordig dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fmacros.h"
+#include <sys/types.h>
+#include <fcntl.h>
+#include <string.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <limits.h>
+#include <stdlib.h>
+
+#include "net.h"
+#include "sds.h"
+#include "sockcompat.h"
+#include "win32.h"
+
+/* Defined in hiredis.c */
+void __redisSetError(redisContext *c, int type, const char *str);
+
+void redisNetClose(redisContext *c) {
+    if (c && c->fd != REDIS_INVALID_FD) {
+        close(c->fd);
+        c->fd = REDIS_INVALID_FD;
+    }
+}
+
+ssize_t redisNetRead(redisContext *c, char *buf, size_t bufcap) {
+    ssize_t nread = recv(c->fd, buf, bufcap, 0);
+    if (nread == -1) {
+        if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) {
+            /* Try again later */
+            return 0;
+        } else if(errno == ETIMEDOUT && (c->flags & REDIS_BLOCK)) {
+            /* especially in windows */
+            __redisSetError(c, REDIS_ERR_TIMEOUT, "recv timeout");
+            return -1;
+        } else {
+            __redisSetError(c, REDIS_ERR_IO, NULL);
+            return -1;
+        }
+    } else if (nread == 0) {
+        __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection");
+        return -1;
+    } else {
+        return nread;
+    }
+}
+
+ssize_t redisNetWrite(redisContext *c) {
+    ssize_t nwritten = send(c->fd, c->obuf, sdslen(c->obuf), 0);
+    if (nwritten < 0) {
+        if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) {
+            /* Try again later */
+        } else {
+            __redisSetError(c, REDIS_ERR_IO, NULL);
+            return -1;
+        }
+    }
+    return nwritten;
+}
+
+static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) {
+    int errorno = errno;  /* snprintf() may change errno */
+    char buf[128] = { 0 };
+    size_t len = 0;
+
+    if (prefix != NULL)
+        len = snprintf(buf,sizeof(buf),"%s: ",prefix);
+    strerror_r(errorno, (char *)(buf + len), sizeof(buf) - len);
+    __redisSetError(c,type,buf);
+}
+
+static int redisSetReuseAddr(redisContext *c) {
+    int on = 1;
+    if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) {
+        __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
+        redisNetClose(c);
+        return REDIS_ERR;
+    }
+    return REDIS_OK;
+}
+
+static int redisCreateSocket(redisContext *c, int type) {
+    redisFD s;
+    if ((s = socket(type, SOCK_STREAM, 0)) == REDIS_INVALID_FD) {
+        __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
+        return REDIS_ERR;
+    }
+    c->fd = s;
+    if (type == AF_INET) {
+        if (redisSetReuseAddr(c) == REDIS_ERR) {
+            return REDIS_ERR;
+        }
+    }
+    return REDIS_OK;
+}
+
+static int redisSetBlocking(redisContext *c, int blocking) {
+#ifndef _WIN32
+    int flags;
+
+    /* Set the socket nonblocking.
+     * Note that fcntl(2) for F_GETFL and F_SETFL can't be
+     * interrupted by a signal. */
+    if ((flags = fcntl(c->fd, F_GETFL)) == -1) {
+        __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)");
+        redisNetClose(c);
+        return REDIS_ERR;
+    }
+
+    if (blocking)
+        flags &= ~O_NONBLOCK;
+    else
+        flags |= O_NONBLOCK;
+
+    if (fcntl(c->fd, F_SETFL, flags) == -1) {
+        __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)");
+        redisNetClose(c);
+        return REDIS_ERR;
+    }
+#else
+    u_long mode = blocking ? 0 : 1;
+    if (ioctl(c->fd, FIONBIO, &mode) == -1) {
+        __redisSetErrorFromErrno(c, REDIS_ERR_IO, "ioctl(FIONBIO)");
+        redisNetClose(c);
+        return REDIS_ERR;
+    }
+#endif /* _WIN32 */
+    return REDIS_OK;
+}
+
+int redisKeepAlive(redisContext *c, int interval) {
+    int val = 1;
+    redisFD fd = c->fd;
+
+    if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){
+        __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
+        return REDIS_ERR;
+    }
+
+    val = interval;
+
+#if defined(__APPLE__) && defined(__MACH__)
+    if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &val, sizeof(val)) < 0) {
+        __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
+        return REDIS_ERR;
+    }
+#else
+#if defined(__GLIBC__) && !defined(__FreeBSD_kernel__)
+    if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) {
+        __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
+        return REDIS_ERR;
+    }
+
+    val = interval/3;
+    if (val == 0) val = 1;
+    if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &val, sizeof(val)) < 0) {
+        __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
+        return REDIS_ERR;
+    }
+
+    val = 3;
+    if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &val, sizeof(val)) < 0) {
+        __redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
+        return REDIS_ERR;
+    }
+#endif
+#endif
+
+    return REDIS_OK;
+}
+
+int redisSetTcpNoDelay(redisContext *c) {
+    int yes = 1;
+    if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) {
+        __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)");
+        redisNetClose(c);
+        return REDIS_ERR;
+    }
+    return REDIS_OK;
+}
+
+#define __MAX_MSEC (((LONG_MAX) - 999) / 1000)
+
+static int redisContextTimeoutMsec(redisContext *c, long *result)
+{
+    const struct timeval *timeout = c->connect_timeout;
+    long msec = -1;
+
+    /* Only use timeout when not NULL. */
+    if (timeout != NULL) {
+        if (timeout->tv_usec > 1000000 || timeout->tv_sec > __MAX_MSEC) {
+            *result = msec;
+            return REDIS_ERR;
+        }
+
+        msec = (timeout->tv_sec * 1000) + ((timeout->tv_usec + 999) / 1000);
+
+        if (msec < 0 || msec > INT_MAX) {
+            msec = INT_MAX;
+        }
+    }
+
+    *result = msec;
+    return REDIS_OK;
+}
+
+static int redisContextWaitReady(redisContext *c, long msec) {
+    struct pollfd   wfd[1];
+
+    wfd[0].fd     = c->fd;
+    wfd[0].events = POLLOUT;
+
+    if (errno == EINPROGRESS) {
+        int res;
+
+        if ((res = poll(wfd, 1, msec)) == -1) {
+            __redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)");
+            redisNetClose(c);
+            return REDIS_ERR;
+        } else if (res == 0) {
+            errno = ETIMEDOUT;
+            __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
+            redisNetClose(c);
+            return REDIS_ERR;
+        }
+
+        if (redisCheckConnectDone(c, &res) != REDIS_OK || res == 0) {
+            redisCheckSocketError(c);
+            return REDIS_ERR;
+        }
+
+        return REDIS_OK;
+    }
+
+    __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
+    redisNetClose(c);
+    return REDIS_ERR;
+}
+
+int redisCheckConnectDone(redisContext *c, int *completed) {
+    int rc = connect(c->fd, (const struct sockaddr *)c->saddr, c->addrlen);
+    if (rc == 0) {
+        *completed = 1;
+        return REDIS_OK;
+    }
+    switch (errno) {
+    case EISCONN:
+        *completed = 1;
+        return REDIS_OK;
+    case EALREADY:
+    case EINPROGRESS:
+    case EWOULDBLOCK:
+        *completed = 0;
+        return REDIS_OK;
+    default:
+        return REDIS_ERR;
+    }
+}
+
+int redisCheckSocketError(redisContext *c) {
+    int err = 0, errno_saved = errno;
+    socklen_t errlen = sizeof(err);
+
+    if (getsockopt(c->fd, SOL_SOCKET, SO_ERROR, &err, &errlen) == -1) {
+        __redisSetErrorFromErrno(c,REDIS_ERR_IO,"getsockopt(SO_ERROR)");
+        return REDIS_ERR;
+    }
+
+    if (err == 0) {
+        err = errno_saved;
+    }
+
+    if (err) {
+        errno = err;
+        __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
+        return REDIS_ERR;
+    }
+
+    return REDIS_OK;
+}
+
+int redisContextSetTimeout(redisContext *c, const struct timeval tv) {
+    const void *to_ptr = &tv;
+    size_t to_sz = sizeof(tv);
+
+    if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,to_ptr,to_sz) == -1) {
+        __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)");
+        return REDIS_ERR;
+    }
+    if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,to_ptr,to_sz) == -1) {
+        __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_SNDTIMEO)");
+        return REDIS_ERR;
+    }
+    return REDIS_OK;
+}
+
+int redisContextUpdateConnectTimeout(redisContext *c, const struct timeval *timeout) {
+    /* Same timeval struct, short circuit */
+    if (c->connect_timeout == timeout)
+        return REDIS_OK;
+
+    /* Allocate context timeval if we need to */
+    if (c->connect_timeout == NULL) {
+        c->connect_timeout = hi_malloc(sizeof(*c->connect_timeout));
+        if (c->connect_timeout == NULL)
+            return REDIS_ERR;
+    }
+
+    memcpy(c->connect_timeout, timeout, sizeof(*c->connect_timeout));
+    return REDIS_OK;
+}
+
+int redisContextUpdateCommandTimeout(redisContext *c, const struct timeval *timeout) {
+    /* Same timeval struct, short circuit */
+    if (c->command_timeout == timeout)
+        return REDIS_OK;
+
+    /* Allocate context timeval if we need to */
+    if (c->command_timeout == NULL) {
+        c->command_timeout = hi_malloc(sizeof(*c->command_timeout));
+        if (c->command_timeout == NULL)
+            return REDIS_ERR;
+    }
+
+    memcpy(c->command_timeout, timeout, sizeof(*c->command_timeout));
+    return REDIS_OK;
+}
+
+static int _redisContextConnectTcp(redisContext *c, const char *addr, int port,
+                                   const struct timeval *timeout,
+                                   const char *source_addr) {
+    redisFD s;
+    int rv, n;
+    char _port[6];  /* strlen("65535"); */
+    struct addrinfo hints, *servinfo, *bservinfo, *p, *b;
+    int blocking = (c->flags & REDIS_BLOCK);
+    int reuseaddr = (c->flags & REDIS_REUSEADDR);
+    int reuses = 0;
+    long timeout_msec = -1;
+
+    servinfo = NULL;
+    c->connection_type = REDIS_CONN_TCP;
+    c->tcp.port = port;
+
+    /* We need to take possession of the passed parameters
+     * to make them reusable for a reconnect.
+     * We also carefully check we don't free data we already own,
+     * as in the case of the reconnect method.
+     *
+     * This is a bit ugly, but atleast it works and doesn't leak memory.
+     **/
+    if (c->tcp.host != addr) {
+        hi_free(c->tcp.host);
+
+        c->tcp.host = hi_strdup(addr);
+        if (c->tcp.host == NULL)
+            goto oom;
+    }
+
+    if (timeout) {
+        if (redisContextUpdateConnectTimeout(c, timeout) == REDIS_ERR)
+            goto oom;
+    } else {
+        hi_free(c->connect_timeout);
+        c->connect_timeout = NULL;
+    }
+
+    if (redisContextTimeoutMsec(c, &timeout_msec) != REDIS_OK) {
+        __redisSetError(c, REDIS_ERR_IO, "Invalid timeout specified");
+        goto error;
+    }
+
+    if (source_addr == NULL) {
+        hi_free(c->tcp.source_addr);
+        c->tcp.source_addr = NULL;
+    } else if (c->tcp.source_addr != source_addr) {
+        hi_free(c->tcp.source_addr);
+        c->tcp.source_addr = hi_strdup(source_addr);
+    }
+
+    snprintf(_port, 6, "%d", port);
+    memset(&hints,0,sizeof(hints));
+    hints.ai_family = AF_INET;
+    hints.ai_socktype = SOCK_STREAM;
+
+    /* Try with IPv6 if no IPv4 address was found. We do it in this order since
+     * in a Redis client you can't afford to test if you have IPv6 connectivity
+     * as this would add latency to every connect. Otherwise a more sensible
+     * route could be: Use IPv6 if both addresses are available and there is IPv6
+     * connectivity. */
+    if ((rv = getaddrinfo(c->tcp.host,_port,&hints,&servinfo)) != 0) {
+         hints.ai_family = AF_INET6;
+         if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) {
+            __redisSetError(c,REDIS_ERR_OTHER,gai_strerror(rv));
+            return REDIS_ERR;
+        }
+    }
+    for (p = servinfo; p != NULL; p = p->ai_next) {
+addrretry:
+        if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == REDIS_INVALID_FD)
+            continue;
+
+        c->fd = s;
+        if (redisSetBlocking(c,0) != REDIS_OK)
+            goto error;
+        if (c->tcp.source_addr) {
+            int bound = 0;
+            /* Using getaddrinfo saves us from self-determining IPv4 vs IPv6 */
+            if ((rv = getaddrinfo(c->tcp.source_addr, NULL, &hints, &bservinfo)) != 0) {
+                char buf[128];
+                snprintf(buf,sizeof(buf),"Can't get addr: %s",gai_strerror(rv));
+                __redisSetError(c,REDIS_ERR_OTHER,buf);
+                goto error;
+            }
+
+            if (reuseaddr) {
+                n = 1;
+                if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char*) &n,
+                               sizeof(n)) < 0) {
+                    freeaddrinfo(bservinfo);
+                    goto error;
+                }
+            }
+
+            for (b = bservinfo; b != NULL; b = b->ai_next) {
+                if (bind(s,b->ai_addr,b->ai_addrlen) != -1) {
+                    bound = 1;
+                    break;
+                }
+            }
+            freeaddrinfo(bservinfo);
+            if (!bound) {
+                char buf[128];
+                snprintf(buf,sizeof(buf),"Can't bind socket: %s",strerror(errno));
+                __redisSetError(c,REDIS_ERR_OTHER,buf);
+                goto error;
+            }
+        }
+
+        /* For repeat connection */
+        hi_free(c->saddr);
+        c->saddr = hi_malloc(p->ai_addrlen);
+        if (c->saddr == NULL)
+            goto oom;
+
+        memcpy(c->saddr, p->ai_addr, p->ai_addrlen);
+        c->addrlen = p->ai_addrlen;
+
+        if (connect(s,p->ai_addr,p->ai_addrlen) == -1) {
+            if (errno == EHOSTUNREACH) {
+                redisNetClose(c);
+                continue;
+            } else if (errno == EINPROGRESS) {
+                if (blocking) {
+                    goto wait_for_ready;
+                }
+                /* This is ok.
+                 * Note that even when it's in blocking mode, we unset blocking
+                 * for `connect()`
+                 */
+            } else if (errno == EADDRNOTAVAIL && reuseaddr) {
+                if (++reuses >= REDIS_CONNECT_RETRIES) {
+                    goto error;
+                } else {
+                    redisNetClose(c);
+                    goto addrretry;
+                }
+            } else {
+                wait_for_ready:
+                if (redisContextWaitReady(c,timeout_msec) != REDIS_OK)
+                    goto error;
+                if (redisSetTcpNoDelay(c) != REDIS_OK)
+                    goto error;
+            }
+        }
+        if (blocking && redisSetBlocking(c,1) != REDIS_OK)
+            goto error;
+
+        c->flags |= REDIS_CONNECTED;
+        rv = REDIS_OK;
+        goto end;
+    }
+    if (p == NULL) {
+        char buf[128];
+        snprintf(buf,sizeof(buf),"Can't create socket: %s",strerror(errno));
+        __redisSetError(c,REDIS_ERR_OTHER,buf);
+        goto error;
+    }
+
+oom:
+    __redisSetError(c, REDIS_ERR_OOM, "Out of memory");
+error:
+    rv = REDIS_ERR;
+end:
+    if(servinfo) {
+        freeaddrinfo(servinfo);
+    }
+
+    return rv;  // Need to return REDIS_OK if alright
+}
+
+int redisContextConnectTcp(redisContext *c, const char *addr, int port,
+                           const struct timeval *timeout) {
+    return _redisContextConnectTcp(c, addr, port, timeout, NULL);
+}
+
+int redisContextConnectBindTcp(redisContext *c, const char *addr, int port,
+                               const struct timeval *timeout,
+                               const char *source_addr) {
+    return _redisContextConnectTcp(c, addr, port, timeout, source_addr);
+}
+
+int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) {
+#ifndef _WIN32
+    int blocking = (c->flags & REDIS_BLOCK);
+    struct sockaddr_un *sa;
+    long timeout_msec = -1;
+
+    if (redisCreateSocket(c,AF_UNIX) < 0)
+        return REDIS_ERR;
+    if (redisSetBlocking(c,0) != REDIS_OK)
+        return REDIS_ERR;
+
+    c->connection_type = REDIS_CONN_UNIX;
+    if (c->unix_sock.path != path) {
+        hi_free(c->unix_sock.path);
+
+        c->unix_sock.path = hi_strdup(path);
+        if (c->unix_sock.path == NULL)
+            goto oom;
+    }
+
+    if (timeout) {
+        if (redisContextUpdateConnectTimeout(c, timeout) == REDIS_ERR)
+            goto oom;
+    } else {
+        hi_free(c->connect_timeout);
+        c->connect_timeout = NULL;
+    }
+
+    if (redisContextTimeoutMsec(c,&timeout_msec) != REDIS_OK)
+        return REDIS_ERR;
+
+    /* Don't leak sockaddr if we're reconnecting */
+    if (c->saddr) hi_free(c->saddr);
+
+    sa = (struct sockaddr_un*)(c->saddr = hi_malloc(sizeof(struct sockaddr_un)));
+    if (sa == NULL)
+        goto oom;
+
+    c->addrlen = sizeof(struct sockaddr_un);
+    sa->sun_family = AF_UNIX;
+    strncpy(sa->sun_path, path, sizeof(sa->sun_path) - 1);
+    if (connect(c->fd, (struct sockaddr*)sa, sizeof(*sa)) == -1) {
+        if (errno == EINPROGRESS && !blocking) {
+            /* This is ok. */
+        } else {
+            if (redisContextWaitReady(c,timeout_msec) != REDIS_OK)
+                return REDIS_ERR;
+        }
+    }
+
+    /* Reset socket to be blocking after connect(2). */
+    if (blocking && redisSetBlocking(c,1) != REDIS_OK)
+        return REDIS_ERR;
+
+    c->flags |= REDIS_CONNECTED;
+    return REDIS_OK;
+#else
+    /* We currently do not support Unix sockets for Windows. */
+    /* TODO(m): https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/ */
+    errno = EPROTONOSUPPORT;
+    return REDIS_ERR;
+#endif /* _WIN32 */
+oom:
+    __redisSetError(c, REDIS_ERR_OOM, "Out of memory");
+    return REDIS_ERR;
+}
diff --git a/net.h b/net.h
new file mode 100644
index 000000000..9f43283a5
--- /dev/null
+++ b/net.h
@@ -0,0 +1,56 @@
+/* Extracted from anet.c to work properly with Hiredis error reporting.
+ *
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ * Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
+ *                     Jan-Erik Rediger <janerik at fnordig dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __NET_H
+#define __NET_H
+
+#include "hiredis.h"
+
+void redisNetClose(redisContext *c);
+ssize_t redisNetRead(redisContext *c, char *buf, size_t bufcap);
+ssize_t redisNetWrite(redisContext *c);
+
+int redisCheckSocketError(redisContext *c);
+int redisContextSetTimeout(redisContext *c, const struct timeval tv);
+int redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout);
+int redisContextConnectBindTcp(redisContext *c, const char *addr, int port,
+                               const struct timeval *timeout,
+                               const char *source_addr);
+int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout);
+int redisKeepAlive(redisContext *c, int interval);
+int redisCheckConnectDone(redisContext *c, int *completed);
+
+int redisSetTcpNoDelay(redisContext *c);
+
+#endif
diff --git a/read.c b/read.c
new file mode 100644
index 000000000..09524692b
--- /dev/null
+++ b/read.c
@@ -0,0 +1,739 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fmacros.h"
+#include <string.h>
+#include <stdlib.h>
+#ifndef _MSC_VER
+#include <unistd.h>
+#include <strings.h>
+#endif
+#include <assert.h>
+#include <errno.h>
+#include <ctype.h>
+#include <limits.h>
+#include <math.h>
+
+#include "alloc.h"
+#include "read.h"
+#include "sds.h"
+#include "win32.h"
+
+/* Initial size of our nested reply stack and how much we grow it when needd */
+#define REDIS_READER_STACK_SIZE 9
+
+static void __redisReaderSetError(redisReader *r, int type, const char *str) {
+    size_t len;
+
+    if (r->reply != NULL && r->fn && r->fn->freeObject) {
+        r->fn->freeObject(r->reply);
+        r->reply = NULL;
+    }
+
+    /* Clear input buffer on errors. */
+    sdsfree(r->buf);
+    r->buf = NULL;
+    r->pos = r->len = 0;
+
+    /* Reset task stack. */
+    r->ridx = -1;
+
+    /* Set error. */
+    r->err = type;
+    len = strlen(str);
+    len = len < (sizeof(r->errstr)-1) ? len : (sizeof(r->errstr)-1);
+    memcpy(r->errstr,str,len);
+    r->errstr[len] = '\0';
+}
+
+static size_t chrtos(char *buf, size_t size, char byte) {
+    size_t len = 0;
+
+    switch(byte) {
+    case '\\':
+    case '"':
+        len = snprintf(buf,size,"\"\\%c\"",byte);
+        break;
+    case '\n': len = snprintf(buf,size,"\"\\n\""); break;
+    case '\r': len = snprintf(buf,size,"\"\\r\""); break;
+    case '\t': len = snprintf(buf,size,"\"\\t\""); break;
+    case '\a': len = snprintf(buf,size,"\"\\a\""); break;
+    case '\b': len = snprintf(buf,size,"\"\\b\""); break;
+    default:
+        if (isprint(byte))
+            len = snprintf(buf,size,"\"%c\"",byte);
+        else
+            len = snprintf(buf,size,"\"\\x%02x\"",(unsigned char)byte);
+        break;
+    }
+
+    return len;
+}
+
+static void __redisReaderSetErrorProtocolByte(redisReader *r, char byte) {
+    char cbuf[8], sbuf[128];
+
+    chrtos(cbuf,sizeof(cbuf),byte);
+    snprintf(sbuf,sizeof(sbuf),
+        "Protocol error, got %s as reply type byte", cbuf);
+    __redisReaderSetError(r,REDIS_ERR_PROTOCOL,sbuf);
+}
+
+static void __redisReaderSetErrorOOM(redisReader *r) {
+    __redisReaderSetError(r,REDIS_ERR_OOM,"Out of memory");
+}
+
+static char *readBytes(redisReader *r, unsigned int bytes) {
+    char *p;
+    if (r->len-r->pos >= bytes) {
+        p = r->buf+r->pos;
+        r->pos += bytes;
+        return p;
+    }
+    return NULL;
+}
+
+/* Find pointer to \r\n. */
+static char *seekNewline(char *s, size_t len) {
+    int pos = 0;
+    int _len = len-1;
+
+    /* Position should be < len-1 because the character at "pos" should be
+     * followed by a \n. Note that strchr cannot be used because it doesn't
+     * allow to search a limited length and the buffer that is being searched
+     * might not have a trailing NULL character. */
+    while (pos < _len) {
+        while(pos < _len && s[pos] != '\r') pos++;
+        if (pos==_len) {
+            /* Not found. */
+            return NULL;
+        } else {
+            if (s[pos+1] == '\n') {
+                /* Found. */
+                return s+pos;
+            } else {
+                /* Continue searching. */
+                pos++;
+            }
+        }
+    }
+    return NULL;
+}
+
+/* Convert a string into a long long. Returns REDIS_OK if the string could be
+ * parsed into a (non-overflowing) long long, REDIS_ERR otherwise. The value
+ * will be set to the parsed value when appropriate.
+ *
+ * Note that this function demands that the string strictly represents
+ * a long long: no spaces or other characters before or after the string
+ * representing the number are accepted, nor zeroes at the start if not
+ * for the string "0" representing the zero number.
+ *
+ * Because of its strictness, it is safe to use this function to check if
+ * you can convert a string into a long long, and obtain back the string
+ * from the number without any loss in the string representation. */
+static int string2ll(const char *s, size_t slen, long long *value) {
+    const char *p = s;
+    size_t plen = 0;
+    int negative = 0;
+    unsigned long long v;
+
+    if (plen == slen)
+        return REDIS_ERR;
+
+    /* Special case: first and only digit is 0. */
+    if (slen == 1 && p[0] == '0') {
+        if (value != NULL) *value = 0;
+        return REDIS_OK;
+    }
+
+    if (p[0] == '-') {
+        negative = 1;
+        p++; plen++;
+
+        /* Abort on only a negative sign. */
+        if (plen == slen)
+            return REDIS_ERR;
+    }
+
+    /* First digit should be 1-9, otherwise the string should just be 0. */
+    if (p[0] >= '1' && p[0] <= '9') {
+        v = p[0]-'0';
+        p++; plen++;
+    } else if (p[0] == '0' && slen == 1) {
+        *value = 0;
+        return REDIS_OK;
+    } else {
+        return REDIS_ERR;
+    }
+
+    while (plen < slen && p[0] >= '0' && p[0] <= '9') {
+        if (v > (ULLONG_MAX / 10)) /* Overflow. */
+            return REDIS_ERR;
+        v *= 10;
+
+        if (v > (ULLONG_MAX - (p[0]-'0'))) /* Overflow. */
+            return REDIS_ERR;
+        v += p[0]-'0';
+
+        p++; plen++;
+    }
+
+    /* Return if not all bytes were used. */
+    if (plen < slen)
+        return REDIS_ERR;
+
+    if (negative) {
+        if (v > ((unsigned long long)(-(LLONG_MIN+1))+1)) /* Overflow. */
+            return REDIS_ERR;
+        if (value != NULL) *value = -v;
+    } else {
+        if (v > LLONG_MAX) /* Overflow. */
+            return REDIS_ERR;
+        if (value != NULL) *value = v;
+    }
+    return REDIS_OK;
+}
+
+static char *readLine(redisReader *r, int *_len) {
+    char *p, *s;
+    int len;
+
+    p = r->buf+r->pos;
+    s = seekNewline(p,(r->len-r->pos));
+    if (s != NULL) {
+        len = s-(r->buf+r->pos);
+        r->pos += len+2; /* skip \r\n */
+        if (_len) *_len = len;
+        return p;
+    }
+    return NULL;
+}
+
+static void moveToNextTask(redisReader *r) {
+    redisReadTask *cur, *prv;
+    while (r->ridx >= 0) {
+        /* Return a.s.a.p. when the stack is now empty. */
+        if (r->ridx == 0) {
+            r->ridx--;
+            return;
+        }
+
+        cur = r->task[r->ridx];
+        prv = r->task[r->ridx-1];
+        assert(prv->type == REDIS_REPLY_ARRAY ||
+               prv->type == REDIS_REPLY_MAP ||
+               prv->type == REDIS_REPLY_SET ||
+               prv->type == REDIS_REPLY_PUSH);
+        if (cur->idx == prv->elements-1) {
+            r->ridx--;
+        } else {
+            /* Reset the type because the next item can be anything */
+            assert(cur->idx < prv->elements);
+            cur->type = -1;
+            cur->elements = -1;
+            cur->idx++;
+            return;
+        }
+    }
+}
+
+static int processLineItem(redisReader *r) {
+    redisReadTask *cur = r->task[r->ridx];
+    void *obj;
+    char *p;
+    int len;
+
+    if ((p = readLine(r,&len)) != NULL) {
+        if (cur->type == REDIS_REPLY_INTEGER) {
+            if (r->fn && r->fn->createInteger) {
+                long long v;
+                if (string2ll(p, len, &v) == REDIS_ERR) {
+                    __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
+                            "Bad integer value");
+                    return REDIS_ERR;
+                }
+                obj = r->fn->createInteger(cur,v);
+            } else {
+                obj = (void*)REDIS_REPLY_INTEGER;
+            }
+        } else if (cur->type == REDIS_REPLY_DOUBLE) {
+            if (r->fn && r->fn->createDouble) {
+                char buf[326], *eptr;
+                double d;
+
+                if ((size_t)len >= sizeof(buf)) {
+                    __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
+                            "Double value is too large");
+                    return REDIS_ERR;
+                }
+
+                memcpy(buf,p,len);
+                buf[len] = '\0';
+
+                if (strcasecmp(buf,",inf") == 0) {
+                    d = INFINITY; /* Positive infinite. */
+                } else if (strcasecmp(buf,",-inf") == 0) {
+                    d = -INFINITY; /* Negative infinite. */
+                } else {
+                    d = strtod((char*)buf,&eptr);
+                    if (buf[0] == '\0' || eptr[0] != '\0' || isnan(d)) {
+                        __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
+                                "Bad double value");
+                        return REDIS_ERR;
+                    }
+                }
+                obj = r->fn->createDouble(cur,d,buf,len);
+            } else {
+                obj = (void*)REDIS_REPLY_DOUBLE;
+            }
+        } else if (cur->type == REDIS_REPLY_NIL) {
+            if (r->fn && r->fn->createNil)
+                obj = r->fn->createNil(cur);
+            else
+                obj = (void*)REDIS_REPLY_NIL;
+        } else if (cur->type == REDIS_REPLY_BOOL) {
+            int bval = p[0] == 't' || p[0] == 'T';
+            if (r->fn && r->fn->createBool)
+                obj = r->fn->createBool(cur,bval);
+            else
+                obj = (void*)REDIS_REPLY_BOOL;
+        } else {
+            /* Type will be error or status. */
+            if (r->fn && r->fn->createString)
+                obj = r->fn->createString(cur,p,len);
+            else
+                obj = (void*)(size_t)(cur->type);
+        }
+
+        if (obj == NULL) {
+            __redisReaderSetErrorOOM(r);
+            return REDIS_ERR;
+        }
+
+        /* Set reply if this is the root object. */
+        if (r->ridx == 0) r->reply = obj;
+        moveToNextTask(r);
+        return REDIS_OK;
+    }
+
+    return REDIS_ERR;
+}
+
+static int processBulkItem(redisReader *r) {
+    redisReadTask *cur = r->task[r->ridx];
+    void *obj = NULL;
+    char *p, *s;
+    long long len;
+    unsigned long bytelen;
+    int success = 0;
+
+    p = r->buf+r->pos;
+    s = seekNewline(p,r->len-r->pos);
+    if (s != NULL) {
+        p = r->buf+r->pos;
+        bytelen = s-(r->buf+r->pos)+2; /* include \r\n */
+
+        if (string2ll(p, bytelen - 2, &len) == REDIS_ERR) {
+            __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
+                    "Bad bulk string length");
+            return REDIS_ERR;
+        }
+
+        if (len < -1 || (LLONG_MAX > SIZE_MAX && len > (long long)SIZE_MAX)) {
+            __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
+                    "Bulk string length out of range");
+            return REDIS_ERR;
+        }
+
+        if (len == -1) {
+            /* The nil object can always be created. */
+            if (r->fn && r->fn->createNil)
+                obj = r->fn->createNil(cur);
+            else
+                obj = (void*)REDIS_REPLY_NIL;
+            success = 1;
+        } else {
+            /* Only continue when the buffer contains the entire bulk item. */
+            bytelen += len+2; /* include \r\n */
+            if (r->pos+bytelen <= r->len) {
+                if ((cur->type == REDIS_REPLY_VERB && len < 4) ||
+                    (cur->type == REDIS_REPLY_VERB && s[5] != ':'))
+                {
+                    __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
+                            "Verbatim string 4 bytes of content type are "
+                            "missing or incorrectly encoded.");
+                    return REDIS_ERR;
+                }
+                if (r->fn && r->fn->createString)
+                    obj = r->fn->createString(cur,s+2,len);
+                else
+                    obj = (void*)(long)cur->type;
+                success = 1;
+            }
+        }
+
+        /* Proceed when obj was created. */
+        if (success) {
+            if (obj == NULL) {
+                __redisReaderSetErrorOOM(r);
+                return REDIS_ERR;
+            }
+
+            r->pos += bytelen;
+
+            /* Set reply if this is the root object. */
+            if (r->ridx == 0) r->reply = obj;
+            moveToNextTask(r);
+            return REDIS_OK;
+        }
+    }
+
+    return REDIS_ERR;
+}
+
+static int redisReaderGrow(redisReader *r) {
+    redisReadTask **aux;
+    int newlen;
+
+    /* Grow our stack size */
+    newlen = r->tasks + REDIS_READER_STACK_SIZE;
+    aux = hi_realloc(r->task, sizeof(*r->task) * newlen);
+    if (aux == NULL)
+        goto oom;
+
+    r->task = aux;
+
+    /* Allocate new tasks */
+    for (; r->tasks < newlen; r->tasks++) {
+        r->task[r->tasks] = hi_calloc(1, sizeof(**r->task));
+        if (r->task[r->tasks] == NULL)
+            goto oom;
+    }
+
+    return REDIS_OK;
+oom:
+    __redisReaderSetErrorOOM(r);
+    return REDIS_ERR;
+}
+
+/* Process the array, map and set types. */
+static int processAggregateItem(redisReader *r) {
+    redisReadTask *cur = r->task[r->ridx];
+    void *obj;
+    char *p;
+    long long elements;
+    int root = 0, len;
+
+    /* Set error for nested multi bulks with depth > 7 */
+    if (r->ridx == r->tasks - 1) {
+        if (redisReaderGrow(r) == REDIS_ERR)
+            return REDIS_ERR;
+    }
+
+    if ((p = readLine(r,&len)) != NULL) {
+        if (string2ll(p, len, &elements) == REDIS_ERR) {
+            __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
+                    "Bad multi-bulk length");
+            return REDIS_ERR;
+        }
+
+        root = (r->ridx == 0);
+
+        if (elements < -1 || (LLONG_MAX > SIZE_MAX && elements > SIZE_MAX) ||
+            (r->maxelements > 0 && elements > r->maxelements))
+        {
+            __redisReaderSetError(r,REDIS_ERR_PROTOCOL,
+                    "Multi-bulk length out of range");
+            return REDIS_ERR;
+        }
+
+        if (elements == -1) {
+            if (r->fn && r->fn->createNil)
+                obj = r->fn->createNil(cur);
+            else
+                obj = (void*)REDIS_REPLY_NIL;
+
+            if (obj == NULL) {
+                __redisReaderSetErrorOOM(r);
+                return REDIS_ERR;
+            }
+
+            moveToNextTask(r);
+        } else {
+            if (cur->type == REDIS_REPLY_MAP) elements *= 2;
+
+            if (r->fn && r->fn->createArray)
+                obj = r->fn->createArray(cur,elements);
+            else
+                obj = (void*)(long)cur->type;
+
+            if (obj == NULL) {
+                __redisReaderSetErrorOOM(r);
+                return REDIS_ERR;
+            }
+
+            /* Modify task stack when there are more than 0 elements. */
+            if (elements > 0) {
+                cur->elements = elements;
+                cur->obj = obj;
+                r->ridx++;
+                r->task[r->ridx]->type = -1;
+                r->task[r->ridx]->elements = -1;
+                r->task[r->ridx]->idx = 0;
+                r->task[r->ridx]->obj = NULL;
+                r->task[r->ridx]->parent = cur;
+                r->task[r->ridx]->privdata = r->privdata;
+            } else {
+                moveToNextTask(r);
+            }
+        }
+
+        /* Set reply if this is the root object. */
+        if (root) r->reply = obj;
+        return REDIS_OK;
+    }
+
+    return REDIS_ERR;
+}
+
+static int processItem(redisReader *r) {
+    redisReadTask *cur = r->task[r->ridx];
+    char *p;
+
+    /* check if we need to read type */
+    if (cur->type < 0) {
+        if ((p = readBytes(r,1)) != NULL) {
+            switch (p[0]) {
+            case '-':
+                cur->type = REDIS_REPLY_ERROR;
+                break;
+            case '+':
+                cur->type = REDIS_REPLY_STATUS;
+                break;
+            case ':':
+                cur->type = REDIS_REPLY_INTEGER;
+                break;
+            case ',':
+                cur->type = REDIS_REPLY_DOUBLE;
+                break;
+            case '_':
+                cur->type = REDIS_REPLY_NIL;
+                break;
+            case '$':
+                cur->type = REDIS_REPLY_STRING;
+                break;
+            case '*':
+                cur->type = REDIS_REPLY_ARRAY;
+                break;
+            case '%':
+                cur->type = REDIS_REPLY_MAP;
+                break;
+            case '~':
+                cur->type = REDIS_REPLY_SET;
+                break;
+            case '#':
+                cur->type = REDIS_REPLY_BOOL;
+                break;
+            case '=':
+                cur->type = REDIS_REPLY_VERB;
+                break;
+            case '>':
+                cur->type = REDIS_REPLY_PUSH;
+                break;
+            default:
+                __redisReaderSetErrorProtocolByte(r,*p);
+                return REDIS_ERR;
+            }
+        } else {
+            /* could not consume 1 byte */
+            return REDIS_ERR;
+        }
+    }
+
+    /* process typed item */
+    switch(cur->type) {
+    case REDIS_REPLY_ERROR:
+    case REDIS_REPLY_STATUS:
+    case REDIS_REPLY_INTEGER:
+    case REDIS_REPLY_DOUBLE:
+    case REDIS_REPLY_NIL:
+    case REDIS_REPLY_BOOL:
+        return processLineItem(r);
+    case REDIS_REPLY_STRING:
+    case REDIS_REPLY_VERB:
+        return processBulkItem(r);
+    case REDIS_REPLY_ARRAY:
+    case REDIS_REPLY_MAP:
+    case REDIS_REPLY_SET:
+    case REDIS_REPLY_PUSH:
+        return processAggregateItem(r);
+    default:
+        assert(NULL);
+        return REDIS_ERR; /* Avoid warning. */
+    }
+}
+
+redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) {
+    redisReader *r;
+
+    r = hi_calloc(1,sizeof(redisReader));
+    if (r == NULL)
+        return NULL;
+
+    r->buf = sdsempty();
+    if (r->buf == NULL)
+        goto oom;
+
+    r->task = hi_calloc(REDIS_READER_STACK_SIZE, sizeof(*r->task));
+    if (r->task == NULL)
+        goto oom;
+
+    for (; r->tasks < REDIS_READER_STACK_SIZE; r->tasks++) {
+        r->task[r->tasks] = hi_calloc(1, sizeof(**r->task));
+        if (r->task[r->tasks] == NULL)
+            goto oom;
+    }
+
+    r->fn = fn;
+    r->maxbuf = REDIS_READER_MAX_BUF;
+    r->maxelements = REDIS_READER_MAX_ARRAY_ELEMENTS;
+    r->ridx = -1;
+
+    return r;
+oom:
+    redisReaderFree(r);
+    return NULL;
+}
+
+void redisReaderFree(redisReader *r) {
+    if (r == NULL)
+        return;
+
+    if (r->reply != NULL && r->fn && r->fn->freeObject)
+        r->fn->freeObject(r->reply);
+
+    if (r->task) {
+        /* We know r->task[i] is allocated if i < r->tasks */
+        for (int i = 0; i < r->tasks; i++) {
+            hi_free(r->task[i]);
+        }
+
+        hi_free(r->task);
+    }
+
+    sdsfree(r->buf);
+    hi_free(r);
+}
+
+int redisReaderFeed(redisReader *r, const char *buf, size_t len) {
+    sds newbuf;
+
+    /* Return early when this reader is in an erroneous state. */
+    if (r->err)
+        return REDIS_ERR;
+
+    /* Copy the provided buffer. */
+    if (buf != NULL && len >= 1) {
+        /* Destroy internal buffer when it is empty and is quite large. */
+        if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) {
+            sdsfree(r->buf);
+            r->buf = sdsempty();
+            if (r->buf == 0) goto oom;
+
+            r->pos = 0;
+        }
+
+        newbuf = sdscatlen(r->buf,buf,len);
+        if (newbuf == NULL) goto oom;
+
+        r->buf = newbuf;
+        r->len = sdslen(r->buf);
+    }
+
+    return REDIS_OK;
+oom:
+    __redisReaderSetErrorOOM(r);
+    return REDIS_ERR;
+}
+
+int redisReaderGetReply(redisReader *r, void **reply) {
+    /* Default target pointer to NULL. */
+    if (reply != NULL)
+        *reply = NULL;
+
+    /* Return early when this reader is in an erroneous state. */
+    if (r->err)
+        return REDIS_ERR;
+
+    /* When the buffer is empty, there will never be a reply. */
+    if (r->len == 0)
+        return REDIS_OK;
+
+    /* Set first item to process when the stack is empty. */
+    if (r->ridx == -1) {
+        r->task[0]->type = -1;
+        r->task[0]->elements = -1;
+        r->task[0]->idx = -1;
+        r->task[0]->obj = NULL;
+        r->task[0]->parent = NULL;
+        r->task[0]->privdata = r->privdata;
+        r->ridx = 0;
+    }
+
+    /* Process items in reply. */
+    while (r->ridx >= 0)
+        if (processItem(r) != REDIS_OK)
+            break;
+
+    /* Return ASAP when an error occurred. */
+    if (r->err)
+        return REDIS_ERR;
+
+    /* Discard part of the buffer when we've consumed at least 1k, to avoid
+     * doing unnecessary calls to memmove() in sds.c. */
+    if (r->pos >= 1024) {
+        if (sdsrange(r->buf,r->pos,-1) < 0) return REDIS_ERR;
+        r->pos = 0;
+        r->len = sdslen(r->buf);
+    }
+
+    /* Emit a reply when there is one. */
+    if (r->ridx == -1) {
+        if (reply != NULL) {
+            *reply = r->reply;
+        } else if (r->reply != NULL && r->fn && r->fn->freeObject) {
+            r->fn->freeObject(r->reply);
+        }
+        r->reply = NULL;
+    }
+    return REDIS_OK;
+}
diff --git a/read.h b/read.h
new file mode 100644
index 000000000..2d74d77a5
--- /dev/null
+++ b/read.h
@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#ifndef __HIREDIS_READ_H
+#define __HIREDIS_READ_H
+#include <stdio.h> /* for size_t */
+
+#define REDIS_ERR -1
+#define REDIS_OK 0
+
+/* When an error occurs, the err flag in a context is set to hold the type of
+ * error that occurred. REDIS_ERR_IO means there was an I/O error and you
+ * should use the "errno" variable to find out what is wrong.
+ * For other values, the "errstr" field will hold a description. */
+#define REDIS_ERR_IO 1 /* Error in read or write */
+#define REDIS_ERR_EOF 3 /* End of file */
+#define REDIS_ERR_PROTOCOL 4 /* Protocol error */
+#define REDIS_ERR_OOM 5 /* Out of memory */
+#define REDIS_ERR_TIMEOUT 6 /* Timed out */
+#define REDIS_ERR_OTHER 2 /* Everything else... */
+
+#define REDIS_REPLY_STRING 1
+#define REDIS_REPLY_ARRAY 2
+#define REDIS_REPLY_INTEGER 3
+#define REDIS_REPLY_NIL 4
+#define REDIS_REPLY_STATUS 5
+#define REDIS_REPLY_ERROR 6
+#define REDIS_REPLY_DOUBLE 7
+#define REDIS_REPLY_BOOL 8
+#define REDIS_REPLY_MAP 9
+#define REDIS_REPLY_SET 10
+#define REDIS_REPLY_ATTR 11
+#define REDIS_REPLY_PUSH 12
+#define REDIS_REPLY_BIGNUM 13
+#define REDIS_REPLY_VERB 14
+
+/* Default max unused reader buffer. */
+#define REDIS_READER_MAX_BUF (1024*16)
+
+/* Default multi-bulk element limit */
+#define REDIS_READER_MAX_ARRAY_ELEMENTS ((1LL<<32) - 1)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct redisReadTask {
+    int type;
+    long long elements; /* number of elements in multibulk container */
+    int idx; /* index in parent (array) object */
+    void *obj; /* holds user-generated value for a read task */
+    struct redisReadTask *parent; /* parent task */
+    void *privdata; /* user-settable arbitrary field */
+} redisReadTask;
+
+typedef struct redisReplyObjectFunctions {
+    void *(*createString)(const redisReadTask*, char*, size_t);
+    void *(*createArray)(const redisReadTask*, size_t);
+    void *(*createInteger)(const redisReadTask*, long long);
+    void *(*createDouble)(const redisReadTask*, double, char*, size_t);
+    void *(*createNil)(const redisReadTask*);
+    void *(*createBool)(const redisReadTask*, int);
+    void (*freeObject)(void*);
+} redisReplyObjectFunctions;
+
+typedef struct redisReader {
+    int err; /* Error flags, 0 when there is no error */
+    char errstr[128]; /* String representation of error when applicable */
+
+    char *buf; /* Read buffer */
+    size_t pos; /* Buffer cursor */
+    size_t len; /* Buffer length */
+    size_t maxbuf; /* Max length of unused buffer */
+    long long maxelements; /* Max multi-bulk elements */
+
+    redisReadTask **task;
+    int tasks;
+
+    int ridx; /* Index of current read task */
+    void *reply; /* Temporary reply pointer */
+
+    redisReplyObjectFunctions *fn;
+    void *privdata;
+} redisReader;
+
+/* Public API for the protocol parser. */
+redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn);
+void redisReaderFree(redisReader *r);
+int redisReaderFeed(redisReader *r, const char *buf, size_t len);
+int redisReaderGetReply(redisReader *r, void **reply);
+
+#define redisReaderSetPrivdata(_r, _p) (int)(((redisReader*)(_r))->privdata = (_p))
+#define redisReaderGetObject(_r) (((redisReader*)(_r))->reply)
+#define redisReaderGetError(_r) (((redisReader*)(_r))->errstr)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/sds.c b/sds.c
new file mode 100644
index 000000000..49d2096b7
--- /dev/null
+++ b/sds.c
@@ -0,0 +1,1289 @@
+/* SDSLib 2.0 -- A C dynamic strings library
+ *
+ * Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2015, Oran Agra
+ * Copyright (c) 2015, Redis Labs, Inc
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "fmacros.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <assert.h>
+#include <limits.h>
+#include "sds.h"
+#include "sdsalloc.h"
+
+static inline int sdsHdrSize(char type) {
+    switch(type&SDS_TYPE_MASK) {
+        case SDS_TYPE_5:
+            return sizeof(struct sdshdr5);
+        case SDS_TYPE_8:
+            return sizeof(struct sdshdr8);
+        case SDS_TYPE_16:
+            return sizeof(struct sdshdr16);
+        case SDS_TYPE_32:
+            return sizeof(struct sdshdr32);
+        case SDS_TYPE_64:
+            return sizeof(struct sdshdr64);
+    }
+    return 0;
+}
+
+static inline char sdsReqType(size_t string_size) {
+    if (string_size < 32)
+        return SDS_TYPE_5;
+    if (string_size < 0xff)
+        return SDS_TYPE_8;
+    if (string_size < 0xffff)
+        return SDS_TYPE_16;
+    if (string_size < 0xffffffff)
+        return SDS_TYPE_32;
+    return SDS_TYPE_64;
+}
+
+/* Create a new sds string with the content specified by the 'init' pointer
+ * and 'initlen'.
+ * If NULL is used for 'init' the string is initialized with zero bytes.
+ *
+ * The string is always null-termined (all the sds strings are, always) so
+ * even if you create an sds string with:
+ *
+ * mystring = sdsnewlen("abc",3);
+ *
+ * You can print the string with printf() as there is an implicit \0 at the
+ * end of the string. However the string is binary safe and can contain
+ * \0 characters in the middle, as the length is stored in the sds header. */
+sds sdsnewlen(const void *init, size_t initlen) {
+    void *sh;
+    sds s;
+    char type = sdsReqType(initlen);
+    /* Empty strings are usually created in order to append. Use type 8
+     * since type 5 is not good at this. */
+    if (type == SDS_TYPE_5 && initlen == 0) type = SDS_TYPE_8;
+    int hdrlen = sdsHdrSize(type);
+    unsigned char *fp; /* flags pointer. */
+
+    sh = s_malloc(hdrlen+initlen+1);
+    if (sh == NULL) return NULL;
+    if (!init)
+        memset(sh, 0, hdrlen+initlen+1);
+    s = (char*)sh+hdrlen;
+    fp = ((unsigned char*)s)-1;
+    switch(type) {
+        case SDS_TYPE_5: {
+            *fp = type | (initlen << SDS_TYPE_BITS);
+            break;
+        }
+        case SDS_TYPE_8: {
+            SDS_HDR_VAR(8,s);
+            sh->len = initlen;
+            sh->alloc = initlen;
+            *fp = type;
+            break;
+        }
+        case SDS_TYPE_16: {
+            SDS_HDR_VAR(16,s);
+            sh->len = initlen;
+            sh->alloc = initlen;
+            *fp = type;
+            break;
+        }
+        case SDS_TYPE_32: {
+            SDS_HDR_VAR(32,s);
+            sh->len = initlen;
+            sh->alloc = initlen;
+            *fp = type;
+            break;
+        }
+        case SDS_TYPE_64: {
+            SDS_HDR_VAR(64,s);
+            sh->len = initlen;
+            sh->alloc = initlen;
+            *fp = type;
+            break;
+        }
+    }
+    if (initlen && init)
+        memcpy(s, init, initlen);
+    s[initlen] = '\0';
+    return s;
+}
+
+/* Create an empty (zero length) sds string. Even in this case the string
+ * always has an implicit null term. */
+sds sdsempty(void) {
+    return sdsnewlen("",0);
+}
+
+/* Create a new sds string starting from a null terminated C string. */
+sds sdsnew(const char *init) {
+    size_t initlen = (init == NULL) ? 0 : strlen(init);
+    return sdsnewlen(init, initlen);
+}
+
+/* Duplicate an sds string. */
+sds sdsdup(const sds s) {
+    return sdsnewlen(s, sdslen(s));
+}
+
+/* Free an sds string. No operation is performed if 's' is NULL. */
+void sdsfree(sds s) {
+    if (s == NULL) return;
+    s_free((char*)s-sdsHdrSize(s[-1]));
+}
+
+/* Set the sds string length to the length as obtained with strlen(), so
+ * considering as content only up to the first null term character.
+ *
+ * This function is useful when the sds string is hacked manually in some
+ * way, like in the following example:
+ *
+ * s = sdsnew("foobar");
+ * s[2] = '\0';
+ * sdsupdatelen(s);
+ * printf("%d\n", sdslen(s));
+ *
+ * The output will be "2", but if we comment out the call to sdsupdatelen()
+ * the output will be "6" as the string was modified but the logical length
+ * remains 6 bytes. */
+void sdsupdatelen(sds s) {
+    int reallen = strlen(s);
+    sdssetlen(s, reallen);
+}
+
+/* Modify an sds string in-place to make it empty (zero length).
+ * However all the existing buffer is not discarded but set as free space
+ * so that next append operations will not require allocations up to the
+ * number of bytes previously available. */
+void sdsclear(sds s) {
+    sdssetlen(s, 0);
+    s[0] = '\0';
+}
+
+/* Enlarge the free space at the end of the sds string so that the caller
+ * is sure that after calling this function can overwrite up to addlen
+ * bytes after the end of the string, plus one more byte for nul term.
+ *
+ * Note: this does not change the *length* of the sds string as returned
+ * by sdslen(), but only the free buffer space we have. */
+sds sdsMakeRoomFor(sds s, size_t addlen) {
+    void *sh, *newsh;
+    size_t avail = sdsavail(s);
+    size_t len, newlen;
+    char type, oldtype = s[-1] & SDS_TYPE_MASK;
+    int hdrlen;
+
+    /* Return ASAP if there is enough space left. */
+    if (avail >= addlen) return s;
+
+    len = sdslen(s);
+    sh = (char*)s-sdsHdrSize(oldtype);
+    newlen = (len+addlen);
+    if (newlen < SDS_MAX_PREALLOC)
+        newlen *= 2;
+    else
+        newlen += SDS_MAX_PREALLOC;
+
+    type = sdsReqType(newlen);
+
+    /* Don't use type 5: the user is appending to the string and type 5 is
+     * not able to remember empty space, so sdsMakeRoomFor() must be called
+     * at every appending operation. */
+    if (type == SDS_TYPE_5) type = SDS_TYPE_8;
+
+    hdrlen = sdsHdrSize(type);
+    if (oldtype==type) {
+        newsh = s_realloc(sh, hdrlen+newlen+1);
+        if (newsh == NULL) return NULL;
+        s = (char*)newsh+hdrlen;
+    } else {
+        /* Since the header size changes, need to move the string forward,
+         * and can't use realloc */
+        newsh = s_malloc(hdrlen+newlen+1);
+        if (newsh == NULL) return NULL;
+        memcpy((char*)newsh+hdrlen, s, len+1);
+        s_free(sh);
+        s = (char*)newsh+hdrlen;
+        s[-1] = type;
+        sdssetlen(s, len);
+    }
+    sdssetalloc(s, newlen);
+    return s;
+}
+
+/* Reallocate the sds string so that it has no free space at the end. The
+ * contained string remains not altered, but next concatenation operations
+ * will require a reallocation.
+ *
+ * After the call, the passed sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call. */
+sds sdsRemoveFreeSpace(sds s) {
+    void *sh, *newsh;
+    char type, oldtype = s[-1] & SDS_TYPE_MASK;
+    int hdrlen;
+    size_t len = sdslen(s);
+    sh = (char*)s-sdsHdrSize(oldtype);
+
+    type = sdsReqType(len);
+    hdrlen = sdsHdrSize(type);
+    if (oldtype==type) {
+        newsh = s_realloc(sh, hdrlen+len+1);
+        if (newsh == NULL) return NULL;
+        s = (char*)newsh+hdrlen;
+    } else {
+        newsh = s_malloc(hdrlen+len+1);
+        if (newsh == NULL) return NULL;
+        memcpy((char*)newsh+hdrlen, s, len+1);
+        s_free(sh);
+        s = (char*)newsh+hdrlen;
+        s[-1] = type;
+        sdssetlen(s, len);
+    }
+    sdssetalloc(s, len);
+    return s;
+}
+
+/* Return the total size of the allocation of the specifed sds string,
+ * including:
+ * 1) The sds header before the pointer.
+ * 2) The string.
+ * 3) The free buffer at the end if any.
+ * 4) The implicit null term.
+ */
+size_t sdsAllocSize(sds s) {
+    size_t alloc = sdsalloc(s);
+    return sdsHdrSize(s[-1])+alloc+1;
+}
+
+/* Return the pointer of the actual SDS allocation (normally SDS strings
+ * are referenced by the start of the string buffer). */
+void *sdsAllocPtr(sds s) {
+    return (void*) (s-sdsHdrSize(s[-1]));
+}
+
+/* Increment the sds length and decrements the left free space at the
+ * end of the string according to 'incr'. Also set the null term
+ * in the new end of the string.
+ *
+ * This function is used in order to fix the string length after the
+ * user calls sdsMakeRoomFor(), writes something after the end of
+ * the current string, and finally needs to set the new length.
+ *
+ * Note: it is possible to use a negative increment in order to
+ * right-trim the string.
+ *
+ * Usage example:
+ *
+ * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the
+ * following schema, to cat bytes coming from the kernel to the end of an
+ * sds string without copying into an intermediate buffer:
+ *
+ * oldlen = sdslen(s);
+ * s = sdsMakeRoomFor(s, BUFFER_SIZE);
+ * nread = read(fd, s+oldlen, BUFFER_SIZE);
+ * ... check for nread <= 0 and handle it ...
+ * sdsIncrLen(s, nread);
+ */
+void sdsIncrLen(sds s, int incr) {
+    unsigned char flags = s[-1];
+    size_t len;
+    switch(flags&SDS_TYPE_MASK) {
+        case SDS_TYPE_5: {
+            unsigned char *fp = ((unsigned char*)s)-1;
+            unsigned char oldlen = SDS_TYPE_5_LEN(flags);
+            assert((incr > 0 && oldlen+incr < 32) || (incr < 0 && oldlen >= (unsigned int)(-incr)));
+            *fp = SDS_TYPE_5 | ((oldlen+incr) << SDS_TYPE_BITS);
+            len = oldlen+incr;
+            break;
+        }
+        case SDS_TYPE_8: {
+            SDS_HDR_VAR(8,s);
+            assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr)));
+            len = (sh->len += incr);
+            break;
+        }
+        case SDS_TYPE_16: {
+            SDS_HDR_VAR(16,s);
+            assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr)));
+            len = (sh->len += incr);
+            break;
+        }
+        case SDS_TYPE_32: {
+            SDS_HDR_VAR(32,s);
+            assert((incr >= 0 && sh->alloc-sh->len >= (unsigned int)incr) || (incr < 0 && sh->len >= (unsigned int)(-incr)));
+            len = (sh->len += incr);
+            break;
+        }
+        case SDS_TYPE_64: {
+            SDS_HDR_VAR(64,s);
+            assert((incr >= 0 && sh->alloc-sh->len >= (uint64_t)incr) || (incr < 0 && sh->len >= (uint64_t)(-incr)));
+            len = (sh->len += incr);
+            break;
+        }
+        default: len = 0; /* Just to avoid compilation warnings. */
+    }
+    s[len] = '\0';
+}
+
+/* Grow the sds to have the specified length. Bytes that were not part of
+ * the original length of the sds will be set to zero.
+ *
+ * if the specified length is smaller than the current length, no operation
+ * is performed. */
+sds sdsgrowzero(sds s, size_t len) {
+    size_t curlen = sdslen(s);
+
+    if (len <= curlen) return s;
+    s = sdsMakeRoomFor(s,len-curlen);
+    if (s == NULL) return NULL;
+
+    /* Make sure added region doesn't contain garbage */
+    memset(s+curlen,0,(len-curlen+1)); /* also set trailing \0 byte */
+    sdssetlen(s, len);
+    return s;
+}
+
+/* Append the specified binary-safe string pointed by 't' of 'len' bytes to the
+ * end of the specified sds string 's'.
+ *
+ * After the call, the passed sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call. */
+sds sdscatlen(sds s, const void *t, size_t len) {
+    size_t curlen = sdslen(s);
+
+    s = sdsMakeRoomFor(s,len);
+    if (s == NULL) return NULL;
+    memcpy(s+curlen, t, len);
+    sdssetlen(s, curlen+len);
+    s[curlen+len] = '\0';
+    return s;
+}
+
+/* Append the specified null termianted C string to the sds string 's'.
+ *
+ * After the call, the passed sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call. */
+sds sdscat(sds s, const char *t) {
+    return sdscatlen(s, t, strlen(t));
+}
+
+/* Append the specified sds 't' to the existing sds 's'.
+ *
+ * After the call, the modified sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call. */
+sds sdscatsds(sds s, const sds t) {
+    return sdscatlen(s, t, sdslen(t));
+}
+
+/* Destructively modify the sds string 's' to hold the specified binary
+ * safe string pointed by 't' of length 'len' bytes. */
+sds sdscpylen(sds s, const char *t, size_t len) {
+    if (sdsalloc(s) < len) {
+        s = sdsMakeRoomFor(s,len-sdslen(s));
+        if (s == NULL) return NULL;
+    }
+    memcpy(s, t, len);
+    s[len] = '\0';
+    sdssetlen(s, len);
+    return s;
+}
+
+/* Like sdscpylen() but 't' must be a null-termined string so that the length
+ * of the string is obtained with strlen(). */
+sds sdscpy(sds s, const char *t) {
+    return sdscpylen(s, t, strlen(t));
+}
+
+/* Helper for sdscatlonglong() doing the actual number -> string
+ * conversion. 's' must point to a string with room for at least
+ * SDS_LLSTR_SIZE bytes.
+ *
+ * The function returns the length of the null-terminated string
+ * representation stored at 's'. */
+#define SDS_LLSTR_SIZE 21
+int sdsll2str(char *s, long long value) {
+    char *p, aux;
+    unsigned long long v;
+    size_t l;
+
+    /* Generate the string representation, this method produces
+     * an reversed string. */
+    v = (value < 0) ? -value : value;
+    p = s;
+    do {
+        *p++ = '0'+(v%10);
+        v /= 10;
+    } while(v);
+    if (value < 0) *p++ = '-';
+
+    /* Compute length and add null term. */
+    l = p-s;
+    *p = '\0';
+
+    /* Reverse the string. */
+    p--;
+    while(s < p) {
+        aux = *s;
+        *s = *p;
+        *p = aux;
+        s++;
+        p--;
+    }
+    return l;
+}
+
+/* Identical sdsll2str(), but for unsigned long long type. */
+int sdsull2str(char *s, unsigned long long v) {
+    char *p, aux;
+    size_t l;
+
+    /* Generate the string representation, this method produces
+     * an reversed string. */
+    p = s;
+    do {
+        *p++ = '0'+(v%10);
+        v /= 10;
+    } while(v);
+
+    /* Compute length and add null term. */
+    l = p-s;
+    *p = '\0';
+
+    /* Reverse the string. */
+    p--;
+    while(s < p) {
+        aux = *s;
+        *s = *p;
+        *p = aux;
+        s++;
+        p--;
+    }
+    return l;
+}
+
+/* Create an sds string from a long long value. It is much faster than:
+ *
+ * sdscatprintf(sdsempty(),"%lld\n", value);
+ */
+sds sdsfromlonglong(long long value) {
+    char buf[SDS_LLSTR_SIZE];
+    int len = sdsll2str(buf,value);
+
+    return sdsnewlen(buf,len);
+}
+
+/* Like sdscatprintf() but gets va_list instead of being variadic. */
+sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
+    va_list cpy;
+    char staticbuf[1024], *buf = staticbuf, *t;
+    size_t buflen = strlen(fmt)*2;
+
+    /* We try to start using a static buffer for speed.
+     * If not possible we revert to heap allocation. */
+    if (buflen > sizeof(staticbuf)) {
+        buf = s_malloc(buflen);
+        if (buf == NULL) return NULL;
+    } else {
+        buflen = sizeof(staticbuf);
+    }
+
+    /* Try with buffers two times bigger every time we fail to
+     * fit the string in the current buffer size. */
+    while(1) {
+        buf[buflen-2] = '\0';
+        va_copy(cpy,ap);
+        vsnprintf(buf, buflen, fmt, cpy);
+        va_end(cpy);
+        if (buf[buflen-2] != '\0') {
+            if (buf != staticbuf) s_free(buf);
+            buflen *= 2;
+            buf = s_malloc(buflen);
+            if (buf == NULL) return NULL;
+            continue;
+        }
+        break;
+    }
+
+    /* Finally concat the obtained string to the SDS string and return it. */
+    t = sdscat(s, buf);
+    if (buf != staticbuf) s_free(buf);
+    return t;
+}
+
+/* Append to the sds string 's' a string obtained using printf-alike format
+ * specifier.
+ *
+ * After the call, the modified sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call.
+ *
+ * Example:
+ *
+ * s = sdsnew("Sum is: ");
+ * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b).
+ *
+ * Often you need to create a string from scratch with the printf-alike
+ * format. When this is the need, just use sdsempty() as the target string:
+ *
+ * s = sdscatprintf(sdsempty(), "... your format ...", args);
+ */
+sds sdscatprintf(sds s, const char *fmt, ...) {
+    va_list ap;
+    char *t;
+    va_start(ap, fmt);
+    t = sdscatvprintf(s,fmt,ap);
+    va_end(ap);
+    return t;
+}
+
+/* This function is similar to sdscatprintf, but much faster as it does
+ * not rely on sprintf() family functions implemented by the libc that
+ * are often very slow. Moreover directly handling the sds string as
+ * new data is concatenated provides a performance improvement.
+ *
+ * However this function only handles an incompatible subset of printf-alike
+ * format specifiers:
+ *
+ * %s - C String
+ * %S - SDS string
+ * %i - signed int
+ * %I - 64 bit signed integer (long long, int64_t)
+ * %u - unsigned int
+ * %U - 64 bit unsigned integer (unsigned long long, uint64_t)
+ * %% - Verbatim "%" character.
+ */
+sds sdscatfmt(sds s, char const *fmt, ...) {
+    const char *f = fmt;
+    int i;
+    va_list ap;
+
+    va_start(ap,fmt);
+    i = sdslen(s); /* Position of the next byte to write to dest str. */
+    while(*f) {
+        char next, *str;
+        size_t l;
+        long long num;
+        unsigned long long unum;
+
+        /* Make sure there is always space for at least 1 char. */
+        if (sdsavail(s)==0) {
+            s = sdsMakeRoomFor(s,1);
+            if (s == NULL) goto fmt_error;
+        }
+
+        switch(*f) {
+        case '%':
+            next = *(f+1);
+            f++;
+            switch(next) {
+            case 's':
+            case 'S':
+                str = va_arg(ap,char*);
+                l = (next == 's') ? strlen(str) : sdslen(str);
+                if (sdsavail(s) < l) {
+                    s = sdsMakeRoomFor(s,l);
+                    if (s == NULL) goto fmt_error;
+                }
+                memcpy(s+i,str,l);
+                sdsinclen(s,l);
+                i += l;
+                break;
+            case 'i':
+            case 'I':
+                if (next == 'i')
+                    num = va_arg(ap,int);
+                else
+                    num = va_arg(ap,long long);
+                {
+                    char buf[SDS_LLSTR_SIZE];
+                    l = sdsll2str(buf,num);
+                    if (sdsavail(s) < l) {
+                        s = sdsMakeRoomFor(s,l);
+                        if (s == NULL) goto fmt_error;
+                    }
+                    memcpy(s+i,buf,l);
+                    sdsinclen(s,l);
+                    i += l;
+                }
+                break;
+            case 'u':
+            case 'U':
+                if (next == 'u')
+                    unum = va_arg(ap,unsigned int);
+                else
+                    unum = va_arg(ap,unsigned long long);
+                {
+                    char buf[SDS_LLSTR_SIZE];
+                    l = sdsull2str(buf,unum);
+                    if (sdsavail(s) < l) {
+                        s = sdsMakeRoomFor(s,l);
+                        if (s == NULL) goto fmt_error;
+                    }
+                    memcpy(s+i,buf,l);
+                    sdsinclen(s,l);
+                    i += l;
+                }
+                break;
+            default: /* Handle %% and generally %<unknown>. */
+                s[i++] = next;
+                sdsinclen(s,1);
+                break;
+            }
+            break;
+        default:
+            s[i++] = *f;
+            sdsinclen(s,1);
+            break;
+        }
+        f++;
+    }
+    va_end(ap);
+
+    /* Add null-term */
+    s[i] = '\0';
+    return s;
+
+fmt_error:
+    va_end(ap);
+    return NULL;
+}
+
+/* Remove the part of the string from left and from right composed just of
+ * contiguous characters found in 'cset', that is a null terminted C string.
+ *
+ * After the call, the modified sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call.
+ *
+ * Example:
+ *
+ * s = sdsnew("AA...AA.a.aa.aHelloWorld     :::");
+ * s = sdstrim(s,"Aa. :");
+ * printf("%s\n", s);
+ *
+ * Output will be just "Hello World".
+ */
+sds sdstrim(sds s, const char *cset) {
+    char *start, *end, *sp, *ep;
+    size_t len;
+
+    sp = start = s;
+    ep = end = s+sdslen(s)-1;
+    while(sp <= end && strchr(cset, *sp)) sp++;
+    while(ep > sp && strchr(cset, *ep)) ep--;
+    len = (sp > ep) ? 0 : ((ep-sp)+1);
+    if (s != sp) memmove(s, sp, len);
+    s[len] = '\0';
+    sdssetlen(s,len);
+    return s;
+}
+
+/* Turn the string into a smaller (or equal) string containing only the
+ * substring specified by the 'start' and 'end' indexes.
+ *
+ * start and end can be negative, where -1 means the last character of the
+ * string, -2 the penultimate character, and so forth.
+ *
+ * The interval is inclusive, so the start and end characters will be part
+ * of the resulting string.
+ *
+ * The string is modified in-place.
+ *
+ * Return value:
+ * -1 (error) if sdslen(s) is larger than maximum positive ssize_t value.
+ *  0 on success.
+ *
+ * Example:
+ *
+ * s = sdsnew("Hello World");
+ * sdsrange(s,1,-1); => "ello World"
+ */
+int sdsrange(sds s, ssize_t start, ssize_t end) {
+    size_t newlen, len = sdslen(s);
+    if (len > SSIZE_MAX) return -1;
+
+    if (len == 0) return 0;
+    if (start < 0) {
+        start = len+start;
+        if (start < 0) start = 0;
+    }
+    if (end < 0) {
+        end = len+end;
+        if (end < 0) end = 0;
+    }
+    newlen = (start > end) ? 0 : (end-start)+1;
+    if (newlen != 0) {
+        if (start >= (ssize_t)len) {
+            newlen = 0;
+        } else if (end >= (ssize_t)len) {
+            end = len-1;
+            newlen = (start > end) ? 0 : (end-start)+1;
+        }
+    } else {
+        start = 0;
+    }
+    if (start && newlen) memmove(s, s+start, newlen);
+    s[newlen] = 0;
+    sdssetlen(s,newlen);
+    return 0;
+}
+
+/* Apply tolower() to every character of the sds string 's'. */
+void sdstolower(sds s) {
+    int len = sdslen(s), j;
+
+    for (j = 0; j < len; j++) s[j] = tolower(s[j]);
+}
+
+/* Apply toupper() to every character of the sds string 's'. */
+void sdstoupper(sds s) {
+    int len = sdslen(s), j;
+
+    for (j = 0; j < len; j++) s[j] = toupper(s[j]);
+}
+
+/* Compare two sds strings s1 and s2 with memcmp().
+ *
+ * Return value:
+ *
+ *     positive if s1 > s2.
+ *     negative if s1 < s2.
+ *     0 if s1 and s2 are exactly the same binary string.
+ *
+ * If two strings share exactly the same prefix, but one of the two has
+ * additional characters, the longer string is considered to be greater than
+ * the smaller one. */
+int sdscmp(const sds s1, const sds s2) {
+    size_t l1, l2, minlen;
+    int cmp;
+
+    l1 = sdslen(s1);
+    l2 = sdslen(s2);
+    minlen = (l1 < l2) ? l1 : l2;
+    cmp = memcmp(s1,s2,minlen);
+    if (cmp == 0) return l1-l2;
+    return cmp;
+}
+
+/* Split 's' with separator in 'sep'. An array
+ * of sds strings is returned. *count will be set
+ * by reference to the number of tokens returned.
+ *
+ * On out of memory, zero length string, zero length
+ * separator, NULL is returned.
+ *
+ * Note that 'sep' is able to split a string using
+ * a multi-character separator. For example
+ * sdssplit("foo_-_bar","_-_"); will return two
+ * elements "foo" and "bar".
+ *
+ * This version of the function is binary-safe but
+ * requires length arguments. sdssplit() is just the
+ * same function but for zero-terminated strings.
+ */
+sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) {
+    int elements = 0, slots = 5, start = 0, j;
+    sds *tokens;
+
+    if (seplen < 1 || len < 0) return NULL;
+
+    tokens = s_malloc(sizeof(sds)*slots);
+    if (tokens == NULL) return NULL;
+
+    if (len == 0) {
+        *count = 0;
+        return tokens;
+    }
+    for (j = 0; j < (len-(seplen-1)); j++) {
+        /* make sure there is room for the next element and the final one */
+        if (slots < elements+2) {
+            sds *newtokens;
+
+            slots *= 2;
+            newtokens = s_realloc(tokens,sizeof(sds)*slots);
+            if (newtokens == NULL) goto cleanup;
+            tokens = newtokens;
+        }
+        /* search the separator */
+        if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) {
+            tokens[elements] = sdsnewlen(s+start,j-start);
+            if (tokens[elements] == NULL) goto cleanup;
+            elements++;
+            start = j+seplen;
+            j = j+seplen-1; /* skip the separator */
+        }
+    }
+    /* Add the final element. We are sure there is room in the tokens array. */
+    tokens[elements] = sdsnewlen(s+start,len-start);
+    if (tokens[elements] == NULL) goto cleanup;
+    elements++;
+    *count = elements;
+    return tokens;
+
+cleanup:
+    {
+        int i;
+        for (i = 0; i < elements; i++) sdsfree(tokens[i]);
+        s_free(tokens);
+        *count = 0;
+        return NULL;
+    }
+}
+
+/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */
+void sdsfreesplitres(sds *tokens, int count) {
+    if (!tokens) return;
+    while(count--)
+        sdsfree(tokens[count]);
+    s_free(tokens);
+}
+
+/* Append to the sds string "s" an escaped string representation where
+ * all the non-printable characters (tested with isprint()) are turned into
+ * escapes in the form "\n\r\a...." or "\x<hex-number>".
+ *
+ * After the call, the modified sds string is no longer valid and all the
+ * references must be substituted with the new pointer returned by the call. */
+sds sdscatrepr(sds s, const char *p, size_t len) {
+    s = sdscatlen(s,"\"",1);
+    while(len--) {
+        switch(*p) {
+        case '\\':
+        case '"':
+            s = sdscatprintf(s,"\\%c",*p);
+            break;
+        case '\n': s = sdscatlen(s,"\\n",2); break;
+        case '\r': s = sdscatlen(s,"\\r",2); break;
+        case '\t': s = sdscatlen(s,"\\t",2); break;
+        case '\a': s = sdscatlen(s,"\\a",2); break;
+        case '\b': s = sdscatlen(s,"\\b",2); break;
+        default:
+            if (isprint(*p))
+                s = sdscatprintf(s,"%c",*p);
+            else
+                s = sdscatprintf(s,"\\x%02x",(unsigned char)*p);
+            break;
+        }
+        p++;
+    }
+    return sdscatlen(s,"\"",1);
+}
+
+/* Helper function for sdssplitargs() that converts a hex digit into an
+ * integer from 0 to 15 */
+int hex_digit_to_int(char c) {
+    switch(c) {
+    case '0': return 0;
+    case '1': return 1;
+    case '2': return 2;
+    case '3': return 3;
+    case '4': return 4;
+    case '5': return 5;
+    case '6': return 6;
+    case '7': return 7;
+    case '8': return 8;
+    case '9': return 9;
+    case 'a': case 'A': return 10;
+    case 'b': case 'B': return 11;
+    case 'c': case 'C': return 12;
+    case 'd': case 'D': return 13;
+    case 'e': case 'E': return 14;
+    case 'f': case 'F': return 15;
+    default: return 0;
+    }
+}
+
+/* Split a line into arguments, where every argument can be in the
+ * following programming-language REPL-alike form:
+ *
+ * foo bar "newline are supported\n" and "\xff\x00otherstuff"
+ *
+ * The number of arguments is stored into *argc, and an array
+ * of sds is returned.
+ *
+ * The caller should free the resulting array of sds strings with
+ * sdsfreesplitres().
+ *
+ * Note that sdscatrepr() is able to convert back a string into
+ * a quoted string in the same format sdssplitargs() is able to parse.
+ *
+ * The function returns the allocated tokens on success, even when the
+ * input string is empty, or NULL if the input contains unbalanced
+ * quotes or closed quotes followed by non space characters
+ * as in: "foo"bar or "foo'
+ */
+sds *sdssplitargs(const char *line, int *argc) {
+    const char *p = line;
+    char *current = NULL;
+    char **vector = NULL;
+
+    *argc = 0;
+    while(1) {
+        /* skip blanks */
+        while(*p && isspace(*p)) p++;
+        if (*p) {
+            /* get a token */
+            int inq=0;  /* set to 1 if we are in "quotes" */
+            int insq=0; /* set to 1 if we are in 'single quotes' */
+            int done=0;
+
+            if (current == NULL) current = sdsempty();
+            while(!done) {
+                if (inq) {
+                    if (*p == '\\' && *(p+1) == 'x' &&
+                                             isxdigit(*(p+2)) &&
+                                             isxdigit(*(p+3)))
+                    {
+                        unsigned char byte;
+
+                        byte = (hex_digit_to_int(*(p+2))*16)+
+                                hex_digit_to_int(*(p+3));
+                        current = sdscatlen(current,(char*)&byte,1);
+                        p += 3;
+                    } else if (*p == '\\' && *(p+1)) {
+                        char c;
+
+                        p++;
+                        switch(*p) {
+                        case 'n': c = '\n'; break;
+                        case 'r': c = '\r'; break;
+                        case 't': c = '\t'; break;
+                        case 'b': c = '\b'; break;
+                        case 'a': c = '\a'; break;
+                        default: c = *p; break;
+                        }
+                        current = sdscatlen(current,&c,1);
+                    } else if (*p == '"') {
+                        /* closing quote must be followed by a space or
+                         * nothing at all. */
+                        if (*(p+1) && !isspace(*(p+1))) goto err;
+                        done=1;
+                    } else if (!*p) {
+                        /* unterminated quotes */
+                        goto err;
+                    } else {
+                        current = sdscatlen(current,p,1);
+                    }
+                } else if (insq) {
+                    if (*p == '\\' && *(p+1) == '\'') {
+                        p++;
+                        current = sdscatlen(current,"'",1);
+                    } else if (*p == '\'') {
+                        /* closing quote must be followed by a space or
+                         * nothing at all. */
+                        if (*(p+1) && !isspace(*(p+1))) goto err;
+                        done=1;
+                    } else if (!*p) {
+                        /* unterminated quotes */
+                        goto err;
+                    } else {
+                        current = sdscatlen(current,p,1);
+                    }
+                } else {
+                    switch(*p) {
+                    case ' ':
+                    case '\n':
+                    case '\r':
+                    case '\t':
+                    case '\0':
+                        done=1;
+                        break;
+                    case '"':
+                        inq=1;
+                        break;
+                    case '\'':
+                        insq=1;
+                        break;
+                    default:
+                        current = sdscatlen(current,p,1);
+                        break;
+                    }
+                }
+                if (*p) p++;
+            }
+            /* add the token to the vector */
+            {
+                char **new_vector = s_realloc(vector,((*argc)+1)*sizeof(char*));
+                if (new_vector == NULL) {
+                    s_free(vector);
+                    return NULL;
+                }
+
+                vector = new_vector;
+                vector[*argc] = current;
+                (*argc)++;
+                current = NULL;
+            }
+        } else {
+            /* Even on empty input string return something not NULL. */
+            if (vector == NULL) vector = s_malloc(sizeof(void*));
+            return vector;
+        }
+    }
+
+err:
+    while((*argc)--)
+        sdsfree(vector[*argc]);
+    s_free(vector);
+    if (current) sdsfree(current);
+    *argc = 0;
+    return NULL;
+}
+
+/* Modify the string substituting all the occurrences of the set of
+ * characters specified in the 'from' string to the corresponding character
+ * in the 'to' array.
+ *
+ * For instance: sdsmapchars(mystring, "ho", "01", 2)
+ * will have the effect of turning the string "hello" into "0ell1".
+ *
+ * The function returns the sds string pointer, that is always the same
+ * as the input pointer since no resize is needed. */
+sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) {
+    size_t j, i, l = sdslen(s);
+
+    for (j = 0; j < l; j++) {
+        for (i = 0; i < setlen; i++) {
+            if (s[j] == from[i]) {
+                s[j] = to[i];
+                break;
+            }
+        }
+    }
+    return s;
+}
+
+/* Join an array of C strings using the specified separator (also a C string).
+ * Returns the result as an sds string. */
+sds sdsjoin(char **argv, int argc, char *sep) {
+    sds join = sdsempty();
+    int j;
+
+    for (j = 0; j < argc; j++) {
+        join = sdscat(join, argv[j]);
+        if (j != argc-1) join = sdscat(join,sep);
+    }
+    return join;
+}
+
+/* Like sdsjoin, but joins an array of SDS strings. */
+sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) {
+    sds join = sdsempty();
+    int j;
+
+    for (j = 0; j < argc; j++) {
+        join = sdscatsds(join, argv[j]);
+        if (j != argc-1) join = sdscatlen(join,sep,seplen);
+    }
+    return join;
+}
+
+/* Wrappers to the allocators used by SDS. Note that SDS will actually
+ * just use the macros defined into sdsalloc.h in order to avoid to pay
+ * the overhead of function calls. Here we define these wrappers only for
+ * the programs SDS is linked to, if they want to touch the SDS internals
+ * even if they use a different allocator. */
+void *sds_malloc(size_t size) { return s_malloc(size); }
+void *sds_realloc(void *ptr, size_t size) { return s_realloc(ptr,size); }
+void sds_free(void *ptr) { s_free(ptr); }
+
+#if defined(SDS_TEST_MAIN)
+#include <stdio.h>
+#include "testhelp.h"
+#include "limits.h"
+
+#define UNUSED(x) (void)(x)
+int sdsTest(void) {
+    {
+        sds x = sdsnew("foo"), y;
+
+        test_cond("Create a string and obtain the length",
+            sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0)
+
+        sdsfree(x);
+        x = sdsnewlen("foo",2);
+        test_cond("Create a string with specified length",
+            sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0)
+
+        x = sdscat(x,"bar");
+        test_cond("Strings concatenation",
+            sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0);
+
+        x = sdscpy(x,"a");
+        test_cond("sdscpy() against an originally longer string",
+            sdslen(x) == 1 && memcmp(x,"a\0",2) == 0)
+
+        x = sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk");
+        test_cond("sdscpy() against an originally shorter string",
+            sdslen(x) == 33 &&
+            memcmp(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk\0",33) == 0)
+
+        sdsfree(x);
+        x = sdscatprintf(sdsempty(),"%d",123);
+        test_cond("sdscatprintf() seems working in the base case",
+            sdslen(x) == 3 && memcmp(x,"123\0",4) == 0)
+
+        sdsfree(x);
+        x = sdsnew("--");
+        x = sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX);
+        test_cond("sdscatfmt() seems working in the base case",
+            sdslen(x) == 60 &&
+            memcmp(x,"--Hello Hi! World -9223372036854775808,"
+                     "9223372036854775807--",60) == 0)
+        printf("[%s]\n",x);
+
+        sdsfree(x);
+        x = sdsnew("--");
+        x = sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX);
+        test_cond("sdscatfmt() seems working with unsigned numbers",
+            sdslen(x) == 35 &&
+            memcmp(x,"--4294967295,18446744073709551615--",35) == 0)
+
+        sdsfree(x);
+        x = sdsnew(" x ");
+        sdstrim(x," x");
+        test_cond("sdstrim() works when all chars match",
+            sdslen(x) == 0)
+
+        sdsfree(x);
+        x = sdsnew(" x ");
+        sdstrim(x," ");
+        test_cond("sdstrim() works when a single char remains",
+            sdslen(x) == 1 && x[0] == 'x')
+
+        sdsfree(x);
+        x = sdsnew("xxciaoyyy");
+        sdstrim(x,"xy");
+        test_cond("sdstrim() correctly trims characters",
+            sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0)
+
+        y = sdsdup(x);
+        sdsrange(y,1,1);
+        test_cond("sdsrange(...,1,1)",
+            sdslen(y) == 1 && memcmp(y,"i\0",2) == 0)
+
+        sdsfree(y);
+        y = sdsdup(x);
+        sdsrange(y,1,-1);
+        test_cond("sdsrange(...,1,-1)",
+            sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
+
+        sdsfree(y);
+        y = sdsdup(x);
+        sdsrange(y,-2,-1);
+        test_cond("sdsrange(...,-2,-1)",
+            sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0)
+
+        sdsfree(y);
+        y = sdsdup(x);
+        sdsrange(y,2,1);
+        test_cond("sdsrange(...,2,1)",
+            sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
+
+        sdsfree(y);
+        y = sdsdup(x);
+        sdsrange(y,1,100);
+        test_cond("sdsrange(...,1,100)",
+            sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
+
+        sdsfree(y);
+        y = sdsdup(x);
+        sdsrange(y,100,100);
+        test_cond("sdsrange(...,100,100)",
+            sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
+
+        sdsfree(y);
+        sdsfree(x);
+        x = sdsnew("foo");
+        y = sdsnew("foa");
+        test_cond("sdscmp(foo,foa)", sdscmp(x,y) > 0)
+
+        sdsfree(y);
+        sdsfree(x);
+        x = sdsnew("bar");
+        y = sdsnew("bar");
+        test_cond("sdscmp(bar,bar)", sdscmp(x,y) == 0)
+
+        sdsfree(y);
+        sdsfree(x);
+        x = sdsnew("aar");
+        y = sdsnew("bar");
+        test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0)
+
+        sdsfree(y);
+        sdsfree(x);
+        x = sdsnewlen("\a\n\0foo\r",7);
+        y = sdscatrepr(sdsempty(),x,sdslen(x));
+        test_cond("sdscatrepr(...data...)",
+            memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0)
+
+        {
+            unsigned int oldfree;
+            char *p;
+            int step = 10, j, i;
+
+            sdsfree(x);
+            sdsfree(y);
+            x = sdsnew("0");
+            test_cond("sdsnew() free/len buffers", sdslen(x) == 1 && sdsavail(x) == 0);
+
+            /* Run the test a few times in order to hit the first two
+             * SDS header types. */
+            for (i = 0; i < 10; i++) {
+                int oldlen = sdslen(x);
+                x = sdsMakeRoomFor(x,step);
+                int type = x[-1]&SDS_TYPE_MASK;
+
+                test_cond("sdsMakeRoomFor() len", sdslen(x) == oldlen);
+                if (type != SDS_TYPE_5) {
+                    test_cond("sdsMakeRoomFor() free", sdsavail(x) >= step);
+                    oldfree = sdsavail(x);
+                }
+                p = x+oldlen;
+                for (j = 0; j < step; j++) {
+                    p[j] = 'A'+j;
+                }
+                sdsIncrLen(x,step);
+            }
+            test_cond("sdsMakeRoomFor() content",
+                memcmp("0ABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJ",x,101) == 0);
+            test_cond("sdsMakeRoomFor() final length",sdslen(x)==101);
+
+            sdsfree(x);
+        }
+    }
+    test_report()
+    return 0;
+}
+#endif
+
+#ifdef SDS_TEST_MAIN
+int main(void) {
+    return sdsTest();
+}
+#endif
diff --git a/sds.h b/sds.h
new file mode 100644
index 000000000..eda8833b5
--- /dev/null
+++ b/sds.h
@@ -0,0 +1,278 @@
+/* SDSLib 2.0 -- A C dynamic strings library
+ *
+ * Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2015, Oran Agra
+ * Copyright (c) 2015, Redis Labs, Inc
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __SDS_H
+#define __SDS_H
+
+#define SDS_MAX_PREALLOC (1024*1024)
+#ifdef _MSC_VER
+#define __attribute__(x)
+typedef long long ssize_t;
+#define SSIZE_MAX (LLONG_MAX >> 1)
+#endif
+
+#include <sys/types.h>
+#include <stdarg.h>
+#include <stdint.h>
+
+typedef char *sds;
+
+/* Note: sdshdr5 is never used, we just access the flags byte directly.
+ * However is here to document the layout of type 5 SDS strings. */
+struct __attribute__ ((__packed__)) sdshdr5 {
+    unsigned char flags; /* 3 lsb of type, and 5 msb of string length */
+    char buf[];
+};
+struct __attribute__ ((__packed__)) sdshdr8 {
+    uint8_t len; /* used */
+    uint8_t alloc; /* excluding the header and null terminator */
+    unsigned char flags; /* 3 lsb of type, 5 unused bits */
+    char buf[];
+};
+struct __attribute__ ((__packed__)) sdshdr16 {
+    uint16_t len; /* used */
+    uint16_t alloc; /* excluding the header and null terminator */
+    unsigned char flags; /* 3 lsb of type, 5 unused bits */
+    char buf[];
+};
+struct __attribute__ ((__packed__)) sdshdr32 {
+    uint32_t len; /* used */
+    uint32_t alloc; /* excluding the header and null terminator */
+    unsigned char flags; /* 3 lsb of type, 5 unused bits */
+    char buf[];
+};
+struct __attribute__ ((__packed__)) sdshdr64 {
+    uint64_t len; /* used */
+    uint64_t alloc; /* excluding the header and null terminator */
+    unsigned char flags; /* 3 lsb of type, 5 unused bits */
+    char buf[];
+};
+
+#define SDS_TYPE_5  0
+#define SDS_TYPE_8  1
+#define SDS_TYPE_16 2
+#define SDS_TYPE_32 3
+#define SDS_TYPE_64 4
+#define SDS_TYPE_MASK 7
+#define SDS_TYPE_BITS 3
+#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T)));
+#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))))
+#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS)
+
+static inline size_t sdslen(const sds s) {
+    unsigned char flags = s[-1];
+    switch(flags&SDS_TYPE_MASK) {
+        case SDS_TYPE_5:
+            return SDS_TYPE_5_LEN(flags);
+        case SDS_TYPE_8:
+            return SDS_HDR(8,s)->len;
+        case SDS_TYPE_16:
+            return SDS_HDR(16,s)->len;
+        case SDS_TYPE_32:
+            return SDS_HDR(32,s)->len;
+        case SDS_TYPE_64:
+            return SDS_HDR(64,s)->len;
+    }
+    return 0;
+}
+
+static inline size_t sdsavail(const sds s) {
+    unsigned char flags = s[-1];
+    switch(flags&SDS_TYPE_MASK) {
+        case SDS_TYPE_5: {
+            return 0;
+        }
+        case SDS_TYPE_8: {
+            SDS_HDR_VAR(8,s);
+            return sh->alloc - sh->len;
+        }
+        case SDS_TYPE_16: {
+            SDS_HDR_VAR(16,s);
+            return sh->alloc - sh->len;
+        }
+        case SDS_TYPE_32: {
+            SDS_HDR_VAR(32,s);
+            return sh->alloc - sh->len;
+        }
+        case SDS_TYPE_64: {
+            SDS_HDR_VAR(64,s);
+            return sh->alloc - sh->len;
+        }
+    }
+    return 0;
+}
+
+static inline void sdssetlen(sds s, size_t newlen) {
+    unsigned char flags = s[-1];
+    switch(flags&SDS_TYPE_MASK) {
+        case SDS_TYPE_5:
+            {
+                unsigned char *fp = ((unsigned char*)s)-1;
+                *fp = (unsigned char)(SDS_TYPE_5 | (newlen << SDS_TYPE_BITS));
+            }
+            break;
+        case SDS_TYPE_8:
+            SDS_HDR(8,s)->len = (uint8_t)newlen;
+            break;
+        case SDS_TYPE_16:
+            SDS_HDR(16,s)->len = (uint16_t)newlen;
+            break;
+        case SDS_TYPE_32:
+            SDS_HDR(32,s)->len = (uint32_t)newlen;
+            break;
+        case SDS_TYPE_64:
+            SDS_HDR(64,s)->len = (uint64_t)newlen;
+            break;
+    }
+}
+
+static inline void sdsinclen(sds s, size_t inc) {
+    unsigned char flags = s[-1];
+    switch(flags&SDS_TYPE_MASK) {
+        case SDS_TYPE_5:
+            {
+                unsigned char *fp = ((unsigned char*)s)-1;
+                unsigned char newlen = SDS_TYPE_5_LEN(flags)+(unsigned char)inc;
+                *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
+            }
+            break;
+        case SDS_TYPE_8:
+            SDS_HDR(8,s)->len += (uint8_t)inc;
+            break;
+        case SDS_TYPE_16:
+            SDS_HDR(16,s)->len += (uint16_t)inc;
+            break;
+        case SDS_TYPE_32:
+            SDS_HDR(32,s)->len += (uint32_t)inc;
+            break;
+        case SDS_TYPE_64:
+            SDS_HDR(64,s)->len += (uint64_t)inc;
+            break;
+    }
+}
+
+/* sdsalloc() = sdsavail() + sdslen() */
+static inline size_t sdsalloc(const sds s) {
+    unsigned char flags = s[-1];
+    switch(flags&SDS_TYPE_MASK) {
+        case SDS_TYPE_5:
+            return SDS_TYPE_5_LEN(flags);
+        case SDS_TYPE_8:
+            return SDS_HDR(8,s)->alloc;
+        case SDS_TYPE_16:
+            return SDS_HDR(16,s)->alloc;
+        case SDS_TYPE_32:
+            return SDS_HDR(32,s)->alloc;
+        case SDS_TYPE_64:
+            return SDS_HDR(64,s)->alloc;
+    }
+    return 0;
+}
+
+static inline void sdssetalloc(sds s, size_t newlen) {
+    unsigned char flags = s[-1];
+    switch(flags&SDS_TYPE_MASK) {
+        case SDS_TYPE_5:
+            /* Nothing to do, this type has no total allocation info. */
+            break;
+        case SDS_TYPE_8:
+            SDS_HDR(8,s)->alloc = (uint8_t)newlen;
+            break;
+        case SDS_TYPE_16:
+            SDS_HDR(16,s)->alloc = (uint16_t)newlen;
+            break;
+        case SDS_TYPE_32:
+            SDS_HDR(32,s)->alloc = (uint32_t)newlen;
+            break;
+        case SDS_TYPE_64:
+            SDS_HDR(64,s)->alloc = (uint64_t)newlen;
+            break;
+    }
+}
+
+sds sdsnewlen(const void *init, size_t initlen);
+sds sdsnew(const char *init);
+sds sdsempty(void);
+sds sdsdup(const sds s);
+void sdsfree(sds s);
+sds sdsgrowzero(sds s, size_t len);
+sds sdscatlen(sds s, const void *t, size_t len);
+sds sdscat(sds s, const char *t);
+sds sdscatsds(sds s, const sds t);
+sds sdscpylen(sds s, const char *t, size_t len);
+sds sdscpy(sds s, const char *t);
+
+sds sdscatvprintf(sds s, const char *fmt, va_list ap);
+#ifdef __GNUC__
+sds sdscatprintf(sds s, const char *fmt, ...)
+    __attribute__((format(printf, 2, 3)));
+#else
+sds sdscatprintf(sds s, const char *fmt, ...);
+#endif
+
+sds sdscatfmt(sds s, char const *fmt, ...);
+sds sdstrim(sds s, const char *cset);
+int sdsrange(sds s, ssize_t start, ssize_t end);
+void sdsupdatelen(sds s);
+void sdsclear(sds s);
+int sdscmp(const sds s1, const sds s2);
+sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count);
+void sdsfreesplitres(sds *tokens, int count);
+void sdstolower(sds s);
+void sdstoupper(sds s);
+sds sdsfromlonglong(long long value);
+sds sdscatrepr(sds s, const char *p, size_t len);
+sds *sdssplitargs(const char *line, int *argc);
+sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen);
+sds sdsjoin(char **argv, int argc, char *sep);
+sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen);
+
+/* Low level functions exposed to the user API */
+sds sdsMakeRoomFor(sds s, size_t addlen);
+void sdsIncrLen(sds s, int incr);
+sds sdsRemoveFreeSpace(sds s);
+size_t sdsAllocSize(sds s);
+void *sdsAllocPtr(sds s);
+
+/* Export the allocator used by SDS to the program using SDS.
+ * Sometimes the program SDS is linked to, may use a different set of
+ * allocators, but may want to allocate or free things that SDS will
+ * respectively free or allocate. */
+void *sds_malloc(size_t size);
+void *sds_realloc(void *ptr, size_t size);
+void sds_free(void *ptr);
+
+#ifdef REDIS_TEST
+int sdsTest(int argc, char *argv[]);
+#endif
+
+#endif
diff --git a/sdsalloc.h b/sdsalloc.h
new file mode 100644
index 000000000..5538dd94c
--- /dev/null
+++ b/sdsalloc.h
@@ -0,0 +1,44 @@
+/* SDSLib 2.0 -- A C dynamic strings library
+ *
+ * Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2015, Oran Agra
+ * Copyright (c) 2015, Redis Labs, Inc
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* SDS allocator selection.
+ *
+ * This file is used in order to change the SDS allocator at compile time.
+ * Just define the following defines to what you want to use. Also add
+ * the include of your alternate allocator if needed (not needed in order
+ * to use the default libc allocator). */
+
+#include "alloc.h"
+
+#define s_malloc hi_malloc
+#define s_realloc hi_realloc
+#define s_free hi_free
diff --git a/sockcompat.c b/sockcompat.c
new file mode 100644
index 000000000..f99d14b05
--- /dev/null
+++ b/sockcompat.c
@@ -0,0 +1,248 @@
+/*
+ * Copyright (c) 2019, Marcus Geelnard <m at bitsnbites dot eu>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define REDIS_SOCKCOMPAT_IMPLEMENTATION
+#include "sockcompat.h"
+
+#ifdef _WIN32
+static int _wsaErrorToErrno(int err) {
+    switch (err) {
+        case WSAEWOULDBLOCK:
+            return EWOULDBLOCK;
+        case WSAEINPROGRESS:
+            return EINPROGRESS;
+        case WSAEALREADY:
+            return EALREADY;
+        case WSAENOTSOCK:
+            return ENOTSOCK;
+        case WSAEDESTADDRREQ:
+            return EDESTADDRREQ;
+        case WSAEMSGSIZE:
+            return EMSGSIZE;
+        case WSAEPROTOTYPE:
+            return EPROTOTYPE;
+        case WSAENOPROTOOPT:
+            return ENOPROTOOPT;
+        case WSAEPROTONOSUPPORT:
+            return EPROTONOSUPPORT;
+        case WSAEOPNOTSUPP:
+            return EOPNOTSUPP;
+        case WSAEAFNOSUPPORT:
+            return EAFNOSUPPORT;
+        case WSAEADDRINUSE:
+            return EADDRINUSE;
+        case WSAEADDRNOTAVAIL:
+            return EADDRNOTAVAIL;
+        case WSAENETDOWN:
+            return ENETDOWN;
+        case WSAENETUNREACH:
+            return ENETUNREACH;
+        case WSAENETRESET:
+            return ENETRESET;
+        case WSAECONNABORTED:
+            return ECONNABORTED;
+        case WSAECONNRESET:
+            return ECONNRESET;
+        case WSAENOBUFS:
+            return ENOBUFS;
+        case WSAEISCONN:
+            return EISCONN;
+        case WSAENOTCONN:
+            return ENOTCONN;
+        case WSAETIMEDOUT:
+            return ETIMEDOUT;
+        case WSAECONNREFUSED:
+            return ECONNREFUSED;
+        case WSAELOOP:
+            return ELOOP;
+        case WSAENAMETOOLONG:
+            return ENAMETOOLONG;
+        case WSAEHOSTUNREACH:
+            return EHOSTUNREACH;
+        case WSAENOTEMPTY:
+            return ENOTEMPTY;
+        default:
+            /* We just return a generic I/O error if we could not find a relevant error. */
+            return EIO;
+    }
+}
+
+static void _updateErrno(int success) {
+    errno = success ? 0 : _wsaErrorToErrno(WSAGetLastError());
+}
+
+static int _initWinsock() {
+    static int s_initialized = 0;
+    if (!s_initialized) {
+        static WSADATA wsadata;
+        int err = WSAStartup(MAKEWORD(2,2), &wsadata);
+        if (err != 0) {
+            errno = _wsaErrorToErrno(err);
+            return 0;
+        }
+        s_initialized = 1;
+    }
+    return 1;
+}
+
+int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res) {
+    /* Note: This function is likely to be called before other functions, so run init here. */
+    if (!_initWinsock()) {
+        return EAI_FAIL;
+    }
+
+    switch (getaddrinfo(node, service, hints, res)) {
+        case 0:                     return 0;
+        case WSATRY_AGAIN:          return EAI_AGAIN;
+        case WSAEINVAL:             return EAI_BADFLAGS;
+        case WSAEAFNOSUPPORT:       return EAI_FAMILY;
+        case WSA_NOT_ENOUGH_MEMORY: return EAI_MEMORY;
+        case WSAHOST_NOT_FOUND:     return EAI_NONAME;
+        case WSATYPE_NOT_FOUND:     return EAI_SERVICE;
+        case WSAESOCKTNOSUPPORT:    return EAI_SOCKTYPE;
+        default:                    return EAI_FAIL;     /* Including WSANO_RECOVERY */
+    }
+}
+
+const char *win32_gai_strerror(int errcode) {
+    switch (errcode) {
+        case 0:            errcode = 0;                     break;
+        case EAI_AGAIN:    errcode = WSATRY_AGAIN;          break;
+        case EAI_BADFLAGS: errcode = WSAEINVAL;             break;
+        case EAI_FAMILY:   errcode = WSAEAFNOSUPPORT;       break;
+        case EAI_MEMORY:   errcode = WSA_NOT_ENOUGH_MEMORY; break;
+        case EAI_NONAME:   errcode = WSAHOST_NOT_FOUND;     break;
+        case EAI_SERVICE:  errcode = WSATYPE_NOT_FOUND;     break;
+        case EAI_SOCKTYPE: errcode = WSAESOCKTNOSUPPORT;    break;
+        default:           errcode = WSANO_RECOVERY;        break; /* Including EAI_FAIL */
+    }
+    return gai_strerror(errcode);
+}
+
+void win32_freeaddrinfo(struct addrinfo *res) {
+    freeaddrinfo(res);
+}
+
+SOCKET win32_socket(int domain, int type, int protocol) {
+    SOCKET s;
+
+    /* Note: This function is likely to be called before other functions, so run init here. */
+    if (!_initWinsock()) {
+        return INVALID_SOCKET;
+    }
+
+    _updateErrno((s = socket(domain, type, protocol)) != INVALID_SOCKET);
+    return s;
+}
+
+int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp) {
+    int ret = ioctlsocket(fd, (long)request, argp);
+    _updateErrno(ret != SOCKET_ERROR);
+    return ret != SOCKET_ERROR ? ret : -1;
+}
+
+int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) {
+    int ret = bind(sockfd, addr, addrlen);
+    _updateErrno(ret != SOCKET_ERROR);
+    return ret != SOCKET_ERROR ? ret : -1;
+}
+
+int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) {
+    int ret = connect(sockfd, addr, addrlen);
+    _updateErrno(ret != SOCKET_ERROR);
+
+    /* For Winsock connect(), the WSAEWOULDBLOCK error means the same thing as
+     * EINPROGRESS for POSIX connect(), so we do that translation to keep POSIX
+     * logic consistent. */
+    if (errno == EWOULDBLOCK) {
+        errno = EINPROGRESS;
+    }
+
+    return ret != SOCKET_ERROR ? ret : -1;
+}
+
+int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen) {
+    int ret = 0;
+    if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) {
+        if (*optlen >= sizeof (struct timeval)) {
+            struct timeval *tv = optval;
+            DWORD timeout = 0;
+            socklen_t dwlen = 0;
+            ret = getsockopt(sockfd, level, optname, (char *)&timeout, &dwlen);
+            tv->tv_sec = timeout / 1000;
+            tv->tv_usec = (timeout * 1000) % 1000000;
+        } else {
+            ret = WSAEFAULT;
+        }
+        *optlen = sizeof (struct timeval);
+    } else {
+        ret = getsockopt(sockfd, level, optname, (char*)optval, optlen);
+    }
+    _updateErrno(ret != SOCKET_ERROR);
+    return ret != SOCKET_ERROR ? ret : -1;
+}
+
+int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen) {
+    int ret = 0;
+    if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) {
+        const struct timeval *tv = optval;
+        DWORD timeout = tv->tv_sec * 1000 + tv->tv_usec / 1000;
+        ret = setsockopt(sockfd, level, optname, (const char*)&timeout, sizeof(DWORD));
+    } else {
+        ret = setsockopt(sockfd, level, optname, (const char*)optval, optlen);
+    }
+    _updateErrno(ret != SOCKET_ERROR);
+    return ret != SOCKET_ERROR ? ret : -1;
+}
+
+int win32_close(SOCKET fd) {
+    int ret = closesocket(fd);
+    _updateErrno(ret != SOCKET_ERROR);
+    return ret != SOCKET_ERROR ? ret : -1;
+}
+
+ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags) {
+    int ret = recv(sockfd, (char*)buf, (int)len, flags);
+    _updateErrno(ret != SOCKET_ERROR);
+    return ret != SOCKET_ERROR ? ret : -1;
+}
+
+ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags) {
+    int ret = send(sockfd, (const char*)buf, (int)len, flags);
+    _updateErrno(ret != SOCKET_ERROR);
+    return ret != SOCKET_ERROR ? ret : -1;
+}
+
+int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout) {
+    int ret = WSAPoll(fds, nfds, timeout);
+    _updateErrno(ret != SOCKET_ERROR);
+    return ret != SOCKET_ERROR ? ret : -1;
+}
+#endif /* _WIN32 */
diff --git a/sockcompat.h b/sockcompat.h
new file mode 100644
index 000000000..85810e848
--- /dev/null
+++ b/sockcompat.h
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2019, Marcus Geelnard <m at bitsnbites dot eu>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __SOCKCOMPAT_H
+#define __SOCKCOMPAT_H
+
+#ifndef _WIN32
+/* For POSIX systems we use the standard BSD socket API. */
+#include <unistd.h>
+#include <sys/socket.h>
+#include <sys/select.h>
+#include <sys/un.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <poll.h>
+#else
+/* For Windows we use winsock. */
+#undef _WIN32_WINNT
+#define _WIN32_WINNT 0x0600 /* To get WSAPoll etc. */
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#include <stddef.h>
+#include <errno.h>
+
+#ifdef _MSC_VER
+typedef long long ssize_t;
+#endif
+
+/* Emulate the parts of the BSD socket API that we need (override the winsock signatures). */
+int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res);
+const char *win32_gai_strerror(int errcode);
+void win32_freeaddrinfo(struct addrinfo *res);
+SOCKET win32_socket(int domain, int type, int protocol);
+int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp);
+int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen);
+int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen);
+int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen);
+int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen);
+int win32_close(SOCKET fd);
+ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags);
+ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags);
+typedef ULONG nfds_t;
+int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout);
+
+#ifndef REDIS_SOCKCOMPAT_IMPLEMENTATION
+#define getaddrinfo(node, service, hints, res) win32_getaddrinfo(node, service, hints, res)
+#undef gai_strerror
+#define gai_strerror(errcode) win32_gai_strerror(errcode)
+#define freeaddrinfo(res) win32_freeaddrinfo(res)
+#define socket(domain, type, protocol) win32_socket(domain, type, protocol)
+#define ioctl(fd, request, argp) win32_ioctl(fd, request, argp)
+#define bind(sockfd, addr, addrlen) win32_bind(sockfd, addr, addrlen)
+#define connect(sockfd, addr, addrlen) win32_connect(sockfd, addr, addrlen)
+#define getsockopt(sockfd, level, optname, optval, optlen) win32_getsockopt(sockfd, level, optname, optval, optlen)
+#define setsockopt(sockfd, level, optname, optval, optlen) win32_setsockopt(sockfd, level, optname, optval, optlen)
+#define close(fd) win32_close(fd)
+#define recv(sockfd, buf, len, flags) win32_recv(sockfd, buf, len, flags)
+#define send(sockfd, buf, len, flags) win32_send(sockfd, buf, len, flags)
+#define poll(fds, nfds, timeout) win32_poll(fds, nfds, timeout)
+#endif /* REDIS_SOCKCOMPAT_IMPLEMENTATION */
+#endif /* _WIN32 */
+
+#endif /* __SOCKCOMPAT_H */
diff --git a/ssl.c b/ssl.c
new file mode 100644
index 000000000..7df58fbde
--- /dev/null
+++ b/ssl.c
@@ -0,0 +1,526 @@
+/*
+ * Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ * Copyright (c) 2019, Redis Labs
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "hiredis.h"
+#include "async.h"
+
+#include <assert.h>
+#include <errno.h>
+#include <string.h>
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <pthread.h>
+#endif
+
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+
+#include "win32.h"
+#include "async_private.h"
+#include "hiredis_ssl.h"
+
+void __redisSetError(redisContext *c, int type, const char *str);
+
+struct redisSSLContext {
+    /* Associated OpenSSL SSL_CTX as created by redisCreateSSLContext() */
+    SSL_CTX *ssl_ctx;
+
+    /* Requested SNI, or NULL */
+    char *server_name;
+};
+
+/* The SSL connection context is attached to SSL/TLS connections as a privdata. */
+typedef struct redisSSL {
+    /**
+     * OpenSSL SSL object.
+     */
+    SSL *ssl;
+
+    /**
+     * SSL_write() requires to be called again with the same arguments it was
+     * previously called with in the event of an SSL_read/SSL_write situation
+     */
+    size_t lastLen;
+
+    /** Whether the SSL layer requires read (possibly before a write) */
+    int wantRead;
+
+    /**
+     * Whether a write was requested prior to a read. If set, the write()
+     * should resume whenever a read takes place, if possible
+     */
+    int pendingWrite;
+} redisSSL;
+
+/* Forward declaration */
+redisContextFuncs redisContextSSLFuncs;
+
+/**
+ * OpenSSL global initialization and locking handling callbacks.
+ * Note that this is only required for OpenSSL < 1.1.0.
+ */
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+#define HIREDIS_USE_CRYPTO_LOCKS
+#endif
+
+#ifdef HIREDIS_USE_CRYPTO_LOCKS
+#ifdef _WIN32
+typedef CRITICAL_SECTION sslLockType;
+static void sslLockInit(sslLockType* l) {
+    InitializeCriticalSection(l);
+}
+static void sslLockAcquire(sslLockType* l) {
+    EnterCriticalSection(l);
+}
+static void sslLockRelease(sslLockType* l) {
+    LeaveCriticalSection(l);
+}
+#else
+typedef pthread_mutex_t sslLockType;
+static void sslLockInit(sslLockType *l) {
+    pthread_mutex_init(l, NULL);
+}
+static void sslLockAcquire(sslLockType *l) {
+    pthread_mutex_lock(l);
+}
+static void sslLockRelease(sslLockType *l) {
+    pthread_mutex_unlock(l);
+}
+#endif
+
+static sslLockType* ossl_locks;
+
+static void opensslDoLock(int mode, int lkid, const char *f, int line) {
+    sslLockType *l = ossl_locks + lkid;
+
+    if (mode & CRYPTO_LOCK) {
+        sslLockAcquire(l);
+    } else {
+        sslLockRelease(l);
+    }
+
+    (void)f;
+    (void)line;
+}
+
+static int initOpensslLocks(void) {
+    unsigned ii, nlocks;
+    if (CRYPTO_get_locking_callback() != NULL) {
+        /* Someone already set the callback before us. Don't destroy it! */
+        return REDIS_OK;
+    }
+    nlocks = CRYPTO_num_locks();
+    ossl_locks = hi_malloc(sizeof(*ossl_locks) * nlocks);
+    if (ossl_locks == NULL)
+        return REDIS_ERR;
+
+    for (ii = 0; ii < nlocks; ii++) {
+        sslLockInit(ossl_locks + ii);
+    }
+    CRYPTO_set_locking_callback(opensslDoLock);
+    return REDIS_OK;
+}
+#endif /* HIREDIS_USE_CRYPTO_LOCKS */
+
+int redisInitOpenSSL(void)
+{
+    SSL_library_init();
+#ifdef HIREDIS_USE_CRYPTO_LOCKS
+    initOpensslLocks();
+#endif
+
+    return REDIS_OK;
+}
+
+/**
+ * redisSSLContext helper context destruction.
+ */
+
+const char *redisSSLContextGetError(redisSSLContextError error)
+{
+    switch (error) {
+        case REDIS_SSL_CTX_NONE:
+            return "No Error";
+        case REDIS_SSL_CTX_CREATE_FAILED:
+            return "Failed to create OpenSSL SSL_CTX";
+        case REDIS_SSL_CTX_CERT_KEY_REQUIRED:
+            return "Client cert and key must both be specified or skipped";
+        case REDIS_SSL_CTX_CA_CERT_LOAD_FAILED:
+            return "Failed to load CA Certificate or CA Path";
+        case REDIS_SSL_CTX_CLIENT_CERT_LOAD_FAILED:
+            return "Failed to load client certificate";
+        case REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED:
+            return "Failed to load private key";
+        default:
+            return "Unknown error code";
+    }
+}
+
+void redisFreeSSLContext(redisSSLContext *ctx)
+{
+    if (!ctx)
+        return;
+
+    if (ctx->server_name) {
+        hi_free(ctx->server_name);
+        ctx->server_name = NULL;
+    }
+
+    if (ctx->ssl_ctx) {
+        SSL_CTX_free(ctx->ssl_ctx);
+        ctx->ssl_ctx = NULL;
+    }
+
+    hi_free(ctx);
+}
+
+
+/**
+ * redisSSLContext helper context initialization.
+ */
+
+redisSSLContext *redisCreateSSLContext(const char *cacert_filename, const char *capath,
+        const char *cert_filename, const char *private_key_filename,
+        const char *server_name, redisSSLContextError *error)
+{
+    redisSSLContext *ctx = hi_calloc(1, sizeof(redisSSLContext));
+    if (ctx == NULL)
+        goto error;
+
+    ctx->ssl_ctx = SSL_CTX_new(SSLv23_client_method());
+    if (!ctx->ssl_ctx) {
+        if (error) *error = REDIS_SSL_CTX_CREATE_FAILED;
+        goto error;
+    }
+
+    SSL_CTX_set_options(ctx->ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3);
+    SSL_CTX_set_verify(ctx->ssl_ctx, SSL_VERIFY_PEER, NULL);
+
+    if ((cert_filename != NULL && private_key_filename == NULL) ||
+            (private_key_filename != NULL && cert_filename == NULL)) {
+        if (error) *error = REDIS_SSL_CTX_CERT_KEY_REQUIRED;
+        goto error;
+    }
+
+    if (capath || cacert_filename) {
+        if (!SSL_CTX_load_verify_locations(ctx->ssl_ctx, cacert_filename, capath)) {
+            if (error) *error = REDIS_SSL_CTX_CA_CERT_LOAD_FAILED;
+            goto error;
+        }
+    }
+
+    if (cert_filename) {
+        if (!SSL_CTX_use_certificate_chain_file(ctx->ssl_ctx, cert_filename)) {
+            if (error) *error = REDIS_SSL_CTX_CLIENT_CERT_LOAD_FAILED;
+            goto error;
+        }
+        if (!SSL_CTX_use_PrivateKey_file(ctx->ssl_ctx, private_key_filename, SSL_FILETYPE_PEM)) {
+            if (error) *error = REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED;
+            goto error;
+        }
+    }
+
+    if (server_name)
+        ctx->server_name = hi_strdup(server_name);
+
+    return ctx;
+
+error:
+    redisFreeSSLContext(ctx);
+    return NULL;
+}
+
+/**
+ * SSL Connection initialization.
+ */
+
+
+static int redisSSLConnect(redisContext *c, SSL *ssl) {
+    if (c->privctx) {
+        __redisSetError(c, REDIS_ERR_OTHER, "redisContext was already associated");
+        return REDIS_ERR;
+    }
+
+    redisSSL *rssl = hi_calloc(1, sizeof(redisSSL));
+    if (rssl == NULL) {
+        __redisSetError(c, REDIS_ERR_OOM, "Out of memory");
+        return REDIS_ERR;
+    }
+
+    c->funcs = &redisContextSSLFuncs;
+    rssl->ssl = ssl;
+
+    SSL_set_mode(rssl->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
+    SSL_set_fd(rssl->ssl, c->fd);
+    SSL_set_connect_state(rssl->ssl);
+
+    ERR_clear_error();
+    int rv = SSL_connect(rssl->ssl);
+    if (rv == 1) {
+        c->privctx = rssl;
+        return REDIS_OK;
+    }
+
+    rv = SSL_get_error(rssl->ssl, rv);
+    if (((c->flags & REDIS_BLOCK) == 0) &&
+        (rv == SSL_ERROR_WANT_READ || rv == SSL_ERROR_WANT_WRITE)) {
+        c->privctx = rssl;
+        return REDIS_OK;
+    }
+
+    if (c->err == 0) {
+        char err[512];
+        if (rv == SSL_ERROR_SYSCALL)
+            snprintf(err,sizeof(err)-1,"SSL_connect failed: %s",strerror(errno));
+        else {
+            unsigned long e = ERR_peek_last_error();
+            snprintf(err,sizeof(err)-1,"SSL_connect failed: %s",
+                    ERR_reason_error_string(e));
+        }
+        __redisSetError(c, REDIS_ERR_IO, err);
+    }
+
+    hi_free(rssl);
+    return REDIS_ERR;
+}
+
+/**
+ * A wrapper around redisSSLConnect() for users who manage their own context and
+ * create their own SSL object.
+ */
+
+int redisInitiateSSL(redisContext *c, SSL *ssl) {
+    return redisSSLConnect(c, ssl);
+}
+
+/**
+ * A wrapper around redisSSLConnect() for users who use redisSSLContext and don't
+ * manage their own SSL objects.
+ */
+
+int redisInitiateSSLWithContext(redisContext *c, redisSSLContext *redis_ssl_ctx)
+{
+    if (!c || !redis_ssl_ctx)
+        return REDIS_ERR;
+
+    /* We want to verify that redisSSLConnect() won't fail on this, as it will
+     * not own the SSL object in that case and we'll end up leaking.
+     */
+    if (c->privctx)
+        return REDIS_ERR;
+
+    SSL *ssl = SSL_new(redis_ssl_ctx->ssl_ctx);
+    if (!ssl) {
+        __redisSetError(c, REDIS_ERR_OTHER, "Couldn't create new SSL instance");
+        goto error;
+    }
+
+    if (redis_ssl_ctx->server_name) {
+        if (!SSL_set_tlsext_host_name(ssl, redis_ssl_ctx->server_name)) {
+            __redisSetError(c, REDIS_ERR_OTHER, "Failed to set server_name/SNI");
+            goto error;
+        }
+    }
+
+    return redisSSLConnect(c, ssl);
+
+error:
+    if (ssl)
+        SSL_free(ssl);
+    return REDIS_ERR;
+}
+
+static int maybeCheckWant(redisSSL *rssl, int rv) {
+    /**
+     * If the error is WANT_READ or WANT_WRITE, the appropriate flags are set
+     * and true is returned. False is returned otherwise
+     */
+    if (rv == SSL_ERROR_WANT_READ) {
+        rssl->wantRead = 1;
+        return 1;
+    } else if (rv == SSL_ERROR_WANT_WRITE) {
+        rssl->pendingWrite = 1;
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+/**
+ * Implementation of redisContextFuncs for SSL connections.
+ */
+
+static void redisSSLFree(void *privctx){
+    redisSSL *rsc = privctx;
+
+    if (!rsc) return;
+    if (rsc->ssl) {
+        SSL_free(rsc->ssl);
+        rsc->ssl = NULL;
+    }
+    hi_free(rsc);
+}
+
+static ssize_t redisSSLRead(redisContext *c, char *buf, size_t bufcap) {
+    redisSSL *rssl = c->privctx;
+
+    int nread = SSL_read(rssl->ssl, buf, bufcap);
+    if (nread > 0) {
+        return nread;
+    } else if (nread == 0) {
+        __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection");
+        return -1;
+    } else {
+        int err = SSL_get_error(rssl->ssl, nread);
+        if (c->flags & REDIS_BLOCK) {
+            /**
+             * In blocking mode, we should never end up in a situation where
+             * we get an error without it being an actual error, except
+             * in the case of EINTR, which can be spuriously received from
+             * debuggers or whatever.
+             */
+            if (errno == EINTR) {
+                return 0;
+            } else {
+                const char *msg = NULL;
+                if (errno == EAGAIN) {
+                    msg = "Resource temporarily unavailable";
+                }
+                __redisSetError(c, REDIS_ERR_IO, msg);
+                return -1;
+            }
+        }
+
+        /**
+         * We can very well get an EWOULDBLOCK/EAGAIN, however
+         */
+        if (maybeCheckWant(rssl, err)) {
+            return 0;
+        } else {
+            __redisSetError(c, REDIS_ERR_IO, NULL);
+            return -1;
+        }
+    }
+}
+
+static ssize_t redisSSLWrite(redisContext *c) {
+    redisSSL *rssl = c->privctx;
+
+    size_t len = rssl->lastLen ? rssl->lastLen : sdslen(c->obuf);
+    int rv = SSL_write(rssl->ssl, c->obuf, len);
+
+    if (rv > 0) {
+        rssl->lastLen = 0;
+    } else if (rv < 0) {
+        rssl->lastLen = len;
+
+        int err = SSL_get_error(rssl->ssl, rv);
+        if ((c->flags & REDIS_BLOCK) == 0 && maybeCheckWant(rssl, err)) {
+            return 0;
+        } else {
+            __redisSetError(c, REDIS_ERR_IO, NULL);
+            return -1;
+        }
+    }
+    return rv;
+}
+
+static void redisSSLAsyncRead(redisAsyncContext *ac) {
+    int rv;
+    redisSSL *rssl = ac->c.privctx;
+    redisContext *c = &ac->c;
+
+    rssl->wantRead = 0;
+
+    if (rssl->pendingWrite) {
+        int done;
+
+        /* This is probably just a write event */
+        rssl->pendingWrite = 0;
+        rv = redisBufferWrite(c, &done);
+        if (rv == REDIS_ERR) {
+            __redisAsyncDisconnect(ac);
+            return;
+        } else if (!done) {
+            _EL_ADD_WRITE(ac);
+        }
+    }
+
+    rv = redisBufferRead(c);
+    if (rv == REDIS_ERR) {
+        __redisAsyncDisconnect(ac);
+    } else {
+        _EL_ADD_READ(ac);
+        redisProcessCallbacks(ac);
+    }
+}
+
+static void redisSSLAsyncWrite(redisAsyncContext *ac) {
+    int rv, done = 0;
+    redisSSL *rssl = ac->c.privctx;
+    redisContext *c = &ac->c;
+
+    rssl->pendingWrite = 0;
+    rv = redisBufferWrite(c, &done);
+    if (rv == REDIS_ERR) {
+        __redisAsyncDisconnect(ac);
+        return;
+    }
+
+    if (!done) {
+        if (rssl->wantRead) {
+            /* Need to read-before-write */
+            rssl->pendingWrite = 1;
+            _EL_DEL_WRITE(ac);
+        } else {
+            /* No extra reads needed, just need to write more */
+            _EL_ADD_WRITE(ac);
+        }
+    } else {
+        /* Already done! */
+        _EL_DEL_WRITE(ac);
+    }
+
+    /* Always reschedule a read */
+    _EL_ADD_READ(ac);
+}
+
+redisContextFuncs redisContextSSLFuncs = {
+    .free_privctx = redisSSLFree,
+    .async_read = redisSSLAsyncRead,
+    .async_write = redisSSLAsyncWrite,
+    .read = redisSSLRead,
+    .write = redisSSLWrite
+};
+
diff --git a/test.c b/test.c
new file mode 100644
index 000000000..c0eeca70d
--- /dev/null
+++ b/test.c
@@ -0,0 +1,1387 @@
+#include "fmacros.h"
+#include "sockcompat.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#ifndef _WIN32
+#include <strings.h>
+#include <sys/time.h>
+#endif
+#include <assert.h>
+#include <signal.h>
+#include <errno.h>
+#include <limits.h>
+
+#include "hiredis.h"
+#include "async.h"
+#ifdef HIREDIS_TEST_SSL
+#include "hiredis_ssl.h"
+#endif
+#include "net.h"
+#include "win32.h"
+
+enum connection_type {
+    CONN_TCP,
+    CONN_UNIX,
+    CONN_FD,
+    CONN_SSL
+};
+
+struct config {
+    enum connection_type type;
+
+    struct {
+        const char *host;
+        int port;
+        struct timeval timeout;
+    } tcp;
+
+    struct {
+        const char *path;
+    } unix_sock;
+
+    struct {
+        const char *host;
+        int port;
+        const char *ca_cert;
+        const char *cert;
+        const char *key;
+    } ssl;
+};
+
+struct privdata {
+    int dtor_counter;
+};
+
+#ifdef HIREDIS_TEST_SSL
+redisSSLContext *_ssl_ctx = NULL;
+#endif
+
+/* The following lines make up our testing "framework" :) */
+static int tests = 0, fails = 0, skips = 0;
+#define test(_s) { printf("#%02d ", ++tests); printf(_s); }
+#define test_cond(_c) if(_c) printf("\033[0;32mPASSED\033[0;0m\n"); else {printf("\033[0;31mFAILED\033[0;0m\n"); fails++;}
+#define test_skipped() { printf("\033[01;33mSKIPPED\033[0;0m\n"); skips++; }
+
+static long long usec(void) {
+#ifndef _MSC_VER
+    struct timeval tv;
+    gettimeofday(&tv,NULL);
+    return (((long long)tv.tv_sec)*1000000)+tv.tv_usec;
+#else
+    FILETIME ft;
+    GetSystemTimeAsFileTime(&ft);
+    return (((long long)ft.dwHighDateTime << 32) | ft.dwLowDateTime) / 10;
+#endif
+}
+
+/* The assert() calls below have side effects, so we need assert()
+ * even if we are compiling without asserts (-DNDEBUG). */
+#ifdef NDEBUG
+#undef assert
+#define assert(e) (void)(e)
+#endif
+
+/* Helper to extract Redis version information.  Aborts on any failure. */
+#define REDIS_VERSION_FIELD "redis_version:"
+void get_redis_version(redisContext *c, int *majorptr, int *minorptr) {
+    redisReply *reply;
+    char *eptr, *s, *e;
+    int major, minor;
+
+    reply = redisCommand(c, "INFO");
+    if (reply == NULL || c->err || reply->type != REDIS_REPLY_STRING)
+        goto abort;
+    if ((s = strstr(reply->str, REDIS_VERSION_FIELD)) == NULL)
+        goto abort;
+
+    s += strlen(REDIS_VERSION_FIELD);
+
+    /* We need a field terminator and at least 'x.y.z' (5) bytes of data */
+    if ((e = strstr(s, "\r\n")) == NULL || (e - s) < 5)
+        goto abort;
+
+    /* Extract version info */
+    major = strtol(s, &eptr, 10);
+    if (*eptr != '.') goto abort;
+    minor = strtol(eptr+1, NULL, 10);
+
+    /* Push info the caller wants */
+    if (majorptr) *majorptr = major;
+    if (minorptr) *minorptr = minor;
+
+    freeReplyObject(reply);
+    return;
+
+abort:
+    freeReplyObject(reply);
+    fprintf(stderr, "Error:  Cannot determine Redis version, aborting\n");
+    exit(1);
+}
+
+static redisContext *select_database(redisContext *c) {
+    redisReply *reply;
+
+    /* Switch to DB 9 for testing, now that we know we can chat. */
+    reply = redisCommand(c,"SELECT 9");
+    assert(reply != NULL);
+    freeReplyObject(reply);
+
+    /* Make sure the DB is emtpy */
+    reply = redisCommand(c,"DBSIZE");
+    assert(reply != NULL);
+    if (reply->type == REDIS_REPLY_INTEGER && reply->integer == 0) {
+        /* Awesome, DB 9 is empty and we can continue. */
+        freeReplyObject(reply);
+    } else {
+        printf("Database #9 is not empty, test can not continue\n");
+        exit(1);
+    }
+
+    return c;
+}
+
+/* Switch protocol */
+static void send_hello(redisContext *c, int version) {
+    redisReply *reply;
+    int expected;
+
+    reply = redisCommand(c, "HELLO %d", version);
+    expected = version == 3 ? REDIS_REPLY_MAP : REDIS_REPLY_ARRAY;
+    assert(reply != NULL && reply->type == expected);
+    freeReplyObject(reply);
+}
+
+/* Togggle client tracking */
+static void send_client_tracking(redisContext *c, const char *str) {
+    redisReply *reply;
+
+    reply = redisCommand(c, "CLIENT TRACKING %s", str);
+    assert(reply != NULL && reply->type == REDIS_REPLY_STATUS);
+    freeReplyObject(reply);
+}
+
+static int disconnect(redisContext *c, int keep_fd) {
+    redisReply *reply;
+
+    /* Make sure we're on DB 9. */
+    reply = redisCommand(c,"SELECT 9");
+    assert(reply != NULL);
+    freeReplyObject(reply);
+    reply = redisCommand(c,"FLUSHDB");
+    assert(reply != NULL);
+    freeReplyObject(reply);
+
+    /* Free the context as well, but keep the fd if requested. */
+    if (keep_fd)
+        return redisFreeKeepFd(c);
+    redisFree(c);
+    return -1;
+}
+
+static void do_ssl_handshake(redisContext *c) {
+#ifdef HIREDIS_TEST_SSL
+    redisInitiateSSLWithContext(c, _ssl_ctx);
+    if (c->err) {
+        printf("SSL error: %s\n", c->errstr);
+        redisFree(c);
+        exit(1);
+    }
+#else
+    (void) c;
+#endif
+}
+
+static redisContext *do_connect(struct config config) {
+    redisContext *c = NULL;
+
+    if (config.type == CONN_TCP) {
+        c = redisConnect(config.tcp.host, config.tcp.port);
+    } else if (config.type == CONN_SSL) {
+        c = redisConnect(config.ssl.host, config.ssl.port);
+    } else if (config.type == CONN_UNIX) {
+        c = redisConnectUnix(config.unix_sock.path);
+    } else if (config.type == CONN_FD) {
+        /* Create a dummy connection just to get an fd to inherit */
+        redisContext *dummy_ctx = redisConnectUnix(config.unix_sock.path);
+        if (dummy_ctx) {
+            int fd = disconnect(dummy_ctx, 1);
+            printf("Connecting to inherited fd %d\n", fd);
+            c = redisConnectFd(fd);
+        }
+    } else {
+        assert(NULL);
+    }
+
+    if (c == NULL) {
+        printf("Connection error: can't allocate redis context\n");
+        exit(1);
+    } else if (c->err) {
+        printf("Connection error: %s\n", c->errstr);
+        redisFree(c);
+        exit(1);
+    }
+
+    if (config.type == CONN_SSL) {
+        do_ssl_handshake(c);
+    }
+
+    return select_database(c);
+}
+
+static void do_reconnect(redisContext *c, struct config config) {
+    redisReconnect(c);
+
+    if (config.type == CONN_SSL) {
+        do_ssl_handshake(c);
+    }
+}
+
+static void test_format_commands(void) {
+    char *cmd;
+    int len;
+
+    test("Format command without interpolation: ");
+    len = redisFormatCommand(&cmd,"SET foo bar");
+    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 &&
+        len == 4+4+(3+2)+4+(3+2)+4+(3+2));
+    hi_free(cmd);
+
+    test("Format command with %%s string interpolation: ");
+    len = redisFormatCommand(&cmd,"SET %s %s","foo","bar");
+    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 &&
+        len == 4+4+(3+2)+4+(3+2)+4+(3+2));
+    hi_free(cmd);
+
+    test("Format command with %%s and an empty string: ");
+    len = redisFormatCommand(&cmd,"SET %s %s","foo","");
+    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 &&
+        len == 4+4+(3+2)+4+(3+2)+4+(0+2));
+    hi_free(cmd);
+
+    test("Format command with an empty string in between proper interpolations: ");
+    len = redisFormatCommand(&cmd,"SET %s %s","","foo");
+    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$0\r\n\r\n$3\r\nfoo\r\n",len) == 0 &&
+        len == 4+4+(3+2)+4+(0+2)+4+(3+2));
+    hi_free(cmd);
+
+    test("Format command with %%b string interpolation: ");
+    len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"b\0r",(size_t)3);
+    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nb\0r\r\n",len) == 0 &&
+        len == 4+4+(3+2)+4+(3+2)+4+(3+2));
+    hi_free(cmd);
+
+    test("Format command with %%b and an empty string: ");
+    len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"",(size_t)0);
+    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 &&
+        len == 4+4+(3+2)+4+(3+2)+4+(0+2));
+    hi_free(cmd);
+
+    test("Format command with literal %%: ");
+    len = redisFormatCommand(&cmd,"SET %% %%");
+    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$1\r\n%\r\n$1\r\n%\r\n",len) == 0 &&
+        len == 4+4+(3+2)+4+(1+2)+4+(1+2));
+    hi_free(cmd);
+
+    /* Vararg width depends on the type. These tests make sure that the
+     * width is correctly determined using the format and subsequent varargs
+     * can correctly be interpolated. */
+#define INTEGER_WIDTH_TEST(fmt, type) do {                                                \
+    type value = 123;                                                                     \
+    test("Format command with printf-delegation (" #type "): ");                          \
+    len = redisFormatCommand(&cmd,"key:%08" fmt " str:%s", value, "hello");               \
+    test_cond(strncmp(cmd,"*2\r\n$12\r\nkey:00000123\r\n$9\r\nstr:hello\r\n",len) == 0 && \
+        len == 4+5+(12+2)+4+(9+2));                                                       \
+    hi_free(cmd);                                                                         \
+} while(0)
+
+#define FLOAT_WIDTH_TEST(type) do {                                                       \
+    type value = 123.0;                                                                   \
+    test("Format command with printf-delegation (" #type "): ");                          \
+    len = redisFormatCommand(&cmd,"key:%08.3f str:%s", value, "hello");                   \
+    test_cond(strncmp(cmd,"*2\r\n$12\r\nkey:0123.000\r\n$9\r\nstr:hello\r\n",len) == 0 && \
+        len == 4+5+(12+2)+4+(9+2));                                                       \
+    hi_free(cmd);                                                                         \
+} while(0)
+
+    INTEGER_WIDTH_TEST("d", int);
+    INTEGER_WIDTH_TEST("hhd", char);
+    INTEGER_WIDTH_TEST("hd", short);
+    INTEGER_WIDTH_TEST("ld", long);
+    INTEGER_WIDTH_TEST("lld", long long);
+    INTEGER_WIDTH_TEST("u", unsigned int);
+    INTEGER_WIDTH_TEST("hhu", unsigned char);
+    INTEGER_WIDTH_TEST("hu", unsigned short);
+    INTEGER_WIDTH_TEST("lu", unsigned long);
+    INTEGER_WIDTH_TEST("llu", unsigned long long);
+    FLOAT_WIDTH_TEST(float);
+    FLOAT_WIDTH_TEST(double);
+
+    test("Format command with invalid printf format: ");
+    len = redisFormatCommand(&cmd,"key:%08p %b",(void*)1234,"foo",(size_t)3);
+    test_cond(len == -1);
+
+    const char *argv[3];
+    argv[0] = "SET";
+    argv[1] = "foo\0xxx";
+    argv[2] = "bar";
+    size_t lens[3] = { 3, 7, 3 };
+    int argc = 3;
+
+    test("Format command by passing argc/argv without lengths: ");
+    len = redisFormatCommandArgv(&cmd,argc,argv,NULL);
+    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 &&
+        len == 4+4+(3+2)+4+(3+2)+4+(3+2));
+    hi_free(cmd);
+
+    test("Format command by passing argc/argv with lengths: ");
+    len = redisFormatCommandArgv(&cmd,argc,argv,lens);
+    test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 &&
+        len == 4+4+(3+2)+4+(7+2)+4+(3+2));
+    hi_free(cmd);
+
+    sds sds_cmd;
+
+    sds_cmd = NULL;
+    test("Format command into sds by passing argc/argv without lengths: ");
+    len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,NULL);
+    test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 &&
+        len == 4+4+(3+2)+4+(3+2)+4+(3+2));
+    sdsfree(sds_cmd);
+
+    sds_cmd = NULL;
+    test("Format command into sds by passing argc/argv with lengths: ");
+    len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,lens);
+    test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 &&
+        len == 4+4+(3+2)+4+(7+2)+4+(3+2));
+    sdsfree(sds_cmd);
+}
+
+static void test_append_formatted_commands(struct config config) {
+    redisContext *c;
+    redisReply *reply;
+    char *cmd;
+    int len;
+
+    c = do_connect(config);
+
+    test("Append format command: ");
+
+    len = redisFormatCommand(&cmd, "SET foo bar");
+
+    test_cond(redisAppendFormattedCommand(c, cmd, len) == REDIS_OK);
+
+    assert(redisGetReply(c, (void*)&reply) == REDIS_OK);
+
+    hi_free(cmd);
+    freeReplyObject(reply);
+
+    disconnect(c, 0);
+}
+
+static void test_reply_reader(void) {
+    redisReader *reader;
+    void *reply, *root;
+    int ret;
+    int i;
+
+    test("Error handling in reply parser: ");
+    reader = redisReaderCreate();
+    redisReaderFeed(reader,(char*)"@foo\r\n",6);
+    ret = redisReaderGetReply(reader,NULL);
+    test_cond(ret == REDIS_ERR &&
+              strcasecmp(reader->errstr,"Protocol error, got \"@\" as reply type byte") == 0);
+    redisReaderFree(reader);
+
+    /* when the reply already contains multiple items, they must be free'd
+     * on an error. valgrind will bark when this doesn't happen. */
+    test("Memory cleanup in reply parser: ");
+    reader = redisReaderCreate();
+    redisReaderFeed(reader,(char*)"*2\r\n",4);
+    redisReaderFeed(reader,(char*)"$5\r\nhello\r\n",11);
+    redisReaderFeed(reader,(char*)"@foo\r\n",6);
+    ret = redisReaderGetReply(reader,NULL);
+    test_cond(ret == REDIS_ERR &&
+              strcasecmp(reader->errstr,"Protocol error, got \"@\" as reply type byte") == 0);
+    redisReaderFree(reader);
+
+    reader = redisReaderCreate();
+    test("Can handle arbitrarily nested multi-bulks: ");
+    for (i = 0; i < 128; i++) {
+        redisReaderFeed(reader,(char*)"*1\r\n", 4);
+    }
+    redisReaderFeed(reader,(char*)"$6\r\nLOLWUT\r\n",12);
+    ret = redisReaderGetReply(reader,&reply);
+    root = reply; /* Keep track of the root reply */
+    test_cond(ret == REDIS_OK &&
+        ((redisReply*)reply)->type == REDIS_REPLY_ARRAY &&
+        ((redisReply*)reply)->elements == 1);
+
+    test("Can parse arbitrarily nested multi-bulks correctly: ");
+    while(i--) {
+        assert(reply != NULL && ((redisReply*)reply)->type == REDIS_REPLY_ARRAY);
+        reply = ((redisReply*)reply)->element[0];
+    }
+    test_cond(((redisReply*)reply)->type == REDIS_REPLY_STRING &&
+        !memcmp(((redisReply*)reply)->str, "LOLWUT", 6));
+    freeReplyObject(root);
+    redisReaderFree(reader);
+
+    test("Correctly parses LLONG_MAX: ");
+    reader = redisReaderCreate();
+    redisReaderFeed(reader, ":9223372036854775807\r\n",22);
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_OK &&
+            ((redisReply*)reply)->type == REDIS_REPLY_INTEGER &&
+            ((redisReply*)reply)->integer == LLONG_MAX);
+    freeReplyObject(reply);
+    redisReaderFree(reader);
+
+    test("Set error when > LLONG_MAX: ");
+    reader = redisReaderCreate();
+    redisReaderFeed(reader, ":9223372036854775808\r\n",22);
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_ERR &&
+              strcasecmp(reader->errstr,"Bad integer value") == 0);
+    freeReplyObject(reply);
+    redisReaderFree(reader);
+
+    test("Correctly parses LLONG_MIN: ");
+    reader = redisReaderCreate();
+    redisReaderFeed(reader, ":-9223372036854775808\r\n",23);
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_OK &&
+            ((redisReply*)reply)->type == REDIS_REPLY_INTEGER &&
+            ((redisReply*)reply)->integer == LLONG_MIN);
+    freeReplyObject(reply);
+    redisReaderFree(reader);
+
+    test("Set error when < LLONG_MIN: ");
+    reader = redisReaderCreate();
+    redisReaderFeed(reader, ":-9223372036854775809\r\n",23);
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_ERR &&
+              strcasecmp(reader->errstr,"Bad integer value") == 0);
+    freeReplyObject(reply);
+    redisReaderFree(reader);
+
+    test("Set error when array < -1: ");
+    reader = redisReaderCreate();
+    redisReaderFeed(reader, "*-2\r\n+asdf\r\n",12);
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_ERR &&
+              strcasecmp(reader->errstr,"Multi-bulk length out of range") == 0);
+    freeReplyObject(reply);
+    redisReaderFree(reader);
+
+    test("Set error when bulk < -1: ");
+    reader = redisReaderCreate();
+    redisReaderFeed(reader, "$-2\r\nasdf\r\n",11);
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_ERR &&
+              strcasecmp(reader->errstr,"Bulk string length out of range") == 0);
+    freeReplyObject(reply);
+    redisReaderFree(reader);
+
+    test("Can configure maximum multi-bulk elements: ");
+    reader = redisReaderCreate();
+    reader->maxelements = 1024;
+    redisReaderFeed(reader, "*1025\r\n", 7);
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_ERR &&
+              strcasecmp(reader->errstr, "Multi-bulk length out of range") == 0);
+    freeReplyObject(reply);
+    redisReaderFree(reader);
+
+#if LLONG_MAX > SIZE_MAX
+    test("Set error when array > SIZE_MAX: ");
+    reader = redisReaderCreate();
+    redisReaderFeed(reader, "*9223372036854775807\r\n+asdf\r\n",29);
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_ERR &&
+            strcasecmp(reader->errstr,"Multi-bulk length out of range") == 0);
+    freeReplyObject(reply);
+    redisReaderFree(reader);
+
+    test("Set error when bulk > SIZE_MAX: ");
+    reader = redisReaderCreate();
+    redisReaderFeed(reader, "$9223372036854775807\r\nasdf\r\n",28);
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_ERR &&
+            strcasecmp(reader->errstr,"Bulk string length out of range") == 0);
+    freeReplyObject(reply);
+    redisReaderFree(reader);
+#endif
+
+    test("Works with NULL functions for reply: ");
+    reader = redisReaderCreate();
+    reader->fn = NULL;
+    redisReaderFeed(reader,(char*)"+OK\r\n",5);
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_OK && reply == (void*)REDIS_REPLY_STATUS);
+    redisReaderFree(reader);
+
+    test("Works when a single newline (\\r\\n) covers two calls to feed: ");
+    reader = redisReaderCreate();
+    reader->fn = NULL;
+    redisReaderFeed(reader,(char*)"+OK\r",4);
+    ret = redisReaderGetReply(reader,&reply);
+    assert(ret == REDIS_OK && reply == NULL);
+    redisReaderFeed(reader,(char*)"\n",1);
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_OK && reply == (void*)REDIS_REPLY_STATUS);
+    redisReaderFree(reader);
+
+    test("Don't reset state after protocol error: ");
+    reader = redisReaderCreate();
+    reader->fn = NULL;
+    redisReaderFeed(reader,(char*)"x",1);
+    ret = redisReaderGetReply(reader,&reply);
+    assert(ret == REDIS_ERR);
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_ERR && reply == NULL);
+    redisReaderFree(reader);
+
+    /* Regression test for issue #45 on GitHub. */
+    test("Don't do empty allocation for empty multi bulk: ");
+    reader = redisReaderCreate();
+    redisReaderFeed(reader,(char*)"*0\r\n",4);
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_OK &&
+        ((redisReply*)reply)->type == REDIS_REPLY_ARRAY &&
+        ((redisReply*)reply)->elements == 0);
+    freeReplyObject(reply);
+    redisReaderFree(reader);
+
+    /* RESP3 verbatim strings (GitHub issue #802) */
+    test("Can parse RESP3 verbatim strings: ");
+    reader = redisReaderCreate();
+    redisReaderFeed(reader,(char*)"=10\r\ntxt:LOLWUT\r\n",17);
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_OK &&
+        ((redisReply*)reply)->type == REDIS_REPLY_VERB &&
+         !memcmp(((redisReply*)reply)->str,"LOLWUT", 6));
+    freeReplyObject(reply);
+    redisReaderFree(reader);
+
+    /* RESP3 push messages (Github issue #815) */
+    test("Can parse RESP3 push messages: ");
+    reader = redisReaderCreate();
+    redisReaderFeed(reader,(char*)">2\r\n$6\r\nLOLWUT\r\n:42\r\n",21);
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_OK &&
+        ((redisReply*)reply)->type == REDIS_REPLY_PUSH &&
+        ((redisReply*)reply)->elements == 2 &&
+        ((redisReply*)reply)->element[0]->type == REDIS_REPLY_STRING &&
+        !memcmp(((redisReply*)reply)->element[0]->str,"LOLWUT",6) &&
+        ((redisReply*)reply)->element[1]->type == REDIS_REPLY_INTEGER &&
+        ((redisReply*)reply)->element[1]->integer == 42);
+    freeReplyObject(reply);
+    redisReaderFree(reader);
+}
+
+static void test_free_null(void) {
+    void *redisCtx = NULL;
+    void *reply = NULL;
+
+    test("Don't fail when redisFree is passed a NULL value: ");
+    redisFree(redisCtx);
+    test_cond(redisCtx == NULL);
+
+    test("Don't fail when freeReplyObject is passed a NULL value: ");
+    freeReplyObject(reply);
+    test_cond(reply == NULL);
+}
+
+static void *hi_malloc_fail(size_t size) {
+    (void)size;
+    return NULL;
+}
+
+static void *hi_calloc_fail(size_t nmemb, size_t size) {
+    (void)nmemb;
+    (void)size;
+    return NULL;
+}
+
+static void *hi_realloc_fail(void *ptr, size_t size) {
+    (void)ptr;
+    (void)size;
+    return NULL;
+}
+
+static void test_allocator_injection(void) {
+    hiredisAllocFuncs ha = {
+        .mallocFn = hi_malloc_fail,
+        .callocFn = hi_calloc_fail,
+        .reallocFn = hi_realloc_fail,
+        .strdupFn = strdup,
+        .freeFn = free,
+    };
+
+    // Override hiredis allocators
+    hiredisSetAllocators(&ha);
+
+    test("redisContext uses injected allocators: ");
+    redisContext *c = redisConnect("localhost", 6379);
+    test_cond(c == NULL);
+
+    test("redisReader uses injected allocators: ");
+    redisReader *reader = redisReaderCreate();
+    test_cond(reader == NULL);
+
+    // Return allocators to default
+    hiredisResetAllocators();
+}
+
+#define HIREDIS_BAD_DOMAIN "idontexist-noreally.com"
+static void test_blocking_connection_errors(void) {
+    redisContext *c;
+    struct addrinfo hints = {.ai_family = AF_INET};
+    struct addrinfo *ai_tmp = NULL;
+
+    int rv = getaddrinfo(HIREDIS_BAD_DOMAIN, "6379", &hints, &ai_tmp);
+    if (rv != 0) {
+        // Address does *not* exist
+        test("Returns error when host cannot be resolved: ");
+        // First see if this domain name *actually* resolves to NXDOMAIN
+        c = redisConnect(HIREDIS_BAD_DOMAIN, 6379);
+        test_cond(
+            c->err == REDIS_ERR_OTHER &&
+            (strcmp(c->errstr, "Name or service not known") == 0 ||
+             strcmp(c->errstr, "Can't resolve: " HIREDIS_BAD_DOMAIN) == 0 ||
+             strcmp(c->errstr, "Name does not resolve") == 0 ||
+             strcmp(c->errstr, "nodename nor servname provided, or not known") == 0 ||
+             strcmp(c->errstr, "No address associated with hostname") == 0 ||
+             strcmp(c->errstr, "Temporary failure in name resolution") == 0 ||
+             strcmp(c->errstr, "hostname nor servname provided, or not known") == 0 ||
+             strcmp(c->errstr, "no address associated with name") == 0 ||
+             strcmp(c->errstr, "No such host is known. ") == 0));
+        redisFree(c);
+    } else {
+        printf("Skipping NXDOMAIN test. Found evil ISP!\n");
+        freeaddrinfo(ai_tmp);
+    }
+
+#ifndef _WIN32
+    test("Returns error when the port is not open: ");
+    c = redisConnect((char*)"localhost", 1);
+    test_cond(c->err == REDIS_ERR_IO &&
+        strcmp(c->errstr,"Connection refused") == 0);
+    redisFree(c);
+
+    test("Returns error when the unix_sock socket path doesn't accept connections: ");
+    c = redisConnectUnix((char*)"/tmp/idontexist.sock");
+    test_cond(c->err == REDIS_ERR_IO); /* Don't care about the message... */
+    redisFree(c);
+#endif
+}
+
+/* Dummy push handler */
+void push_handler(void *privdata, void *reply) {
+    int *counter = privdata;
+    freeReplyObject(reply);
+    *counter += 1;
+}
+
+/* Dummy function just to test setting a callback with redisOptions */
+void push_handler_async(redisAsyncContext *ac, void *reply) {
+    (void)ac;
+    (void)reply;
+}
+
+static void test_resp3_push_handler(redisContext *c) {
+    redisPushFn *old = NULL;
+    redisReply *reply;
+    void *privdata;
+    int n = 0;
+
+    /* Switch to RESP3 and turn on client tracking */
+    send_hello(c, 3);
+    send_client_tracking(c, "ON");
+    privdata = c->privdata;
+    c->privdata = &n;
+
+    reply = redisCommand(c, "GET key:0");
+    assert(reply != NULL);
+    freeReplyObject(reply);
+
+    test("RESP3 PUSH messages are handled out of band by default: ");
+    reply = redisCommand(c, "SET key:0 val:0");
+    test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS);
+    freeReplyObject(reply);
+
+    assert((reply = redisCommand(c, "GET key:0")) != NULL);
+    freeReplyObject(reply);
+
+    old = redisSetPushCallback(c, push_handler);
+    test("We can set a custom RESP3 PUSH handler: ");
+    reply = redisCommand(c, "SET key:0 val:0");
+    test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && n == 1);
+    freeReplyObject(reply);
+
+    /* Unset the push callback and generate an invalidate message making
+     * sure it is not handled out of band. */
+    test("With no handler, PUSH replies come in-band: ");
+    redisSetPushCallback(c, NULL);
+    assert((reply = redisCommand(c, "GET key:0")) != NULL);
+    freeReplyObject(reply);
+    assert((reply = redisCommand(c, "SET key:0 invalid")) != NULL);
+    test_cond(reply->type == REDIS_REPLY_PUSH);
+    freeReplyObject(reply);
+
+    test("With no PUSH handler, no replies are lost: ");
+    assert(redisGetReply(c, (void**)&reply) == REDIS_OK);
+    test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS);
+    freeReplyObject(reply);
+
+    /* Return to the originally set PUSH handler */
+    assert(old != NULL);
+    redisSetPushCallback(c, old);
+
+    /* Switch back to RESP2 and disable tracking */
+    c->privdata = privdata;
+    send_client_tracking(c, "OFF");
+    send_hello(c, 2);
+}
+
+redisOptions get_redis_tcp_options(struct config config) {
+    redisOptions options = {0};
+    REDIS_OPTIONS_SET_TCP(&options, config.tcp.host, config.tcp.port);
+    return options;
+}
+
+static void test_resp3_push_options(struct config config) {
+    redisAsyncContext *ac;
+    redisContext *c;
+    redisOptions options;
+
+    test("We set a default RESP3 handler for redisContext: ");
+    options = get_redis_tcp_options(config);
+    assert((c = redisConnectWithOptions(&options)) != NULL);
+    test_cond(c->push_cb != NULL);
+    redisFree(c);
+
+    test("We don't set a default RESP3 push handler for redisAsyncContext: ");
+    options = get_redis_tcp_options(config);
+    assert((ac = redisAsyncConnectWithOptions(&options)) != NULL);
+    test_cond(ac->c.push_cb == NULL);
+    redisAsyncFree(ac);
+
+    test("Our REDIS_OPT_NO_PUSH_AUTOFREE flag works: ");
+    options = get_redis_tcp_options(config);
+    options.options |= REDIS_OPT_NO_PUSH_AUTOFREE;
+    assert((c = redisConnectWithOptions(&options)) != NULL);
+    test_cond(c->push_cb == NULL);
+    redisFree(c);
+
+    test("We can use redisOptions to set a custom PUSH handler for redisContext: ");
+    options = get_redis_tcp_options(config);
+    options.push_cb = push_handler;
+    assert((c = redisConnectWithOptions(&options)) != NULL);
+    test_cond(c->push_cb == push_handler);
+    redisFree(c);
+
+    test("We can use redisOptions to set a custom PUSH handler for redisAsyncContext: ");
+    options = get_redis_tcp_options(config);
+    options.async_push_cb = push_handler_async;
+    assert((ac = redisAsyncConnectWithOptions(&options)) != NULL);
+    test_cond(ac->push_cb == push_handler_async);
+    redisAsyncFree(ac);
+}
+
+void free_privdata(void *privdata) {
+    struct privdata *data = privdata;
+    data->dtor_counter++;
+}
+
+static void test_privdata_hooks(struct config config) {
+    struct privdata data = {0};
+    redisOptions options;
+    redisContext *c;
+
+    test("We can use redisOptions to set privdata: ");
+    options = get_redis_tcp_options(config);
+    REDIS_OPTIONS_SET_PRIVDATA(&options, &data, free_privdata);
+    assert((c = redisConnectWithOptions(&options)) != NULL);
+    test_cond(c->privdata == &data);
+
+    test("Our privdata destructor fires when we free the context: ");
+    redisFree(c);
+    test_cond(data.dtor_counter == 1);
+}
+
+static void test_blocking_connection(struct config config) {
+    redisContext *c;
+    redisReply *reply;
+    int major;
+
+    c = do_connect(config);
+
+    test("Is able to deliver commands: ");
+    reply = redisCommand(c,"PING");
+    test_cond(reply->type == REDIS_REPLY_STATUS &&
+        strcasecmp(reply->str,"pong") == 0)
+    freeReplyObject(reply);
+
+    test("Is a able to send commands verbatim: ");
+    reply = redisCommand(c,"SET foo bar");
+    test_cond (reply->type == REDIS_REPLY_STATUS &&
+        strcasecmp(reply->str,"ok") == 0)
+    freeReplyObject(reply);
+
+    test("%%s String interpolation works: ");
+    reply = redisCommand(c,"SET %s %s","foo","hello world");
+    freeReplyObject(reply);
+    reply = redisCommand(c,"GET foo");
+    test_cond(reply->type == REDIS_REPLY_STRING &&
+        strcmp(reply->str,"hello world") == 0);
+    freeReplyObject(reply);
+
+    test("%%b String interpolation works: ");
+    reply = redisCommand(c,"SET %b %b","foo",(size_t)3,"hello\x00world",(size_t)11);
+    freeReplyObject(reply);
+    reply = redisCommand(c,"GET foo");
+    test_cond(reply->type == REDIS_REPLY_STRING &&
+        memcmp(reply->str,"hello\x00world",11) == 0)
+
+    test("Binary reply length is correct: ");
+    test_cond(reply->len == 11)
+    freeReplyObject(reply);
+
+    test("Can parse nil replies: ");
+    reply = redisCommand(c,"GET nokey");
+    test_cond(reply->type == REDIS_REPLY_NIL)
+    freeReplyObject(reply);
+
+    /* test 7 */
+    test("Can parse integer replies: ");
+    reply = redisCommand(c,"INCR mycounter");
+    test_cond(reply->type == REDIS_REPLY_INTEGER && reply->integer == 1)
+    freeReplyObject(reply);
+
+    test("Can parse multi bulk replies: ");
+    freeReplyObject(redisCommand(c,"LPUSH mylist foo"));
+    freeReplyObject(redisCommand(c,"LPUSH mylist bar"));
+    reply = redisCommand(c,"LRANGE mylist 0 -1");
+    test_cond(reply->type == REDIS_REPLY_ARRAY &&
+              reply->elements == 2 &&
+              !memcmp(reply->element[0]->str,"bar",3) &&
+              !memcmp(reply->element[1]->str,"foo",3))
+    freeReplyObject(reply);
+
+    /* m/e with multi bulk reply *before* other reply.
+     * specifically test ordering of reply items to parse. */
+    test("Can handle nested multi bulk replies: ");
+    freeReplyObject(redisCommand(c,"MULTI"));
+    freeReplyObject(redisCommand(c,"LRANGE mylist 0 -1"));
+    freeReplyObject(redisCommand(c,"PING"));
+    reply = (redisCommand(c,"EXEC"));
+    test_cond(reply->type == REDIS_REPLY_ARRAY &&
+              reply->elements == 2 &&
+              reply->element[0]->type == REDIS_REPLY_ARRAY &&
+              reply->element[0]->elements == 2 &&
+              !memcmp(reply->element[0]->element[0]->str,"bar",3) &&
+              !memcmp(reply->element[0]->element[1]->str,"foo",3) &&
+              reply->element[1]->type == REDIS_REPLY_STATUS &&
+              strcasecmp(reply->element[1]->str,"pong") == 0);
+    freeReplyObject(reply);
+
+    /* Make sure passing NULL to redisGetReply is safe */
+    test("Can pass NULL to redisGetReply: ");
+    assert(redisAppendCommand(c, "PING") == REDIS_OK);
+    test_cond(redisGetReply(c, NULL) == REDIS_OK);
+
+    get_redis_version(c, &major, NULL);
+    if (major >= 6) test_resp3_push_handler(c);
+    test_resp3_push_options(config);
+
+    test_privdata_hooks(config);
+
+    disconnect(c, 0);
+}
+
+/* Send DEBUG SLEEP 0 to detect if we have this command */
+static int detect_debug_sleep(redisContext *c) {
+    int detected;
+    redisReply *reply = redisCommand(c, "DEBUG SLEEP 0\r\n");
+
+    if (reply == NULL || c->err) {
+        const char *cause = c->err ? c->errstr : "(none)";
+        fprintf(stderr, "Error testing for DEBUG SLEEP (Redis error: %s), exiting\n", cause);
+        exit(-1);
+    }
+
+    detected = reply->type == REDIS_REPLY_STATUS;
+    freeReplyObject(reply);
+
+    return detected;
+}
+
+static void test_blocking_connection_timeouts(struct config config) {
+    redisContext *c;
+    redisReply *reply;
+    ssize_t s;
+    const char *sleep_cmd = "DEBUG SLEEP 3\r\n";
+    struct timeval tv;
+
+    c = do_connect(config);
+    test("Successfully completes a command when the timeout is not exceeded: ");
+    reply = redisCommand(c,"SET foo fast");
+    freeReplyObject(reply);
+    tv.tv_sec = 0;
+    tv.tv_usec = 10000;
+    redisSetTimeout(c, tv);
+    reply = redisCommand(c, "GET foo");
+    test_cond(reply != NULL && reply->type == REDIS_REPLY_STRING && memcmp(reply->str, "fast", 4) == 0);
+    freeReplyObject(reply);
+    disconnect(c, 0);
+
+    c = do_connect(config);
+    test("Does not return a reply when the command times out: ");
+    if (detect_debug_sleep(c)) {
+        redisAppendFormattedCommand(c, sleep_cmd, strlen(sleep_cmd));
+        s = c->funcs->write(c);
+        tv.tv_sec = 0;
+        tv.tv_usec = 10000;
+        redisSetTimeout(c, tv);
+        reply = redisCommand(c, "GET foo");
+#ifndef _WIN32
+        test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_IO &&
+                  strcmp(c->errstr, "Resource temporarily unavailable") == 0);
+#else
+        test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_TIMEOUT &&
+                  strcmp(c->errstr, "recv timeout") == 0);
+#endif
+        freeReplyObject(reply);
+    } else {
+        test_skipped();
+    }
+
+    test("Reconnect properly reconnects after a timeout: ");
+    do_reconnect(c, config);
+    reply = redisCommand(c, "PING");
+    test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0);
+    freeReplyObject(reply);
+
+    test("Reconnect properly uses owned parameters: ");
+    config.tcp.host = "foo";
+    config.unix_sock.path = "foo";
+    do_reconnect(c, config);
+    reply = redisCommand(c, "PING");
+    test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0);
+    freeReplyObject(reply);
+
+    disconnect(c, 0);
+}
+
+static void test_blocking_io_errors(struct config config) {
+    redisContext *c;
+    redisReply *reply;
+    void *_reply;
+    int major, minor;
+
+    /* Connect to target given by config. */
+    c = do_connect(config);
+    get_redis_version(c, &major, &minor);
+
+    test("Returns I/O error when the connection is lost: ");
+    reply = redisCommand(c,"QUIT");
+    if (major > 2 || (major == 2 && minor > 0)) {
+        /* > 2.0 returns OK on QUIT and read() should be issued once more
+         * to know the descriptor is at EOF. */
+        test_cond(strcasecmp(reply->str,"OK") == 0 &&
+            redisGetReply(c,&_reply) == REDIS_ERR);
+        freeReplyObject(reply);
+    } else {
+        test_cond(reply == NULL);
+    }
+
+#ifndef _WIN32
+    /* On 2.0, QUIT will cause the connection to be closed immediately and
+     * the read(2) for the reply on QUIT will set the error to EOF.
+     * On >2.0, QUIT will return with OK and another read(2) needed to be
+     * issued to find out the socket was closed by the server. In both
+     * conditions, the error will be set to EOF. */
+    assert(c->err == REDIS_ERR_EOF &&
+        strcmp(c->errstr,"Server closed the connection") == 0);
+#endif
+    redisFree(c);
+
+    c = do_connect(config);
+    test("Returns I/O error on socket timeout: ");
+    struct timeval tv = { 0, 1000 };
+    assert(redisSetTimeout(c,tv) == REDIS_OK);
+    int respcode = redisGetReply(c,&_reply);
+#ifndef _WIN32
+    test_cond(respcode == REDIS_ERR && c->err == REDIS_ERR_IO && errno == EAGAIN);
+#else
+    test_cond(respcode == REDIS_ERR && c->err == REDIS_ERR_TIMEOUT);
+#endif
+    redisFree(c);
+}
+
+static void test_invalid_timeout_errors(struct config config) {
+    redisContext *c;
+
+    test("Set error when an invalid timeout usec value is given to redisConnectWithTimeout: ");
+
+    config.tcp.timeout.tv_sec = 0;
+    config.tcp.timeout.tv_usec = 10000001;
+
+    c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout);
+
+    test_cond(c->err == REDIS_ERR_IO && strcmp(c->errstr, "Invalid timeout specified") == 0);
+    redisFree(c);
+
+    test("Set error when an invalid timeout sec value is given to redisConnectWithTimeout: ");
+
+    config.tcp.timeout.tv_sec = (((LONG_MAX) - 999) / 1000) + 1;
+    config.tcp.timeout.tv_usec = 0;
+
+    c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout);
+
+    test_cond(c->err == REDIS_ERR_IO && strcmp(c->errstr, "Invalid timeout specified") == 0);
+    redisFree(c);
+}
+
+/* Wrap malloc to abort on failure so OOM checks don't make the test logic
+ * harder to follow. */
+void *hi_malloc_safe(size_t size) {
+    void *ptr = hi_malloc(size);
+    if (ptr == NULL) {
+        fprintf(stderr, "Error:  Out of memory\n");
+        exit(-1);
+    }
+
+    return ptr;
+}
+
+static void test_throughput(struct config config) {
+    redisContext *c = do_connect(config);
+    redisReply **replies;
+    int i, num;
+    long long t1, t2;
+
+    test("Throughput:\n");
+    for (i = 0; i < 500; i++)
+        freeReplyObject(redisCommand(c,"LPUSH mylist foo"));
+
+    num = 1000;
+    replies = hi_malloc_safe(sizeof(redisReply*)*num);
+    t1 = usec();
+    for (i = 0; i < num; i++) {
+        replies[i] = redisCommand(c,"PING");
+        assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_STATUS);
+    }
+    t2 = usec();
+    for (i = 0; i < num; i++) freeReplyObject(replies[i]);
+    hi_free(replies);
+    printf("\t(%dx PING: %.3fs)\n", num, (t2-t1)/1000000.0);
+
+    replies = hi_malloc_safe(sizeof(redisReply*)*num);
+    t1 = usec();
+    for (i = 0; i < num; i++) {
+        replies[i] = redisCommand(c,"LRANGE mylist 0 499");
+        assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_ARRAY);
+        assert(replies[i] != NULL && replies[i]->elements == 500);
+    }
+    t2 = usec();
+    for (i = 0; i < num; i++) freeReplyObject(replies[i]);
+    hi_free(replies);
+    printf("\t(%dx LRANGE with 500 elements: %.3fs)\n", num, (t2-t1)/1000000.0);
+
+    replies = hi_malloc_safe(sizeof(redisReply*)*num);
+    t1 = usec();
+    for (i = 0; i < num; i++) {
+        replies[i] = redisCommand(c, "INCRBY incrkey %d", 1000000);
+        assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_INTEGER);
+    }
+    t2 = usec();
+    for (i = 0; i < num; i++) freeReplyObject(replies[i]);
+    hi_free(replies);
+    printf("\t(%dx INCRBY: %.3fs)\n", num, (t2-t1)/1000000.0);
+
+    num = 10000;
+    replies = hi_malloc_safe(sizeof(redisReply*)*num);
+    for (i = 0; i < num; i++)
+        redisAppendCommand(c,"PING");
+    t1 = usec();
+    for (i = 0; i < num; i++) {
+        assert(redisGetReply(c, (void*)&replies[i]) == REDIS_OK);
+        assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_STATUS);
+    }
+    t2 = usec();
+    for (i = 0; i < num; i++) freeReplyObject(replies[i]);
+    hi_free(replies);
+    printf("\t(%dx PING (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0);
+
+    replies = hi_malloc_safe(sizeof(redisReply*)*num);
+    for (i = 0; i < num; i++)
+        redisAppendCommand(c,"LRANGE mylist 0 499");
+    t1 = usec();
+    for (i = 0; i < num; i++) {
+        assert(redisGetReply(c, (void*)&replies[i]) == REDIS_OK);
+        assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_ARRAY);
+        assert(replies[i] != NULL && replies[i]->elements == 500);
+    }
+    t2 = usec();
+    for (i = 0; i < num; i++) freeReplyObject(replies[i]);
+    hi_free(replies);
+    printf("\t(%dx LRANGE with 500 elements (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0);
+
+    replies = hi_malloc_safe(sizeof(redisReply*)*num);
+    for (i = 0; i < num; i++)
+        redisAppendCommand(c,"INCRBY incrkey %d", 1000000);
+    t1 = usec();
+    for (i = 0; i < num; i++) {
+        assert(redisGetReply(c, (void*)&replies[i]) == REDIS_OK);
+        assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_INTEGER);
+    }
+    t2 = usec();
+    for (i = 0; i < num; i++) freeReplyObject(replies[i]);
+    hi_free(replies);
+    printf("\t(%dx INCRBY (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0);
+
+    disconnect(c, 0);
+}
+
+// static long __test_callback_flags = 0;
+// static void __test_callback(redisContext *c, void *privdata) {
+//     ((void)c);
+//     /* Shift to detect execution order */
+//     __test_callback_flags <<= 8;
+//     __test_callback_flags |= (long)privdata;
+// }
+//
+// static void __test_reply_callback(redisContext *c, redisReply *reply, void *privdata) {
+//     ((void)c);
+//     /* Shift to detect execution order */
+//     __test_callback_flags <<= 8;
+//     __test_callback_flags |= (long)privdata;
+//     if (reply) freeReplyObject(reply);
+// }
+//
+// static redisContext *__connect_nonblock() {
+//     /* Reset callback flags */
+//     __test_callback_flags = 0;
+//     return redisConnectNonBlock("127.0.0.1", port, NULL);
+// }
+//
+// static void test_nonblocking_connection() {
+//     redisContext *c;
+//     int wdone = 0;
+//
+//     test("Calls command callback when command is issued: ");
+//     c = __connect_nonblock();
+//     redisSetCommandCallback(c,__test_callback,(void*)1);
+//     redisCommand(c,"PING");
+//     test_cond(__test_callback_flags == 1);
+//     redisFree(c);
+//
+//     test("Calls disconnect callback on redisDisconnect: ");
+//     c = __connect_nonblock();
+//     redisSetDisconnectCallback(c,__test_callback,(void*)2);
+//     redisDisconnect(c);
+//     test_cond(__test_callback_flags == 2);
+//     redisFree(c);
+//
+//     test("Calls disconnect callback and free callback on redisFree: ");
+//     c = __connect_nonblock();
+//     redisSetDisconnectCallback(c,__test_callback,(void*)2);
+//     redisSetFreeCallback(c,__test_callback,(void*)4);
+//     redisFree(c);
+//     test_cond(__test_callback_flags == ((2 << 8) | 4));
+//
+//     test("redisBufferWrite against empty write buffer: ");
+//     c = __connect_nonblock();
+//     test_cond(redisBufferWrite(c,&wdone) == REDIS_OK && wdone == 1);
+//     redisFree(c);
+//
+//     test("redisBufferWrite against not yet connected fd: ");
+//     c = __connect_nonblock();
+//     redisCommand(c,"PING");
+//     test_cond(redisBufferWrite(c,NULL) == REDIS_ERR &&
+//               strncmp(c->error,"write:",6) == 0);
+//     redisFree(c);
+//
+//     test("redisBufferWrite against closed fd: ");
+//     c = __connect_nonblock();
+//     redisCommand(c,"PING");
+//     redisDisconnect(c);
+//     test_cond(redisBufferWrite(c,NULL) == REDIS_ERR &&
+//               strncmp(c->error,"write:",6) == 0);
+//     redisFree(c);
+//
+//     test("Process callbacks in the right sequence: ");
+//     c = __connect_nonblock();
+//     redisCommandWithCallback(c,__test_reply_callback,(void*)1,"PING");
+//     redisCommandWithCallback(c,__test_reply_callback,(void*)2,"PING");
+//     redisCommandWithCallback(c,__test_reply_callback,(void*)3,"PING");
+//
+//     /* Write output buffer */
+//     wdone = 0;
+//     while(!wdone) {
+//         usleep(500);
+//         redisBufferWrite(c,&wdone);
+//     }
+//
+//     /* Read until at least one callback is executed (the 3 replies will
+//      * arrive in a single packet, causing all callbacks to be executed in
+//      * a single pass). */
+//     while(__test_callback_flags == 0) {
+//         assert(redisBufferRead(c) == REDIS_OK);
+//         redisProcessCallbacks(c);
+//     }
+//     test_cond(__test_callback_flags == 0x010203);
+//     redisFree(c);
+//
+//     test("redisDisconnect executes pending callbacks with NULL reply: ");
+//     c = __connect_nonblock();
+//     redisSetDisconnectCallback(c,__test_callback,(void*)1);
+//     redisCommandWithCallback(c,__test_reply_callback,(void*)2,"PING");
+//     redisDisconnect(c);
+//     test_cond(__test_callback_flags == 0x0201);
+//     redisFree(c);
+// }
+
+int main(int argc, char **argv) {
+    struct config cfg = {
+        .tcp = {
+            .host = "127.0.0.1",
+            .port = 6379
+        },
+        .unix_sock = {
+            .path = "/tmp/redis.sock"
+        }
+    };
+    int throughput = 1;
+    int test_inherit_fd = 1;
+    int skips_as_fails = 0;
+    int test_unix_socket;
+
+    /* Parse command line options. */
+    argv++; argc--;
+    while (argc) {
+        if (argc >= 2 && !strcmp(argv[0],"-h")) {
+            argv++; argc--;
+            cfg.tcp.host = argv[0];
+        } else if (argc >= 2 && !strcmp(argv[0],"-p")) {
+            argv++; argc--;
+            cfg.tcp.port = atoi(argv[0]);
+        } else if (argc >= 2 && !strcmp(argv[0],"-s")) {
+            argv++; argc--;
+            cfg.unix_sock.path = argv[0];
+        } else if (argc >= 1 && !strcmp(argv[0],"--skip-throughput")) {
+            throughput = 0;
+        } else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) {
+            test_inherit_fd = 0;
+        } else if (argc >= 1 && !strcmp(argv[0],"--skips-as-fails")) {
+            skips_as_fails = 1;
+#ifdef HIREDIS_TEST_SSL
+        } else if (argc >= 2 && !strcmp(argv[0],"--ssl-port")) {
+            argv++; argc--;
+            cfg.ssl.port = atoi(argv[0]);
+        } else if (argc >= 2 && !strcmp(argv[0],"--ssl-host")) {
+            argv++; argc--;
+            cfg.ssl.host = argv[0];
+        } else if (argc >= 2 && !strcmp(argv[0],"--ssl-ca-cert")) {
+            argv++; argc--;
+            cfg.ssl.ca_cert  = argv[0];
+        } else if (argc >= 2 && !strcmp(argv[0],"--ssl-cert")) {
+            argv++; argc--;
+            cfg.ssl.cert = argv[0];
+        } else if (argc >= 2 && !strcmp(argv[0],"--ssl-key")) {
+            argv++; argc--;
+            cfg.ssl.key = argv[0];
+#endif
+        } else {
+            fprintf(stderr, "Invalid argument: %s\n", argv[0]);
+            exit(1);
+        }
+        argv++; argc--;
+    }
+
+#ifndef _WIN32
+    /* Ignore broken pipe signal (for I/O error tests). */
+    signal(SIGPIPE, SIG_IGN);
+
+    test_unix_socket = access(cfg.unix_sock.path, F_OK) == 0;
+
+#else
+    /* Unix sockets don't exist in Windows */
+    test_unix_socket = 0;
+#endif
+
+    test_allocator_injection();
+
+    test_format_commands();
+    test_reply_reader();
+    test_blocking_connection_errors();
+    test_free_null();
+
+    printf("\nTesting against TCP connection (%s:%d):\n", cfg.tcp.host, cfg.tcp.port);
+    cfg.type = CONN_TCP;
+    test_blocking_connection(cfg);
+    test_blocking_connection_timeouts(cfg);
+    test_blocking_io_errors(cfg);
+    test_invalid_timeout_errors(cfg);
+    test_append_formatted_commands(cfg);
+    if (throughput) test_throughput(cfg);
+
+    printf("\nTesting against Unix socket connection (%s): ", cfg.unix_sock.path);
+    if (test_unix_socket) {
+        printf("\n");
+        cfg.type = CONN_UNIX;
+        test_blocking_connection(cfg);
+        test_blocking_connection_timeouts(cfg);
+        test_blocking_io_errors(cfg);
+        if (throughput) test_throughput(cfg);
+    } else {
+        test_skipped();
+    }
+
+#ifdef HIREDIS_TEST_SSL
+    if (cfg.ssl.port && cfg.ssl.host) {
+
+        redisInitOpenSSL();
+        _ssl_ctx = redisCreateSSLContext(cfg.ssl.ca_cert, NULL, cfg.ssl.cert, cfg.ssl.key, NULL, NULL);
+        assert(_ssl_ctx != NULL);
+
+        printf("\nTesting against SSL connection (%s:%d):\n", cfg.ssl.host, cfg.ssl.port);
+        cfg.type = CONN_SSL;
+
+        test_blocking_connection(cfg);
+        test_blocking_connection_timeouts(cfg);
+        test_blocking_io_errors(cfg);
+        test_invalid_timeout_errors(cfg);
+        test_append_formatted_commands(cfg);
+        if (throughput) test_throughput(cfg);
+
+        redisFreeSSLContext(_ssl_ctx);
+        _ssl_ctx = NULL;
+    }
+#endif
+
+    if (test_inherit_fd) {
+        printf("\nTesting against inherited fd (%s): ", cfg.unix_sock.path);
+        if (test_unix_socket) {
+            printf("\n");
+            cfg.type = CONN_FD;
+            test_blocking_connection(cfg);
+        } else {
+            test_skipped();
+        }
+    }
+
+    if (fails || (skips_as_fails && skips)) {
+        printf("*** %d TESTS FAILED ***\n", fails);
+        if (skips) {
+            printf("*** %d TESTS SKIPPED ***\n", skips);
+        }
+        return 1;
+    }
+
+    printf("ALL TESTS PASSED (%d skipped)\n", skips);
+    return 0;
+}
diff --git a/test.sh b/test.sh
new file mode 100755
index 000000000..c72bcb0dc
--- /dev/null
+++ b/test.sh
@@ -0,0 +1,78 @@
+#!/bin/sh -ue
+
+REDIS_SERVER=${REDIS_SERVER:-redis-server}
+REDIS_PORT=${REDIS_PORT:-56379}
+REDIS_SSL_PORT=${REDIS_SSL_PORT:-56443}
+TEST_SSL=${TEST_SSL:-0}
+SKIPS_AS_FAILS=${SKIPS_AS_FAILS-:0}
+SSL_TEST_ARGS=
+SKIPS_ARG=
+
+tmpdir=$(mktemp -d)
+PID_FILE=${tmpdir}/hiredis-test-redis.pid
+SOCK_FILE=${tmpdir}/hiredis-test-redis.sock
+
+if [ "$TEST_SSL" = "1" ]; then
+    SSL_CA_CERT=${tmpdir}/ca.crt
+    SSL_CA_KEY=${tmpdir}/ca.key
+    SSL_CERT=${tmpdir}/redis.crt
+    SSL_KEY=${tmpdir}/redis.key
+
+    openssl genrsa -out ${tmpdir}/ca.key 4096
+    openssl req \
+        -x509 -new -nodes -sha256 \
+        -key ${SSL_CA_KEY} \
+        -days 3650 \
+        -subj '/CN=Hiredis Test CA' \
+        -out ${SSL_CA_CERT}
+    openssl genrsa -out ${SSL_KEY} 2048
+    openssl req \
+        -new -sha256 \
+        -key ${SSL_KEY} \
+        -subj '/CN=Hiredis Test Cert' | \
+        openssl x509 \
+            -req -sha256 \
+            -CA ${SSL_CA_CERT} \
+            -CAkey ${SSL_CA_KEY} \
+            -CAserial ${tmpdir}/ca.txt \
+            -CAcreateserial \
+            -days 365 \
+            -out ${SSL_CERT}
+
+    SSL_TEST_ARGS="--ssl-host 127.0.0.1 --ssl-port ${REDIS_SSL_PORT} --ssl-ca-cert ${SSL_CA_CERT} --ssl-cert ${SSL_CERT} --ssl-key ${SSL_KEY}"
+fi
+
+cleanup() {
+  set +e
+  kill $(cat ${PID_FILE})
+  rm -rf ${tmpdir}
+}
+trap cleanup INT TERM EXIT
+
+cat > ${tmpdir}/redis.conf <<EOF
+daemonize yes
+pidfile ${PID_FILE}
+port ${REDIS_PORT}
+bind 127.0.0.1
+unixsocket ${SOCK_FILE}
+EOF
+
+if [ "$TEST_SSL" = "1" ]; then
+    cat >> ${tmpdir}/redis.conf <<EOF
+tls-port ${REDIS_SSL_PORT}
+tls-ca-cert-file ${SSL_CA_CERT}
+tls-cert-file ${SSL_CERT}
+tls-key-file ${SSL_KEY}
+EOF
+fi
+
+cat ${tmpdir}/redis.conf
+${REDIS_SERVER} ${tmpdir}/redis.conf
+
+# Wait until we detect the unix socket
+while [ ! -S "${SOCK_FILE}" ]; do sleep 1; done
+
+# Treat skips as failures if directed
+[ "$SKIPS_AS_FAILS" = 1 ] && SKIPS_ARG="--skips-as-fails"
+
+${TEST_PREFIX:-} ./hiredis-test -h 127.0.0.1 -p ${REDIS_PORT} -s ${SOCK_FILE} ${SSL_TEST_ARGS} ${SKIPS_ARG}
diff --git a/win32.h b/win32.h
new file mode 100644
index 000000000..04289c696
--- /dev/null
+++ b/win32.h
@@ -0,0 +1,56 @@
+#ifndef _WIN32_HELPER_INCLUDE
+#define _WIN32_HELPER_INCLUDE
+#ifdef _MSC_VER
+
+#include <winsock2.h> /* for struct timeval */
+
+#ifndef inline
+#define inline __inline
+#endif
+
+#ifndef strcasecmp
+#define strcasecmp stricmp
+#endif
+
+#ifndef strncasecmp
+#define strncasecmp strnicmp
+#endif
+
+#ifndef va_copy
+#define va_copy(d,s) ((d) = (s))
+#endif
+
+#ifndef snprintf
+#define snprintf c99_snprintf
+
+__inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap)
+{
+    int count = -1;
+
+    if (size != 0)
+        count = _vsnprintf_s(str, size, _TRUNCATE, format, ap);
+    if (count == -1)
+        count = _vscprintf(format, ap);
+
+    return count;
+}
+
+__inline int c99_snprintf(char* str, size_t size, const char* format, ...)
+{
+    int count;
+    va_list ap;
+
+    va_start(ap, format);
+    count = c99_vsnprintf(str, size, format, ap);
+    va_end(ap);
+
+    return count;
+}
+#endif
+#endif /* _MSC_VER */
+
+#ifdef _WIN32
+#define strerror_r(errno,buf,len) strerror_s(buf,len,errno)
+#endif /* _WIN32 */
+
+#endif /* _WIN32_HELPER_INCLUDE */

From e966264188e2bd9d09018a2b62354d5227db8b07 Mon Sep 17 00:00:00 2001
From: fayadexinqing <2418967201@qq.com>
Date: Sat, 8 Aug 2020 04:01:14 +0800
Subject: [PATCH 0546/1098] fix migration's broadcast PONG message, after the
 slot modification (#7590)

---
 src/cluster.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 8860bfc14..3bf8db9af 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4503,6 +4503,9 @@ NULL
                 server.cluster->migrating_slots_to[slot])
                 server.cluster->migrating_slots_to[slot] = NULL;
 
+            clusterDelSlot(slot);
+            clusterAddSlot(n,slot);
+
             /* If this node was importing this slot, assigning the slot to
              * itself also clears the importing status. */
             if (n == myself &&
@@ -4526,8 +4529,6 @@ NULL
                  * soon as possible. */
                 clusterBroadcastPong(CLUSTER_BROADCAST_ALL);
             }
-            clusterDelSlot(slot);
-            clusterAddSlot(n,slot);
         } else {
             addReplyError(c,
                 "Invalid CLUSTER SETSLOT action or number of arguments. Try CLUSTER HELP");

From b118502a053061347b0aa25c506bf8961013f8b6 Mon Sep 17 00:00:00 2001
From: xuannianz <47107252+xuannianz@users.noreply.github.com>
Date: Sat, 8 Aug 2020 05:19:18 +0800
Subject: [PATCH 0547/1098] remove superfluous else block (#7620)

The else block would be executed when newlen == 0 and in the case memmove won't be called, so there's no need to set start.
---
 src/sds.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/sds.c b/src/sds.c
index 118971621..b13bf3467 100644
--- a/src/sds.c
+++ b/src/sds.c
@@ -756,8 +756,6 @@ void sdsrange(sds s, ssize_t start, ssize_t end) {
             end = len-1;
             newlen = (start > end) ? 0 : (end-start)+1;
         }
-    } else {
-        start = 0;
     }
     if (start && newlen) memmove(s, s+start, newlen);
     s[newlen] = 0;

From 1ef014ee6b0ca385451cc74e3cb2da9048975c7e Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuan21@baidu.com>
Date: Sat, 8 Aug 2020 19:36:41 +0800
Subject: [PATCH 0548/1098] Fix applying zero offset to null pointer when
 creating moduleFreeContextReusedClient (#7323)

Before this fix we where attempting to select a db before creating db the DB, see: #7323

This issue doesn't seem to have any implications, since the selected DB index is 0,
the db pointer remains NULL, and will later be correctly set before using this dummy
client for the first time.

As we know, we call 'moduleInitModulesSystem()' before 'initServer()'. We will allocate
memory for server.db in 'initServer', but we call 'createClient()' that will call 'selectDb()'
in 'moduleInitModulesSystem()', before the databases where created. Instead, we should call
'createClient()' for moduleFreeContextReusedClient after 'initServer()'.
---
 src/module.c | 15 +++++++++++----
 src/server.c |  1 +
 src/server.h |  1 +
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/module.c b/src/module.c
index b008929f7..e0309ae6b 100644
--- a/src/module.c
+++ b/src/module.c
@@ -283,7 +283,7 @@ typedef struct RedisModuleKeyspaceSubscriber {
 static list *moduleKeyspaceSubscribers;
 
 /* Static client recycled for when we need to provide a context with a client
- * in a situation where there is no client to provide. This avoidsallocating
+ * in a situation where there is no client to provide. This avoids allocating
  * a new client per round. For instance this is used in the keyspace
  * notifications, timers and cluster messages callbacks. */
 static client *moduleFreeContextReusedClient;
@@ -7249,6 +7249,16 @@ int moduleRegisterApi(const char *funcname, void *funcptr) {
 /* Global initialization at Redis startup. */
 void moduleRegisterCoreAPI(void);
 
+/* Some steps in module initialization need to be done last after server
+ * initialization.
+ * For example, selectDb() in createClient() requires that server.db has
+ * been initialized, see #7323. */
+void moduleInitModulesSystemLast(void) {
+    moduleFreeContextReusedClient = createClient(NULL);
+    moduleFreeContextReusedClient->flags |= CLIENT_MODULE;
+    moduleFreeContextReusedClient->user = NULL; /* root user. */
+}
+
 void moduleInitModulesSystem(void) {
     moduleUnblockedClients = listCreate();
     server.loadmodule_queue = listCreate();
@@ -7256,9 +7266,6 @@ void moduleInitModulesSystem(void) {
 
     /* Set up the keyspace notification susbscriber list and static client */
     moduleKeyspaceSubscribers = listCreate();
-    moduleFreeContextReusedClient = createClient(NULL);
-    moduleFreeContextReusedClient->flags |= CLIENT_MODULE;
-    moduleFreeContextReusedClient->user = NULL; /* root user. */
 
     /* Set up filter list */
     moduleCommandFilters = listCreate();
diff --git a/src/server.c b/src/server.c
index c9d995e0b..b23aa3cf0 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5205,6 +5205,7 @@ int main(int argc, char **argv) {
     #ifdef __linux__
         linuxMemoryWarnings();
     #endif
+        moduleInitModulesSystemLast();
         moduleLoadFromQueue();
         ACLLoadUsersAtStartup();
         InitServerLast();
diff --git a/src/server.h b/src/server.h
index 71bbfec7b..19aab8750 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1572,6 +1572,7 @@ extern dictType modulesDictType;
 
 /* Modules */
 void moduleInitModulesSystem(void);
+void moduleInitModulesSystemLast(void);
 int moduleLoad(const char *path, void **argv, int argc);
 void moduleLoadFromQueue(void);
 int *moduleGetCommandKeysViaAPI(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);

From 3f67b0337833cbd2a2746dad7cd8411a31b75f70 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Sat, 8 Aug 2020 07:42:32 -0400
Subject: [PATCH 0549/1098] fix memory leak in ACLLoadFromFile error handling
 (#7623)

---
 src/acl.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/acl.c b/src/acl.c
index 3ce45f03b..e0432ba5c 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1327,6 +1327,7 @@ sds ACLLoadFromFile(const char *filename) {
             errors = sdscatprintf(errors,
                      "'%s:%d: username '%s' contains invalid characters. ",
                      server.acl_filename, linenum, argv[1]);
+            sdsfreesplitres(argv,argc);
             continue;
         }
 

From 6a06a5a597b349be4cf587e59613bbf4cd2f2c43 Mon Sep 17 00:00:00 2001
From: ShooterIT <wangyuancode@163.com>
Date: Sat, 8 Aug 2020 23:08:27 +0800
Subject: [PATCH 0550/1098] [Redis-benchmark] Remove zrem test, add zpopmin
 test

---
 src/redis-benchmark.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index ef43275b9..9ba381409 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -1433,7 +1433,8 @@ usage:
 " --cluster          Enable cluster mode.\n"
 " --enable-tracking  Send CLIENT TRACKING on before starting benchmark.\n"
 " -k <boolean>       1=keep alive 0=reconnect (default 1)\n"
-" -r <keyspacelen>   Use random keys for SET/GET/INCR, random values for SADD\n"
+" -r <keyspacelen>   Use random keys for SET/GET/INCR, random values for SADD,\n"
+"                    random members and scores for ZADD.\n"
 "  Using this option the benchmark will expand the string __rand_int__\n"
 "  inside an argument with a 12 digits number in the specified range\n"
 "  from 0 to keyspacelen-1. The substitution changes every time a command\n"
@@ -1742,10 +1743,9 @@ int main(int argc, const char **argv) {
             free(cmd);
         }
 
-        if (test_is_selected("zrem")) {
-            len = redisFormatCommand(&cmd,
-                "ZREM myzset:{tag} element:__rand_int__");
-            benchmark("ZREM",cmd,len);
+        if (test_is_selected("zpopmin")) {
+            len = redisFormatCommand(&cmd,"ZPOPMIN myzset:{tag}");
+            benchmark("ZPOPMIN",cmd,len);
             free(cmd);
         }
 

From 81879bc171ed8726f8c0bba1aa123862ee2a08fd Mon Sep 17 00:00:00 2001
From: michael-grunder <michael.grunder@gmail.com>
Date: Thu, 6 Aug 2020 12:43:56 -0700
Subject: [PATCH 0551/1098] Create PUSH handlers in redis-cli

Add logic to redis-cli to display RESP3 PUSH messages when we detect
STDOUT is a tty, with an optional command-line argument to override
the default behavior.

The new argument:  --show-pushes <yn>

Examples:

$ redis-cli -3 --show-pushes no
$ echo "client tracking on\nget k1\nset k1 v1"| redis-cli -3 --show-pushes y
---
 src/redis-cli.c | 117 +++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 101 insertions(+), 16 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index de5d08149..09f95b507 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -234,6 +234,7 @@ static struct config {
     int askpass;
     char *user;
     int output; /* output mode, see OUTPUT_* defines */
+    int push_output; /* Should we display spontaneous PUSH replies */
     sds mb_delim;
     char prompt[128];
     char *eval;
@@ -267,6 +268,8 @@ static long getLongInfoField(char *info, char *field);
  * Utility functions
  *--------------------------------------------------------------------------- */
 
+static void cliPushHandler(void *, void *);
+
 uint16_t crc16(const char *buf, int len);
 
 static long long ustime(void) {
@@ -873,8 +876,8 @@ static int cliConnect(int flags) {
             const char *err = NULL;
             if (cliSecureConnection(context, &err) == REDIS_ERR && err) {
                 fprintf(stderr, "Could not negotiate a TLS connection: %s\n", err);
-                context = NULL;
                 redisFree(context);
+                context = NULL;
                 return REDIS_ERR;
             }
         }
@@ -909,6 +912,12 @@ static int cliConnect(int flags) {
         if (cliSwitchProto() != REDIS_OK)
             return REDIS_ERR;
     }
+
+    /* Set a PUSH handler if configured to do so. */
+    if (config.push_output) {
+        redisSetPushCallback(context, cliPushHandler);
+    }
+
     return REDIS_OK;
 }
 
@@ -917,6 +926,31 @@ static void cliPrintContextError(void) {
     fprintf(stderr,"Error: %s\n",context->errstr);
 }
 
+static int isInvalidateReply(redisReply *reply) {
+    return reply->type == REDIS_REPLY_PUSH && reply->elements == 2 &&
+        reply->element[0]->type == REDIS_REPLY_STRING &&
+        !strncmp(reply->element[0]->str, "invalidate", 10) &&
+        reply->element[1]->type == REDIS_REPLY_ARRAY;
+}
+
+/* Special display handler for RESP3 'invalidate' messages.
+ * This function does not validate the reply, so it should
+ * already be confirmed correct */
+static sds cliFormatInvalidateTTY(redisReply *r) {
+    sds out = sdsnew("-> invalidate: ");
+
+    for (size_t i = 0; i < r->element[1]->elements; i++) {
+        redisReply *key = r->element[1]->element[i];
+        assert(key->type == REDIS_REPLY_STRING);
+
+        out = sdscatfmt(out, "'%s'", key->str, key->len);
+        if (i < r->element[1]->elements - 1)
+            out = sdscatlen(out, ", ", 2);
+    }
+
+    return sdscatlen(out, "\n", 1);
+}
+
 static sds cliFormatReplyTTY(redisReply *r, char *prefix) {
     sds out = sdsempty();
     switch (r->type) {
@@ -955,6 +989,7 @@ static sds cliFormatReplyTTY(redisReply *r, char *prefix) {
     case REDIS_REPLY_ARRAY:
     case REDIS_REPLY_MAP:
     case REDIS_REPLY_SET:
+    case REDIS_REPLY_PUSH:
         if (r->elements == 0) {
             if (r->type == REDIS_REPLY_ARRAY)
                 out = sdscat(out,"(empty array)\n");
@@ -962,6 +997,8 @@ static sds cliFormatReplyTTY(redisReply *r, char *prefix) {
                 out = sdscat(out,"(empty hash)\n");
             else if (r->type == REDIS_REPLY_SET)
                 out = sdscat(out,"(empty set)\n");
+            else if (r->type == REDIS_REPLY_PUSH)
+                out = sdscat(out,"(empty push)\n");
             else
                 out = sdscat(out,"(empty aggregate type)\n");
         } else {
@@ -1113,6 +1150,7 @@ static sds cliFormatReplyRaw(redisReply *r) {
         out = sdscatprintf(out,"%s",r->str);
         break;
     case REDIS_REPLY_ARRAY:
+    case REDIS_REPLY_PUSH:
         for (i = 0; i < r->elements; i++) {
             if (i > 0) out = sdscat(out,config.mb_delim);
             tmp = cliFormatReplyRaw(r->element[i]);
@@ -1169,6 +1207,7 @@ static sds cliFormatReplyCSV(redisReply *r) {
         out = sdscat(out,r->integer ? "true" : "false");
     break;
     case REDIS_REPLY_ARRAY:
+    case REDIS_REPLY_PUSH:
     case REDIS_REPLY_MAP: /* CSV has no map type, just output flat list. */
         for (i = 0; i < r->elements; i++) {
             sds tmp = cliFormatReplyCSV(r->element[i]);
@@ -1184,6 +1223,45 @@ static sds cliFormatReplyCSV(redisReply *r) {
     return out;
 }
 
+/* Generate reply strings in various output modes */
+static sds cliFormatReply(redisReply *reply, int mode, int verbatim) {
+    sds out;
+
+    if (verbatim) {
+        out = cliFormatReplyRaw(reply);
+    }  else if (mode == OUTPUT_STANDARD) {
+        out = cliFormatReplyTTY(reply, "");
+    } else if (mode == OUTPUT_RAW) {
+        out = cliFormatReplyRaw(reply);
+        out = sdscatlen(out, "\n", 1);
+    } else if (mode == OUTPUT_CSV) {
+        out = cliFormatReplyCSV(reply);
+        out = sdscatlen(out, "\n", 1);
+    } else {
+        fprintf(stderr, "Error:  Unknown output encoding %d\n", mode);
+        exit(1);
+    }
+
+    return out;
+}
+
+/* Output any spontaneous PUSH reply we receive */
+static void cliPushHandler(void *privdata, void *reply) {
+    UNUSED(privdata);
+    sds out;
+
+    if (config.output == OUTPUT_STANDARD && isInvalidateReply(reply)) {
+        out = cliFormatInvalidateTTY(reply);
+    } else {
+        out = cliFormatReply(reply, config.output, 0);
+    }
+
+    fwrite(out, sdslen(out), 1, stdout);
+
+    freeReplyObject(reply);
+    sdsfree(out);
+}
+
 static int cliReadReply(int output_raw_strings) {
     void *_reply;
     redisReply *reply;
@@ -1244,19 +1322,7 @@ static int cliReadReply(int output_raw_strings) {
     }
 
     if (output) {
-        if (output_raw_strings) {
-            out = cliFormatReplyRaw(reply);
-        } else {
-            if (config.output == OUTPUT_RAW) {
-                out = cliFormatReplyRaw(reply);
-                out = sdscat(out,"\n");
-            } else if (config.output == OUTPUT_STANDARD) {
-                out = cliFormatReplyTTY(reply,"");
-            } else if (config.output == OUTPUT_CSV) {
-                out = cliFormatReplyCSV(reply);
-                out = sdscat(out,"\n");
-            }
-        }
+        out = cliFormatReply(reply, config.output, output_raw_strings);
         fwrite(out,sdslen(out),1,stdout);
         sdsfree(out);
     }
@@ -1346,6 +1412,10 @@ static int cliSendCommand(int argc, char **argv, long repeat) {
         if (config.pubsub_mode) {
             if (config.output != OUTPUT_RAW)
                 printf("Reading messages... (press Ctrl-C to quit)\n");
+
+            /* Unset our default PUSH handler so this works in RESP2/RESP3 */
+            redisSetPushCallback(context, NULL);
+
             while (1) {
                 if (cliReadReply(output_raw) != REDIS_OK) exit(1);
             }
@@ -1626,6 +1696,16 @@ static int parseOptions(int argc, char **argv) {
             exit(0);
         } else if (!strcmp(argv[i],"-3")) {
             config.resp3 = 1;
+        } else if (!strcmp(argv[i],"--show-pushes") && !lastarg) {
+            char *argval = argv[++i];
+            if (!strncasecmp(argval, "n", 1)) {
+                config.push_output = 0;
+            } else if (!strncasecmp(argval, "y", 1)) {
+                config.push_output = 1;
+            } else {
+                fprintf(stderr, "Unknown --show-pushes value '%s' "
+                        "(valid: '[y]es', '[n]o')\n", argval);
+            }
         } else if (CLUSTER_MANAGER_MODE() && argv[i][0] != '-') {
             if (config.cluster_manager_command.argc == 0) {
                 int j = i + 1;
@@ -1734,6 +1814,8 @@ static void usage(void) {
 "                     not a tty).\n"
 "  --no-raw           Force formatted output even when STDOUT is not a tty.\n"
 "  --csv              Output in CSV format.\n"
+"  --show-pushes <yn> Whether to print RESP3 PUSH messages.  Enabled by default when\n"
+"                     STDOUT is a tty but can be overriden with --show-pushes no.\n"
 "  --stat             Print rolling stats about server: mem, clients, ...\n"
 "  --latency          Enter a special mode continuously sampling latency.\n"
 "                     If you use this mode in an interactive session it runs\n"
@@ -8063,10 +8145,13 @@ int main(int argc, char **argv) {
     spectrum_palette = spectrum_palette_color;
     spectrum_palette_size = spectrum_palette_color_size;
 
-    if (!isatty(fileno(stdout)) && (getenv("FAKETTY") == NULL))
+    if (!isatty(fileno(stdout)) && (getenv("FAKETTY") == NULL)) {
         config.output = OUTPUT_RAW;
-    else
+        config.push_output = 0;
+    } else {
         config.output = OUTPUT_STANDARD;
+        config.push_output = 1;
+    }
     config.mb_delim = sdsnew("\n");
 
     firstarg = parseOptions(argc,argv);

From ea7eeb2fd2e7a73d951f394d5d14a5ba5a75d4b2 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuan21@baidu.com>
Date: Sun, 9 Aug 2020 03:03:56 +0800
Subject: [PATCH 0552/1098] Print error info if failed opening config file
 (#6943)

---
 src/config.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/config.c b/src/config.c
index 52a859a77..b94d1c9f4 100644
--- a/src/config.c
+++ b/src/config.c
@@ -556,7 +556,8 @@ void loadServerConfig(char *filename, char *options) {
         } else {
             if ((fp = fopen(filename,"r")) == NULL) {
                 serverLog(LL_WARNING,
-                    "Fatal error, can't open config file '%s'", filename);
+                    "Fatal error, can't open config file '%s': %s",
+                    filename, strerror(errno));
                 exit(1);
             }
         }

From 6dcc6898beaadac39a8eb758cbf27e2aa5386831 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Sun, 9 Aug 2020 03:11:14 +0800
Subject: [PATCH 0553/1098] Optimize calls to mstime in
 trackInstantaneousMetric() (#6472)

---
 src/server.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/server.c b/src/server.c
index b23aa3cf0..15b962a2e 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1493,7 +1493,8 @@ int allPersistenceDisabled(void) {
 
 /* Add a sample to the operations per second array of samples. */
 void trackInstantaneousMetric(int metric, long long current_reading) {
-    long long t = mstime() - server.inst_metric[metric].last_sample_time;
+    long long now = mstime();
+    long long t = now - server.inst_metric[metric].last_sample_time;
     long long ops = current_reading -
                     server.inst_metric[metric].last_sample_count;
     long long ops_sec;
@@ -1504,7 +1505,7 @@ void trackInstantaneousMetric(int metric, long long current_reading) {
         ops_sec;
     server.inst_metric[metric].idx++;
     server.inst_metric[metric].idx %= STATS_METRIC_SAMPLES;
-    server.inst_metric[metric].last_sample_time = mstime();
+    server.inst_metric[metric].last_sample_time = now;
     server.inst_metric[metric].last_sample_count = current_reading;
 }
 

From ca95b71f67f72e41b05dc155ae015f1ad1f16b4c Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Sat, 8 Aug 2020 15:28:44 -0400
Subject: [PATCH 0554/1098] add sentinel command help (#7579)

---
 src/sentinel.c | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/src/sentinel.c b/src/sentinel.c
index 5bd594955..d93b26131 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -3044,7 +3044,30 @@ int sentinelIsQuorumReachable(sentinelRedisInstance *master, int *usableptr) {
 }
 
 void sentinelCommand(client *c) {
-    if (!strcasecmp(c->argv[1]->ptr,"masters")) {
+    if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
+        const char *help[] = {
+"MASTERS -- Show a list of monitored masters and their state.",
+"MASTER <master-name> -- Show the state and info of the specified master.",
+"REPLICAS <master-name> -- Show a list of replicas for this master and their state.",
+"SENTINELS <master-name> -- Show a list of Sentinel instances for this master and their state.",
+"IS-MASTER-DOWN-BY-ADDR <ip> <port> <current-epoch> <runid> -- Check if the master specified by ip:port is down from current Sentinel's point of view.",
+"GET-MASTER-ADDR-BY-NAME <master-name> -- Return the ip and port number of the master with that name.",
+"RESET <pattern> -- Reset masters for specific master name matching this pattern.",
+"FAILOVER <master-name> -- Manually failover a master node without asking for agreement from other Sentinels",
+"PENDING-SCRIPTS -- Get pending scripts information.", 
+"MONITOR <name> <ip> <port> <quorum> -- Start monitoring a new master with the specified name, ip, port and quorum.",
+"FLUSHCONFIG -- Force Sentinel to rewrite its configuration on disk, including the current Sentinel state.",
+"REMOVE <master-name> -- Remove master from Sentinel's monitor list.",
+"CKQUORUM <master-name> -- Check if the current Sentinel configuration is able to reach the quorum needed to failover a master "
+"and the majority needed to authorize the failover.",
+"SET <master-name> <option> <value> -- Set configuration paramters for certain masters.",
+"INFO-CACHE <master-name> -- Return last cached INFO output from masters and all its replicas.",
+"SIMULATE-FAILURE (crash-after-election|crash-after-promotion|help) -- Simulate a Sentinel crash.",
+"HELP -- Prints this help.",
+NULL
+        };
+        addReplyHelp(c, help);
+    } else if (!strcasecmp(c->argv[1]->ptr,"masters")) {
         /* SENTINEL MASTERS */
         if (c->argc != 2) goto numargserr;
         addReplyDictOfRedisInstances(c,sentinel.masters);
@@ -3353,8 +3376,7 @@ void sentinelCommand(client *c) {
         }
         addReply(c,shared.ok);
     } else {
-        addReplyErrorFormat(c,"Unknown sentinel subcommand '%s'",
-                               (char*)c->argv[1]->ptr);
+        addReplySubcommandSyntaxError(c);
     }
     return;
 

From e2d64485b8262971776fb1be803c7296c98d1572 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 9 Aug 2020 06:08:00 +0300
Subject: [PATCH 0555/1098] Reduce the probability of failure when start redis
 in runtest-cluster #7554 (#7635)

When runtest-cluster, at first, we need to create a cluster use spawn_instance,
a port which is not used is choosen, however sometimes we can't run server on
the port. possibley due to a race with another process taking it first.
such as redis/redis/runs/896537490. It may be due to the machine problem or
In order to reduce the probability of failure when start redis in
runtest-cluster, we attemp to use another port when find server do not start up.

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: yanhui13 <yanhui13@meituan.com>
---
 tests/instances.tcl | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/tests/instances.tcl b/tests/instances.tcl
index 677af6427..e2aa4ab13 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -59,8 +59,6 @@ proc exec_instance {type cfgfile} {
 proc spawn_instance {type base_port count {conf {}}} {
     for {set j 0} {$j < $count} {incr j} {
         set port [find_available_port $base_port $::redis_port_count]
-        incr base_port
-        puts "Starting $type #$j at port $port"
 
         # Create a directory for this instance.
         set dirname "${type}_${j}"
@@ -93,10 +91,30 @@ proc spawn_instance {type base_port count {conf {}}} {
         close $cfg
 
         # Finally exec it and remember the pid for later cleanup.
-        set pid [exec_instance $type $cfgfile]
-        lappend ::pids $pid
+        set retry 100
+        while {$retry} {
+            set pid [exec_instance $type $cfgfile]
 
-        # Check availability
+            # Check availability
+            if {[server_is_up 127.0.0.1 $port 100] == 0} {
+                puts "Starting $type #$j at port $port failed, try another"
+                incr retry -1
+                set port [find_available_port $base_port $::redis_port_count]
+                set cfg [open $cfgfile a+]
+                if {$::tls} {
+                    puts $cfg "tls-port $port"
+                } else {
+                    puts $cfg "port $port"
+                }
+                close $cfg
+            } else {
+                puts "Starting $type #$j at port $port"
+                lappend ::pids $pid
+                break
+            }
+        }
+
+        # Check availability finally
         if {[server_is_up 127.0.0.1 $port 100] == 0} {
             abort_sentinel_test "Problems starting $type #$j: ping timeout"
         }

From 3f494cc49d25929f27fa75a78d9921a9dee771f2 Mon Sep 17 00:00:00 2001
From: "Meir Shpilraien (Spielrein)" <meir@redislabs.com>
Date: Sun, 9 Aug 2020 06:11:47 +0300
Subject: [PATCH 0556/1098] see #7544, added RedisModule_HoldString api.
 (#7577)

Added RedisModule_HoldString that either returns a
shallow copy of the given String (by increasing
the String ref count) or a new deep copy of String
in case its not possible to get a shallow copy.

Co-authored-by: Itamar Haber <itamar@redislabs.com>
---
 src/module.c                             | 60 ++++++++++++++++++++++++
 src/redismodule.h                        |  2 +
 tests/modules/keyspace_events.c          | 17 ++++++-
 tests/unit/moduleapi/keyspace_events.tcl | 12 ++---
 4 files changed, 83 insertions(+), 8 deletions(-)

diff --git a/src/module.c b/src/module.c
index e0309ae6b..8e9526dad 100644
--- a/src/module.c
+++ b/src/module.c
@@ -1138,6 +1138,65 @@ void RM_RetainString(RedisModuleCtx *ctx, RedisModuleString *str) {
     }
 }
 
+/**
+* This function can be used instead of RedisModule_RetainString().
+* The main difference between the two is that this function will always
+* succeed, whereas RedisModule_RetainString() may fail because of an
+* assertion.
+* 
+* The function returns a pointer to RedisModuleString, which is owned
+* by the caller. It requires a call to RedisModule_FreeString() to free
+* the string when automatic memory management is disabled for the context.
+* When automatic memory management is enabled, you can either call
+* RedisModule_FreeString() or let the automation free it.
+* 
+* This function is more efficient than RedisModule_CreateStringFromString()
+* because whenever possible, it avoids copying the underlying
+* RedisModuleString. The disadvantage of using this function is that it
+* might not be possible to use RedisModule_StringAppendBuffer() on the
+* returned RedisModuleString.
+* 
+* It is possible to call this function with a NULL context.
+ */
+RedisModuleString* RM_HoldString(RedisModuleCtx *ctx, RedisModuleString *str) {
+    if (str->refcount == OBJ_STATIC_REFCOUNT) {
+        return RM_CreateStringFromString(ctx, str);
+    }
+
+    incrRefCount(str);
+    if (ctx != NULL) {
+        /*
+         * Put the str in the auto memory management of the ctx.
+         * It might already be there, in this case, the ref count will
+         * be 2 and we will decrease the ref count twice and free the
+         * object in the auto memory free function.
+         *
+         * Why we can not do the same trick of just remove the object
+         * from the auto memory (like in RM_RetainString)?
+         * This code shows the issue:
+         *
+         * RM_AutoMemory(ctx);
+         * str1 = RM_CreateString(ctx, "test", 4);
+         * str2 = RM_HoldString(ctx, str1);
+         * RM_FreeString(str1);
+         * RM_FreeString(str2);
+         *
+         * If after the RM_HoldString we would just remove the string from
+         * the auto memory, this example will cause access to a freed memory
+         * on 'RM_FreeString(str2);' because the String will be free
+         * on 'RM_FreeString(str1);'.
+         *
+         * So it's safer to just increase the ref count
+         * and add the String to auto memory again.
+         *
+         * The limitation is that it is not possible to use RedisModule_StringAppendBuffer
+         * on the String.
+         */
+        autoMemoryAdd(ctx,REDISMODULE_AM_STRING,str);
+    }
+    return str;
+}
+
 /* Given a string module object, this function returns the string pointer
  * and length of the string. The returned pointer and length should only
  * be used for read only accesses and never modified. */
@@ -7832,6 +7891,7 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(LatencyAddSample);
     REGISTER_API(StringAppendBuffer);
     REGISTER_API(RetainString);
+    REGISTER_API(HoldString);
     REGISTER_API(StringCompare);
     REGISTER_API(GetContextFromIO);
     REGISTER_API(GetKeyNameFromIO);
diff --git a/src/redismodule.h b/src/redismodule.h
index 2599d3129..2f2213677 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -569,6 +569,7 @@ void REDISMODULE_API_FUNC(RedisModule__Assert)(const char *estr, const char *fil
 void REDISMODULE_API_FUNC(RedisModule_LatencyAddSample)(const char *event, mstime_t latency);
 int REDISMODULE_API_FUNC(RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len);
 void REDISMODULE_API_FUNC(RedisModule_RetainString)(RedisModuleCtx *ctx, RedisModuleString *str);
+RedisModuleString* REDISMODULE_API_FUNC(RedisModule_HoldString)(RedisModuleCtx *ctx, RedisModuleString *str);
 int REDISMODULE_API_FUNC(RedisModule_StringCompare)(RedisModuleString *a, RedisModuleString *b);
 RedisModuleCtx *REDISMODULE_API_FUNC(RedisModule_GetContextFromIO)(RedisModuleIO *io);
 const RedisModuleString *REDISMODULE_API_FUNC(RedisModule_GetKeyNameFromIO)(RedisModuleIO *io);
@@ -807,6 +808,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(LatencyAddSample);
     REDISMODULE_GET_API(StringAppendBuffer);
     REDISMODULE_GET_API(RetainString);
+    REDISMODULE_GET_API(HoldString);
     REDISMODULE_GET_API(StringCompare);
     REDISMODULE_GET_API(GetContextFromIO);
     REDISMODULE_GET_API(GetKeyNameFromIO);
diff --git a/tests/modules/keyspace_events.c b/tests/modules/keyspace_events.c
index b2296c1cb..db3977be1 100644
--- a/tests/modules/keyspace_events.c
+++ b/tests/modules/keyspace_events.c
@@ -48,7 +48,7 @@ static int KeySpace_Notification(RedisModuleCtx *ctx, int type, const char *even
         int nokey;
         RedisModule_DictGetC(loaded_event_log, (void*)keyName, strlen(keyName), &nokey);
         if(nokey){
-            RedisModule_DictSetC(loaded_event_log, (void*)keyName, strlen(keyName), NULL);
+            RedisModule_DictSetC(loaded_event_log, (void*)keyName, strlen(keyName), RedisModule_HoldString(ctx, key));
         }
     }
 
@@ -63,9 +63,15 @@ static int cmdIsKeyLoaded(RedisModuleCtx *ctx, RedisModuleString **argv, int arg
     const char* key  = RedisModule_StringPtrLen(argv[1], NULL);
 
     int nokey;
-    RedisModule_DictGetC(loaded_event_log, (void*)key, strlen(key), &nokey);
+    RedisModuleString* keyStr = RedisModule_DictGetC(loaded_event_log, (void*)key, strlen(key), &nokey);
 
+    RedisModule_ReplyWithArray(ctx, 2);
     RedisModule_ReplyWithLongLong(ctx, !nokey);
+    if(nokey){
+        RedisModule_ReplyWithNull(ctx);
+    }else{
+        RedisModule_ReplyWithString(ctx, keyStr);
+    }
     return REDISMODULE_OK;
 }
 
@@ -93,6 +99,13 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
 }
 
 int RedisModule_OnUnload(RedisModuleCtx *ctx) {
+    RedisModuleDictIter *iter = RedisModule_DictIteratorStartC(loaded_event_log, "^", NULL, 0);
+    char* key;
+    size_t keyLen;
+    RedisModuleString* val;
+    while((key = RedisModule_DictNextC(iter, &keyLen, (void**)&val))){
+        RedisModule_FreeString(ctx, val);
+    }
     RedisModule_FreeDict(ctx, loaded_event_log);
     loaded_event_log = NULL;
     return REDISMODULE_OK;
diff --git a/tests/unit/moduleapi/keyspace_events.tcl b/tests/unit/moduleapi/keyspace_events.tcl
index cb959ab52..5b3db0c0a 100644
--- a/tests/unit/moduleapi/keyspace_events.tcl
+++ b/tests/unit/moduleapi/keyspace_events.tcl
@@ -11,12 +11,12 @@ tags "modules" {
             r zadd t 1 f1 2 f2
             r xadd s * f v
             r debug reload
-            assert_equal 1 [r keyspace.is_key_loaded x]
-            assert_equal 1 [r keyspace.is_key_loaded y]
-            assert_equal 1 [r keyspace.is_key_loaded z]
-            assert_equal 1 [r keyspace.is_key_loaded p]
-            assert_equal 1 [r keyspace.is_key_loaded t]
-            assert_equal 1 [r keyspace.is_key_loaded s]
+            assert_equal {1 x} [r keyspace.is_key_loaded x]
+            assert_equal {1 y} [r keyspace.is_key_loaded y]
+            assert_equal {1 z} [r keyspace.is_key_loaded z]
+            assert_equal {1 p} [r keyspace.is_key_loaded p]
+            assert_equal {1 t} [r keyspace.is_key_loaded t]
+            assert_equal {1 s} [r keyspace.is_key_loaded s]
         }
 	}
 }
\ No newline at end of file

From 3941a275f7994bc5563b442bb45264abee7511ba Mon Sep 17 00:00:00 2001
From: Hamed Momeni <hamed.ma7@gmail.com>
Date: Sun, 9 Aug 2020 13:32:29 +0430
Subject: [PATCH 0557/1098] fixed a typo in redis.conf (#7636)

---
 redis.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/redis.conf b/redis.conf
index 8440b4750..33f15b9ee 100644
--- a/redis.conf
+++ b/redis.conf
@@ -492,7 +492,7 @@ repl-diskless-sync-delay 5
 # does not immediately store an RDB on disk, it may cause data loss during
 # failovers. RDB diskless load + Redis modules not handling I/O reads may also
 # cause Redis to abort in case of I/O errors during the initial synchronization
-# stage with the master. Use only if your do what you are doing.
+# stage with the master. Use only if you know what you are doing.
 # -----------------------------------------------------------------------------
 #
 # Replica can load the RDB it reads from the replication link directly from the

From d6220f12a9fafeb07ad86fa91e88383ca2a524c8 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Tue, 11 Aug 2020 10:18:21 +0800
Subject: [PATCH 0558/1098] see #7250, fix signature of
 RedisModule_DeauthenticateAndCloseClient (#7645)

In redismodule.h, RedisModule_DeauthenticateAndCloseClient returns void
`void REDISMODULE_API_FUNC(RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id);`
But in module.c, RM_DeauthenticateAndCloseClient returns int
`int RM_DeauthenticateAndCloseClient(RedisModuleCtx *ctx, uint64_t client_id)`

It it safe to change return value from `void` to `int` from the user's perspective.
---
 src/redismodule.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/redismodule.h b/src/redismodule.h
index 2f2213677..5f828b9e3 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -680,7 +680,7 @@ void REDISMODULE_API_FUNC(RedisModule_FreeModuleUser)(RedisModuleUser *user);
 int REDISMODULE_API_FUNC(RedisModule_SetModuleUserACL)(RedisModuleUser *user, const char* acl);
 int REDISMODULE_API_FUNC(RedisModule_AuthenticateClientWithACLUser)(RedisModuleCtx *ctx, const char *name, size_t len, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id);
 int REDISMODULE_API_FUNC(RedisModule_AuthenticateClientWithUser)(RedisModuleCtx *ctx, RedisModuleUser *user, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id);
-void REDISMODULE_API_FUNC(RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id);
+int REDISMODULE_API_FUNC(RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id);
 #endif
 
 #define RedisModule_IsAOFClient(id) ((id) == CLIENT_ID_AOF)

From efe92ee5462f65b6a4dabb3d578a31bedcad2f33 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Tue, 11 Aug 2020 08:11:47 +0300
Subject: [PATCH 0559/1098] Removes dead code (#7642)

Appears to be handled by server.stream_node_max_bytes in reality.
---
 src/t_stream.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 79265a6fb..38bdce8e0 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -31,8 +31,6 @@
 #include "endianconv.h"
 #include "stream.h"
 
-#define STREAM_BYTES_PER_LISTPACK 2048
-
 /* Every stream item inside the listpack, has a flags field that is used to
  * mark the entry as deleted, or having the same field as the "master"
  * entry at the start of the listpack> */
@@ -199,7 +197,7 @@ int streamCompareID(streamID *a, streamID *b) {
  * C_ERR if an ID was given via 'use_id', but adding it failed since the
  * current top ID is greater or equal. */
 int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_id, streamID *use_id) {
-    
+
     /* Generate the new entry ID. */
     streamID id;
     if (use_id)
@@ -209,7 +207,7 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_
 
     /* Check that the new ID is greater than the last entry ID
      * or return an error. Automatically generated IDs might
-     * overflow (and wrap-around) when incrementing the sequence 
+     * overflow (and wrap-around) when incrementing the sequence
        part. */
     if (streamCompareID(&id,&s->last_id) <= 0) return C_ERR;
 

From 229327ad8be5cd87ead56884228dfb08b6017c45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9D=A8=E5=8D=9A=E4=B8=9C?= <yangbodong22011@gmail.com>
Date: Tue, 11 Aug 2020 13:18:09 +0800
Subject: [PATCH 0560/1098] Avoid redundant calls to signalKeyAsReady (#7625)

signalKeyAsReady has some overhead (namely dictFind) so we should
only call it when there are clients blocked on the relevant type (BLOCKED_*)
---
 src/blocked.c  | 18 ++++++++++++++++--
 src/db.c       | 10 ++--------
 src/module.c   |  2 +-
 src/server.h   |  2 +-
 src/t_stream.c |  5 ++---
 5 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/src/blocked.c b/src/blocked.c
index 92f1cee65..c3884fbdf 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -634,6 +634,16 @@ void unblockClientWaitingData(client *c) {
     }
 }
 
+static int getBlockedTypeByType(int type) {
+    switch (type) {
+        case OBJ_LIST: return BLOCKED_LIST;
+        case OBJ_ZSET: return BLOCKED_ZSET;
+        case OBJ_MODULE: return BLOCKED_MODULE;
+        case OBJ_STREAM: return BLOCKED_STREAM;
+        default: return BLOCKED_NONE;
+    }
+}
+
 /* If the specified key has clients blocked waiting for list pushes, this
  * function will put the key reference into the server.ready_keys list.
  * Note that db->ready_keys is a hash table that allows us to avoid putting
@@ -641,9 +651,14 @@ void unblockClientWaitingData(client *c) {
  * made by a script or in the context of MULTI/EXEC.
  *
  * The list will be finally processed by handleClientsBlockedOnKeys() */
-void signalKeyAsReady(redisDb *db, robj *key) {
+void signalKeyAsReady(redisDb *db, robj *key, int type) {
     readyList *rl;
 
+    /* If no clients are blocked on this type, just return */
+    int btype = getBlockedTypeByType(type);
+    if (btype == BLOCKED_NONE || !server.blocked_clients_by_type[btype])
+        return;
+
     /* No clients blocking for this key? No need to queue it. */
     if (dictFind(db->blocking_keys,key) == NULL) return;
 
@@ -664,4 +679,3 @@ void signalKeyAsReady(redisDb *db, robj *key) {
     serverAssert(dictAdd(db->ready_keys,key,NULL) == DICT_OK);
 }
 
-
diff --git a/src/db.c b/src/db.c
index 529b57c7b..70de568fb 100644
--- a/src/db.c
+++ b/src/db.c
@@ -181,10 +181,7 @@ void dbAdd(redisDb *db, robj *key, robj *val) {
     int retval = dictAdd(db->dict, copy, val);
 
     serverAssertWithInfo(NULL,key,retval == DICT_OK);
-    if (val->type == OBJ_LIST ||
-        val->type == OBJ_ZSET ||
-        val->type == OBJ_STREAM)
-        signalKeyAsReady(db, key);
+    signalKeyAsReady(db, key, val->type);
     if (server.cluster_enabled) slotToKeyAdd(key->ptr);
 }
 
@@ -1089,10 +1086,7 @@ void scanDatabaseForReadyLists(redisDb *db) {
     while((de = dictNext(di)) != NULL) {
         robj *key = dictGetKey(de);
         robj *value = lookupKey(db,key,LOOKUP_NOTOUCH);
-        if (value && (value->type == OBJ_LIST ||
-                      value->type == OBJ_STREAM ||
-                      value->type == OBJ_ZSET))
-            signalKeyAsReady(db, key);
+        if (value) signalKeyAsReady(db, key, value->type);
     }
     dictReleaseIterator(di);
 }
diff --git a/src/module.c b/src/module.c
index 8e9526dad..0714458f7 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4579,7 +4579,7 @@ RedisModuleBlockedClient *RM_BlockClientOnKeys(RedisModuleCtx *ctx, RedisModuleC
  * all the clients blocked for this key will get their reply callback called,
  * and if the callback returns REDISMODULE_OK the client will be unblocked. */
 void RM_SignalKeyAsReady(RedisModuleCtx *ctx, RedisModuleString *key) {
-    signalKeyAsReady(ctx->client->db, key);
+    signalKeyAsReady(ctx->client->db, key, OBJ_MODULE);
 }
 
 /* Implements RM_UnblockClient() and moduleUnblockClient(). */
diff --git a/src/server.h b/src/server.h
index 19aab8750..dbfa5aead 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2189,7 +2189,7 @@ void replyToBlockedClientTimedOut(client *c);
 int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int unit);
 void disconnectAllBlockedClients(void);
 void handleClientsBlockedOnKeys(void);
-void signalKeyAsReady(redisDb *db, robj *key);
+void signalKeyAsReady(redisDb *db, robj *key, int type);
 void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids);
 
 /* timeout.c -- Blocked clients timeout and connections timeout. */
diff --git a/src/t_stream.c b/src/t_stream.c
index 38bdce8e0..fc5c4c99e 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1299,8 +1299,7 @@ void xaddCommand(client *c) {
 
     /* We need to signal to blocked clients that there is new data on this
      * stream. */
-    if (server.blocked_clients_by_type[BLOCKED_STREAM])
-        signalKeyAsReady(c->db, c->argv[1]);
+    signalKeyAsReady(c->db, c->argv[1], OBJ_STREAM);
 }
 
 /* XRANGE/XREVRANGE actual implementation. */
@@ -1876,7 +1875,7 @@ NULL
             notifyKeyspaceEvent(NOTIFY_STREAM,"xgroup-destroy",
                                 c->argv[2],c->db->id);
             /* We want to unblock any XREADGROUP consumers with -NOGROUP. */
-            signalKeyAsReady(c->db,c->argv[2]);
+            signalKeyAsReady(c->db,c->argv[2],OBJ_STREAM);
         } else {
             addReply(c,shared.czero);
         }

From f39c9404a397d7c48a6e8347666f9d037861db60 Mon Sep 17 00:00:00 2001
From: Jim Brunner <brunnerj@amazon.com>
Date: Mon, 10 Aug 2020 22:36:34 -0700
Subject: [PATCH 0561/1098] Prevent dictRehashMilliseconds from rehashing while
 a safe iterator is present (#5948)

---
 src/dict.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/dict.c b/src/dict.c
index 45aab66f9..a26ef18d3 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -239,6 +239,8 @@ long long timeInMilliseconds(void) {
 
 /* Rehash for an amount of time between ms milliseconds and ms+1 milliseconds */
 int dictRehashMilliseconds(dict *d, int ms) {
+    if (d->iterators > 0) return 0;
+
     long long start = timeInMilliseconds();
     int rehashes = 0;
 

From 59d437c727d0ad9f3fbb45b89c712d1ad0635572 Mon Sep 17 00:00:00 2001
From: Rajat Pawar <rpawar@g2.com>
Date: Tue, 11 Aug 2020 11:41:26 +0530
Subject: [PATCH 0562/1098] Fix comment about ACLGetCommandPerm()

---
 src/server.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.h b/src/server.h
index dbfa5aead..5c5f939ce 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1884,7 +1884,7 @@ void sendChildCOWInfo(int ptype, char *pname);
 extern rax *Users;
 extern user *DefaultUser;
 void ACLInit(void);
-/* Return values for ACLCheckUserCredentials(). */
+/* Return values for ACLCheckCommandPerm(). */
 #define ACL_OK 0
 #define ACL_DENIED_CMD 1
 #define ACL_DENIED_KEY 2

From 944cca98c84efcf0365b0525f88124c53dbe1c53 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 11 Aug 2020 10:59:47 +0300
Subject: [PATCH 0563/1098] Run daily workflow on main repo only (no forks).
 (#7646)

---
 .github/workflows/daily.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 986bd223b..010a32289 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -12,6 +12,7 @@ jobs:
 
   test-ubuntu-jemalloc:
     runs-on: ubuntu-latest
+    if: github.repository == 'redis/redis'
     timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v2
@@ -30,6 +31,7 @@ jobs:
 
   test-ubuntu-libc-malloc:
     runs-on: ubuntu-latest
+    if: github.repository == 'redis/redis'
     timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v2
@@ -48,6 +50,7 @@ jobs:
 
   test-ubuntu-32bit:
     runs-on: ubuntu-latest
+    if: github.repository == 'redis/redis'
     timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v2
@@ -70,6 +73,7 @@ jobs:
 
   test-ubuntu-tls:
     runs-on: ubuntu-latest
+    if: github.repository == 'redis/redis'
     timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v2
@@ -90,6 +94,7 @@ jobs:
 
   test-valgrind:
     runs-on: ubuntu-latest
+    if: github.repository == 'redis/redis'
     timeout-minutes: 14400
     steps:
     - uses: actions/checkout@v2
@@ -104,6 +109,7 @@ jobs:
 
   test-centos7-jemalloc:
     runs-on: ubuntu-latest
+    if: github.repository == 'redis/redis'
     container: centos:7
     timeout-minutes: 14400
     steps:
@@ -126,6 +132,7 @@ jobs:
 
   test-centos7-tls:
     runs-on: ubuntu-latest
+    if: github.repository == 'redis/redis'
     container: centos:7
     timeout-minutes: 14400
     steps:

From 6f11acbd67f65c5f3fd84223bf07fefd75cfdc37 Mon Sep 17 00:00:00 2001
From: Tyson Andre <tyson.andre@uwaterloo.ca>
Date: Tue, 11 Aug 2020 04:55:06 -0400
Subject: [PATCH 0564/1098] Implement SMISMEMBER key member [member ...]
 (#7615)

This is a rebased version of #3078 originally by shaharmor
with the following patches by TysonAndre made after rebasing
to work with the updated C API:

1. Add 2 more unit tests
   (wrong argument count error message, integer over 64 bits)
2. Use addReplyArrayLen instead of addReplyMultiBulkLen.
3. Undo changes to src/help.h - for the ZMSCORE PR,
   I heard those should instead be automatically
   generated from the redis-doc repo if it gets updated

Motivations:

- Example use case: Client code to efficiently check if each element of a set
  of 1000 items is a member of a set of 10 million items.
  (Similar to reasons for working on #7593)
- HMGET and ZMSCORE already exist. This may lead to developers deciding
  to implement functionality that's best suited to a regular set with a
  data type of sorted set or hash map instead, for the multi-get support.

Currently, multi commands or lua scripting to call sismember multiple times
would almost definitely be less efficient than a native smismember
for the following reasons:

- Need to fetch the set from the string every time
  instead of reusing the C pointer.
- Using pipelining or multi-commands would result in more bytes sent
  and received by the client for the repeated SISMEMBER KEY sections.
- Need to specially encode the data and decode it from the client
  for lua-based solutions.
- Proposed solutions using Lua or SADD/SDIFF could trigger writes to
  memory, which is undesirable on a redis replica server
  or when commands get replicated to replicas.

Co-Authored-By: Shahar Mor <shahar@peer5.com>
Co-Authored-By: Tyson Andre <tysonandre775@hotmail.com>
---
 src/server.c              |  4 ++++
 src/server.h              |  1 +
 src/t_set.c               | 19 +++++++++++++++++++
 tests/support/cluster.tcl |  2 +-
 tests/unit/type/set.tcl   | 34 ++++++++++++++++++++++++++++++++--
 5 files changed, 57 insertions(+), 3 deletions(-)

diff --git a/src/server.c b/src/server.c
index 15b962a2e..162dfa346 100644
--- a/src/server.c
+++ b/src/server.c
@@ -354,6 +354,10 @@ struct redisCommand redisCommandTable[] = {
      "read-only fast @set",
      0,NULL,1,1,1,0,0,0},
 
+    {"smismember",smismemberCommand,-3,
+     "read-only fast @set",
+     0,NULL,1,1,1,0,0,0},
+
     {"scard",scardCommand,2,
      "read-only fast @set",
      0,NULL,1,1,1,0,0,0},
diff --git a/src/server.h b/src/server.h
index 5c5f939ce..7f1e7ea7b 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2278,6 +2278,7 @@ void saddCommand(client *c);
 void sremCommand(client *c);
 void smoveCommand(client *c);
 void sismemberCommand(client *c);
+void smismemberCommand(client *c);
 void scardCommand(client *c);
 void spopCommand(client *c);
 void srandmemberCommand(client *c);
diff --git a/src/t_set.c b/src/t_set.c
index 75678f47c..f474c2b56 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -382,6 +382,25 @@ void sismemberCommand(client *c) {
         addReply(c,shared.czero);
 }
 
+void smismemberCommand(client *c) {
+    robj *set;
+    int j;
+
+    /* Don't abort when the key cannot be found. Non-existing keys are empty
+     * sets, where SMISMEMBER should respond with a series of zeros. */
+    set = lookupKeyRead(c->db,c->argv[1]);
+    if (set && checkType(c,set,OBJ_SET)) return;
+
+    addReplyArrayLen(c,c->argc - 2);
+
+    for (j = 2; j < c->argc; j++) {
+        if (set && setTypeIsMember(set,c->argv[j]->ptr))
+            addReply(c,shared.cone);
+        else
+            addReply(c,shared.czero);
+    }
+}
+
 void scardCommand(client *c) {
     robj *o;
 
diff --git a/tests/support/cluster.tcl b/tests/support/cluster.tcl
index dced8d864..fb8c46e75 100644
--- a/tests/support/cluster.tcl
+++ b/tests/support/cluster.tcl
@@ -24,7 +24,7 @@ set ::redis_cluster::plain_commands {
     get set setnx setex psetex append strlen exists setbit getbit
     setrange getrange substr incr decr rpush lpush rpushx lpushx
     linsert rpop lpop brpop llen lindex lset lrange ltrim lrem
-    sadd srem sismember scard spop srandmember smembers sscan zadd
+    sadd srem sismember smismember scard spop srandmember smembers sscan zadd
     zincrby zrem zremrangebyscore zremrangebyrank zremrangebylex zrange
     zrangebyscore zrevrangebyscore zrangebylex zrevrangebylex zcount
     zlexcount zrevrange zcard zscore zmscore zrank zrevrank zscan hset hsetnx
diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl
index 7b467f1c4..84c31c4e4 100644
--- a/tests/unit/type/set.tcl
+++ b/tests/unit/type/set.tcl
@@ -9,7 +9,7 @@ start_server {
         foreach entry $entries { r sadd $key $entry }
     }
 
-    test {SADD, SCARD, SISMEMBER, SMEMBERS basics - regular set} {
+    test {SADD, SCARD, SISMEMBER, SMISMEMBER, SMEMBERS basics - regular set} {
         create_set myset {foo}
         assert_encoding hashtable myset
         assert_equal 1 [r sadd myset bar]
@@ -18,10 +18,15 @@ start_server {
         assert_equal 1 [r sismember myset foo]
         assert_equal 1 [r sismember myset bar]
         assert_equal 0 [r sismember myset bla]
+        assert_equal {1} [r smismember myset foo]
+        assert_equal {1 1} [r smismember myset foo bar]
+        assert_equal {1 0} [r smismember myset foo bla]
+        assert_equal {0 1} [r smismember myset bla foo]
+        assert_equal {0} [r smismember myset bla]
         assert_equal {bar foo} [lsort [r smembers myset]]
     }
 
-    test {SADD, SCARD, SISMEMBER, SMEMBERS basics - intset} {
+    test {SADD, SCARD, SISMEMBER, SMISMEMBER, SMEMBERS basics - intset} {
         create_set myset {17}
         assert_encoding intset myset
         assert_equal 1 [r sadd myset 16]
@@ -30,9 +35,33 @@ start_server {
         assert_equal 1 [r sismember myset 16]
         assert_equal 1 [r sismember myset 17]
         assert_equal 0 [r sismember myset 18]
+        assert_equal {1} [r smismember myset 16]
+        assert_equal {1 1} [r smismember myset 16 17]
+        assert_equal {1 0} [r smismember myset 16 18]
+        assert_equal {0 1} [r smismember myset 18 16]
+        assert_equal {0} [r smismember myset 18]
         assert_equal {16 17} [lsort [r smembers myset]]
     }
 
+    test {SMISMEMBER against non set} {
+        r lpush mylist foo
+        assert_error WRONGTYPE* {r smismember mylist bar}
+    }
+
+    test {SMISMEMBER non existing key} {
+        assert_equal {0} [r smismember myset1 foo]
+        assert_equal {0 0} [r smismember myset1 foo bar]
+    }
+
+    test {SMISMEMBER requires one or more members} {
+        r del zmscoretest
+        r zadd zmscoretest 10 x
+        r zadd zmscoretest 20 y
+        
+        catch {r smismember zmscoretest} e
+        assert_match {*ERR*wrong*number*arg*} $e
+    }
+
     test {SADD against non set} {
         r lpush mylist foo
         assert_error WRONGTYPE* {r sadd mylist bar}
@@ -49,6 +78,7 @@ start_server {
         create_set myset {213244124402402314402033402}
         assert_encoding hashtable myset
         assert_equal 1 [r sismember myset 213244124402402314402033402]
+        assert_equal {1} [r smismember myset 213244124402402314402033402]
     }
 
     test "SADD overflows the maximum allowed integers in an intset" {

From 8e937ce4cc1462d996bae6a45e8c0a66f71e7ee6 Mon Sep 17 00:00:00 2001
From: YoongHM <yoonghm@users.noreply.github.com>
Date: Tue, 11 Aug 2020 17:30:32 +0800
Subject: [PATCH 0565/1098] Start redis after network is online (#7639)

The two lines allow systemd to start redis.service after the network is online. Only after the network is online that Redis could bind to IP address other than 127.0.0.1 during initial boot up process.
---
 utils/systemd-redis_server.service | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/utils/systemd-redis_server.service b/utils/systemd-redis_server.service
index addee3498..cf158644a 100644
--- a/utils/systemd-redis_server.service
+++ b/utils/systemd-redis_server.service
@@ -20,6 +20,8 @@ Description=Redis data structure server
 Documentation=https://redis.io/documentation
 #Before=your_application.service another_example_application.service
 #AssertPathExists=/var/lib/redis
+Wants=network-online.target
+After=network-online.target
 
 [Service]
 ExecStart=/usr/local/bin/redis-server --supervised systemd --daemonize no

From 09af8447a1ae479839e75b0743266eaa786f35e1 Mon Sep 17 00:00:00 2001
From: pingfan108 <xuanyuan108@126.com>
Date: Tue, 11 Aug 2020 18:35:20 +0800
Subject: [PATCH 0566/1098] Correct error message of runtest-cluster and
 runtest-moduleapi (#7647)

---
 runtest-cluster   | 2 +-
 runtest-moduleapi | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtest-cluster b/runtest-cluster
index 27829a5fe..a86e93141 100755
--- a/runtest-cluster
+++ b/runtest-cluster
@@ -8,7 +8,7 @@ done
 
 if [ -z $TCLSH ]
 then
-    echo "You need tcl 8.5 or newer in order to run the Redis Sentinel test"
+    echo "You need tcl 8.5 or newer in order to run the Redis Cluster test"
     exit 1
 fi
 $TCLSH tests/cluster/run.tcl $*
diff --git a/runtest-moduleapi b/runtest-moduleapi
index 71db27e5e..85386b44a 100755
--- a/runtest-moduleapi
+++ b/runtest-moduleapi
@@ -8,7 +8,7 @@ done
 
 if [ -z $TCLSH ]
 then
-    echo "You need tcl 8.5 or newer in order to run the Redis test"
+    echo "You need tcl 8.5 or newer in order to run the Redis ModuleApi test"
     exit 1
 fi
 

From 2e69bfe44d154d6739da48ff87f1c66045f68e93 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Tue, 30 Oct 2018 00:38:20 +0800
Subject: [PATCH 0567/1098] using proto-max-bulk-len in checkStringLength for
 SETRANGE and APPEND

---
 src/t_string.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index 259f43142..4be758e65 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -35,8 +35,8 @@
  *----------------------------------------------------------------------------*/
 
 static int checkStringLength(client *c, long long size) {
-    if (size > 512*1024*1024) {
-        addReplyError(c,"string exceeds maximum allowed size (512MB)");
+    if (!(c->flags & CLIENT_MASTER) && size > server.proto_max_bulk_len) {
+        addReplyError(c,"string exceeds maximum allowed size (proto-max-bulk-len)");
         return C_ERR;
     }
     return C_OK;

From bd4b33d7a2b00fddcaaccee9705d21b92d97311e Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Tue, 30 Oct 2018 00:47:49 +0800
Subject: [PATCH 0568/1098] config: proto-max-bulk-len must be 1mb or greater

---
 redis.conf   | 2 +-
 src/config.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/redis.conf b/redis.conf
index 33f15b9ee..b8134e8c4 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1651,7 +1651,7 @@ client-output-buffer-limit pubsub 32mb 8mb 60
 
 # In the Redis protocol, bulk requests, that are, elements representing single
 # strings, are normally limited ot 512 mb. However you can change this limit
-# here.
+# here, but must be 1mb or greater
 #
 # proto-max-bulk-len 512mb
 
diff --git a/src/config.c b/src/config.c
index b94d1c9f4..8a1b6d934 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2217,7 +2217,7 @@ standardConfig configs[] = {
     createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, 15000, INTEGER_CONFIG, NULL, NULL),
     createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, 10000, INTEGER_CONFIG, NULL, NULL),
     createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, 0, INTEGER_CONFIG, NULL, NULL),
-    createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */
+    createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 1024*1024, LLONG_MAX, server.proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */
     createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL),
     createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */
 

From 589e610ebc62ced9c5392b7ece122c68c9990988 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <zhaozhao.zz@alibaba-inc.com>
Date: Tue, 30 Oct 2018 00:50:36 +0800
Subject: [PATCH 0569/1098] CLIENT_MASTER should ignore
 server.proto_max_bulk_len

---
 src/networking.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index 0da9bbc9c..8ec85b393 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1694,7 +1694,8 @@ int processMultibulkBuffer(client *c) {
             }
 
             ok = string2ll(c->querybuf+c->qb_pos+1,newline-(c->querybuf+c->qb_pos+1),&ll);
-            if (!ok || ll < 0 || ll > server.proto_max_bulk_len) {
+            if (!ok || ll < 0 ||
+                (!(c->flags & CLIENT_MASTER) && ll > server.proto_max_bulk_len)) {
                 addReplyError(c,"Protocol error: invalid bulk length");
                 setProtocolError("invalid bulk length",c);
                 return C_ERR;

From 5dd499c6d45dcd2c8bd082b03895847d3e8c465b Mon Sep 17 00:00:00 2001
From: Muhammad Zahalqa <m@tryfinally.com>
Date: Tue, 11 Aug 2020 14:16:10 +0300
Subject: [PATCH 0570/1098] Fix unidentical function declaration in bio.c. 
 lazyfree.c: lazyfreeFreeSlotsMapFromBioThread (#7228)

---
 src/bio.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/bio.c b/src/bio.c
index 69c62fc6f..cbbecb6f7 100644
--- a/src/bio.c
+++ b/src/bio.c
@@ -86,7 +86,7 @@ struct bio_job {
 void *bioProcessBackgroundJobs(void *arg);
 void lazyfreeFreeObjectFromBioThread(robj *o);
 void lazyfreeFreeDatabaseFromBioThread(dict *ht1, dict *ht2);
-void lazyfreeFreeSlotsMapFromBioThread(zskiplist *sl);
+void lazyfreeFreeSlotsMapFromBioThread(rax *rt);
 
 /* Make sure we have enough stack to perform all the things we do in the
  * main thread. */

From ff1e4a7063c4de96bba3661f5f282ed61b6dfe8e Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <276441700@qq.com>
Date: Tue, 11 Aug 2020 20:51:27 +0800
Subject: [PATCH 0571/1098] redis-benchmark: fix wrong random key for hset
 (#4895)

---
 src/redis-benchmark.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 9ba381409..fc24afbff 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -1723,7 +1723,7 @@ int main(int argc, const char **argv) {
 
         if (test_is_selected("hset")) {
             len = redisFormatCommand(&cmd,
-                "HSET myhash:{tag}:__rand_int__ element:__rand_int__ %s",data);
+                "HSET myhash:{tag} element:__rand_int__ %s",data);
             benchmark("HSET",cmd,len);
             free(cmd);
         }

From e2a71338ebfb719cba94742f1c75ce755cd6074d Mon Sep 17 00:00:00 2001
From: Wagner Francisco Mezaroba <wagnerfrancisco@gmail.com>
Date: Tue, 11 Aug 2020 19:57:21 +0100
Subject: [PATCH 0572/1098] allow --pattern to be used along with --bigkeys
 (#3586)

Adds --pattern option to cli's --bigkeys, --hotkeys & --scan modes
---
 src/redis-cli.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 09f95b507..4df157d20 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -1846,7 +1846,8 @@ static void usage(void) {
 "  --hotkeys          Sample Redis keys looking for hot keys.\n"
 "                     only works when maxmemory-policy is *lfu.\n"
 "  --scan             List all keys using the SCAN command.\n"
-"  --pattern <pat>    Useful with --scan to specify a SCAN pattern.\n"
+"  --pattern <pat>    Keys pattern when using the --scan, --bigkeys or --hotkeys\n"
+"                     options (default: *).\n"
 "  --intrinsic-latency <sec> Run a test to measure intrinsic system latency.\n"
 "                     The test will run for the specified amount of seconds.\n"
 "  --eval <file>      Send an EVAL command using the Lua script at <file>.\n"
@@ -7275,7 +7276,13 @@ static void pipeMode(void) {
  *--------------------------------------------------------------------------- */
 
 static redisReply *sendScan(unsigned long long *it) {
-    redisReply *reply = redisCommand(context, "SCAN %llu", *it);
+    redisReply *reply;
+
+    if (config.pattern)
+        reply = redisCommand(context,"SCAN %llu MATCH %s",
+            *it,config.pattern);
+    else
+        reply = redisCommand(context,"SCAN %llu",*it);
 
     /* Handle any error conditions */
     if(reply == NULL) {

From fbed632f3aa7381bef1087e39d2506306d62d3c2 Mon Sep 17 00:00:00 2001
From: Mota <motablow@gmail.com>
Date: Wed, 12 Aug 2020 03:01:15 +0800
Subject: [PATCH 0573/1098] Adds redis-cli and redis-benchmark dependencies for
 `make test` target

Obsoletes the need to run `make` before `make test`.
---
 src/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Makefile b/src/Makefile
index 80c627c24..3ca7b1685 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -326,10 +326,10 @@ distclean: clean
 
 .PHONY: distclean
 
-test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME)
+test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME)
 	@(cd ..; ./runtest)
 
-test-sentinel: $(REDIS_SENTINEL_NAME)
+test-sentinel: $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME)
 	@(cd ..; ./runtest-sentinel)
 
 check: test

From ddcbb628a18cbd6febc6c31e71de1a140239e8b4 Mon Sep 17 00:00:00 2001
From: Mota <motablow@gmail.com>
Date: Wed, 12 Aug 2020 10:25:24 +0800
Subject: [PATCH 0574/1098] Test:Fix invalid cases in hash.tcl and dump.tcl
 (#4611)

---
 tests/unit/dump.tcl      | 2 +-
 tests/unit/type/hash.tcl | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/unit/dump.tcl b/tests/unit/dump.tcl
index a9def9206..affce65e5 100644
--- a/tests/unit/dump.tcl
+++ b/tests/unit/dump.tcl
@@ -170,7 +170,7 @@ start_server {tags {"dump"}} {
             $second set list somevalue
             catch {r -1 migrate $second_host $second_port list 9 5000 copy} e
             assert_match {ERR*} $e
-            set res [r -1 migrate $second_host $second_port list 9 5000 copy replace]
+            set ret [r -1 migrate $second_host $second_port list 9 5000 copy replace]
             assert {$ret eq {OK}}
             assert {[$first exists list] == 1}
             assert {[$second exists list] == 1}
diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl
index 9f8a21b1c..257404ea3 100644
--- a/tests/unit/type/hash.tcl
+++ b/tests/unit/type/hash.tcl
@@ -302,7 +302,7 @@ start_server {tags {"hash"}} {
         r hset smallhash str " 11"
         r hset bighash str " 11"
         catch {r hincrby smallhash str 1} smallerr
-        catch {r hincrby smallhash str 1} bigerr
+        catch {r hincrby bighash str 1} bigerr
         set rv {}
         lappend rv [string match "ERR*not an integer*" $smallerr]
         lappend rv [string match "ERR*not an integer*" $bigerr]
@@ -312,7 +312,7 @@ start_server {tags {"hash"}} {
         r hset smallhash str "11 "
         r hset bighash str "11 "
         catch {r hincrby smallhash str 1} smallerr
-        catch {r hincrby smallhash str 1} bigerr
+        catch {r hincrby bighash str 1} bigerr
         set rv {}
         lappend rv [string match "ERR*not an integer*" $smallerr]
         lappend rv [string match "ERR*not an integer*" $bigerr]
@@ -374,7 +374,7 @@ start_server {tags {"hash"}} {
         r hset smallhash str " 11"
         r hset bighash str " 11"
         catch {r hincrbyfloat smallhash str 1} smallerr
-        catch {r hincrbyfloat smallhash str 1} bigerr
+        catch {r hincrbyfloat bighash str 1} bigerr
         set rv {}
         lappend rv [string match "ERR*not*float*" $smallerr]
         lappend rv [string match "ERR*not*float*" $bigerr]
@@ -384,7 +384,7 @@ start_server {tags {"hash"}} {
         r hset smallhash str "11 "
         r hset bighash str "11 "
         catch {r hincrbyfloat smallhash str 1} smallerr
-        catch {r hincrbyfloat smallhash str 1} bigerr
+        catch {r hincrbyfloat bighash str 1} bigerr
         set rv {}
         lappend rv [string match "ERR*not*float*" $smallerr]
         lappend rv [string match "ERR*not*float*" $bigerr]

From f2db379fa3f6e0c0d81350fd0f29febea95df469 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Tue, 11 Aug 2020 20:04:54 -0700
Subject: [PATCH 0575/1098] Replace usage of wrongtypeerr with helper (#7633)

* Replace usage of wrongtypeerr with helper
---
 src/bitops.c   |  2 +-
 src/geo.c      |  4 ++--
 src/object.c   |  3 ++-
 src/t_hash.c   | 12 +++---------
 src/t_list.c   | 21 ++++++++-------------
 src/t_set.c    |  9 +++------
 src/t_stream.c | 10 +++-------
 src/t_string.c | 13 ++++++-------
 src/t_zset.c   | 28 ++++++++++------------------
 9 files changed, 38 insertions(+), 64 deletions(-)

diff --git a/src/bitops.c b/src/bitops.c
index 4b1a09aa4..37fb13f0c 100644
--- a/src/bitops.c
+++ b/src/bitops.c
@@ -480,12 +480,12 @@ int getBitfieldTypeFromArgument(client *c, robj *o, int *sign, int *bits) {
 robj *lookupStringForBitCommand(client *c, size_t maxbit) {
     size_t byte = maxbit >> 3;
     robj *o = lookupKeyWrite(c->db,c->argv[1]);
+    if (checkType(c,o,OBJ_STRING)) return NULL;
 
     if (o == NULL) {
         o = createObject(OBJ_STRING,sdsnewlen(NULL, byte+1));
         dbAdd(c->db,c->argv[1],o);
     } else {
-        if (checkType(c,o,OBJ_STRING)) return NULL;
         o = dbUnshareStringValue(c->db,c->argv[1],o);
         o->ptr = sdsgrowzero(o->ptr,byte+1);
     }
diff --git a/src/geo.c b/src/geo.c
index 3e5d5f606..77f07637a 100644
--- a/src/geo.c
+++ b/src/geo.c
@@ -702,7 +702,7 @@ void geohashCommand(client *c) {
 
     /* Look up the requested zset */
     robj *zobj = lookupKeyRead(c->db, c->argv[1]);
-    if (zobj && checkType(c, zobj, OBJ_ZSET)) return;
+    if (checkType(c, zobj, OBJ_ZSET)) return;
 
     /* Geohash elements one after the other, using a null bulk reply for
      * missing elements. */
@@ -763,7 +763,7 @@ void geoposCommand(client *c) {
 
     /* Look up the requested zset */
     robj *zobj = lookupKeyRead(c->db, c->argv[1]);
-    if (zobj && checkType(c, zobj, OBJ_ZSET)) return;
+    if (checkType(c, zobj, OBJ_ZSET)) return;
 
     /* Report elements one after the other, using a null bulk reply for
      * missing elements. */
diff --git a/src/object.c b/src/object.c
index 6ba18d6a5..184e5ffc0 100644
--- a/src/object.c
+++ b/src/object.c
@@ -402,7 +402,8 @@ robj *resetRefCount(robj *obj) {
 }
 
 int checkType(client *c, robj *o, int type) {
-    if (o->type != type) {
+    /* A NULL is considered an empty key */
+    if (o && o->type != type) {
         addReply(c,shared.wrongtypeerr);
         return 1;
     }
diff --git a/src/t_hash.c b/src/t_hash.c
index 866bcd25b..5dacd735b 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -450,14 +450,11 @@ sds hashTypeCurrentObjectNewSds(hashTypeIterator *hi, int what) {
 
 robj *hashTypeLookupWriteOrCreate(client *c, robj *key) {
     robj *o = lookupKeyWrite(c->db,key);
+    if (checkType(c,o,OBJ_HASH)) return NULL;
+
     if (o == NULL) {
         o = createHashObject();
         dbAdd(c->db,key,o);
-    } else {
-        if (o->type != OBJ_HASH) {
-            addReply(c,shared.wrongtypeerr);
-            return NULL;
-        }
     }
     return o;
 }
@@ -692,10 +689,7 @@ void hmgetCommand(client *c) {
     /* Don't abort when the key cannot be found. Non-existing keys are empty
      * hashes, where HMGET should respond with a series of null bulks. */
     o = lookupKeyRead(c->db, c->argv[1]);
-    if (o != NULL && o->type != OBJ_HASH) {
-        addReply(c, shared.wrongtypeerr);
-        return;
-    }
+    if (checkType(c,o,OBJ_HASH)) return;
 
     addReplyArrayLen(c, c->argc-2);
     for (i = 2; i < c->argc; i++) {
diff --git a/src/t_list.c b/src/t_list.c
index 2c339888d..7737da771 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -198,8 +198,7 @@ void pushGenericCommand(client *c, int where) {
     int j, pushed = 0;
     robj *lobj = lookupKeyWrite(c->db,c->argv[1]);
 
-    if (lobj && lobj->type != OBJ_LIST) {
-        addReply(c,shared.wrongtypeerr);
+    if (checkType(c,lobj,OBJ_LIST)) {
         return;
     }
 
@@ -691,7 +690,7 @@ void rpoplpushCommand(client *c) {
         robj *dobj = lookupKeyWrite(c->db,c->argv[2]);
         robj *touchedkey = c->argv[1];
 
-        if (dobj && checkType(c,dobj,OBJ_LIST)) return;
+        if (checkType(c,dobj,OBJ_LIST)) return;
         value = listTypePop(sobj,LIST_TAIL);
         /* We saved touched key, and protect it, since rpoplpushHandlePush
          * may change the client command argument vector (it does not
@@ -803,8 +802,7 @@ void blockingPopGenericCommand(client *c, int where) {
     for (j = 1; j < c->argc-1; j++) {
         o = lookupKeyWrite(c->db,c->argv[j]);
         if (o != NULL) {
-            if (o->type != OBJ_LIST) {
-                addReply(c,shared.wrongtypeerr);
+            if (checkType(c,o,OBJ_LIST)) {
                 return;
             } else {
                 if (listTypeLength(o) != 0) {
@@ -863,6 +861,7 @@ void brpoplpushCommand(client *c) {
         != C_OK) return;
 
     robj *key = lookupKeyWrite(c->db, c->argv[1]);
+    if (checkType(c,key,OBJ_LIST)) return;
 
     if (key == NULL) {
         if (c->flags & CLIENT_MULTI) {
@@ -874,13 +873,9 @@ void brpoplpushCommand(client *c) {
             blockForKeys(c,BLOCKED_LIST,c->argv + 1,1,timeout,c->argv[2],NULL);
         }
     } else {
-        if (key->type != OBJ_LIST) {
-            addReply(c, shared.wrongtypeerr);
-        } else {
-            /* The list exists and has elements, so
-             * the regular rpoplpushCommand is executed. */
-            serverAssertWithInfo(c,key,listTypeLength(key) > 0);
-            rpoplpushCommand(c);
-        }
+        /* The list exists and has elements, so
+         * the regular rpoplpushCommand is executed. */
+        serverAssertWithInfo(c,key,listTypeLength(key) > 0);
+        rpoplpushCommand(c);
     }
 }
diff --git a/src/t_set.c b/src/t_set.c
index f474c2b56..9166e2b9a 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -266,14 +266,11 @@ void saddCommand(client *c) {
     int j, added = 0;
 
     set = lookupKeyWrite(c->db,c->argv[1]);
+    if (checkType(c,set,OBJ_SET)) return;
+    
     if (set == NULL) {
         set = setTypeCreate(c->argv[2]->ptr);
         dbAdd(c->db,c->argv[1],set);
-    } else {
-        if (set->type != OBJ_SET) {
-            addReply(c,shared.wrongtypeerr);
-            return;
-        }
     }
 
     for (j = 2; j < c->argc; j++) {
@@ -330,7 +327,7 @@ void smoveCommand(client *c) {
     /* If the source key has the wrong type, or the destination key
      * is set and has the wrong type, return with an error. */
     if (checkType(c,srcset,OBJ_SET) ||
-        (dstset && checkType(c,dstset,OBJ_SET))) return;
+        checkType(c,dstset,OBJ_SET)) return;
 
     /* If srcset and dstset are equal, SMOVE is a no-op */
     if (srcset == dstset) {
diff --git a/src/t_stream.c b/src/t_stream.c
index fc5c4c99e..1ab85a3b1 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1108,14 +1108,10 @@ size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start
  * The function creates a key setting it to an empty stream if needed. */
 robj *streamTypeLookupWriteOrCreate(client *c, robj *key) {
     robj *o = lookupKeyWrite(c->db,key);
+    if (checkType(c,o,OBJ_STREAM)) return NULL;
     if (o == NULL) {
         o = createStreamObject();
         dbAdd(c->db,key,o);
-    } else {
-        if (o->type != OBJ_STREAM) {
-            addReply(c,shared.wrongtypeerr);
-            return NULL;
-        }
     }
     return o;
 }
@@ -1460,7 +1456,7 @@ void xreadCommand(client *c) {
         int id_idx = i - streams_arg - streams_count;
         robj *key = c->argv[i-streams_count];
         robj *o = lookupKeyRead(c->db,key);
-        if (o && checkType(c,o,OBJ_STREAM)) goto cleanup;
+        if (checkType(c,o,OBJ_STREAM)) goto cleanup;
         streamCG *group = NULL;
 
         /* If a group was specified, than we need to be sure that the
@@ -2022,7 +2018,7 @@ void xpendingCommand(client *c) {
     robj *o = lookupKeyRead(c->db,c->argv[1]);
     streamCG *group;
 
-    if (o && checkType(c,o,OBJ_STREAM)) return;
+    if (checkType(c,o,OBJ_STREAM)) return;
     if (o == NULL ||
         (group = streamLookupCG(o->ptr,groupname->ptr)) == NULL)
     {
diff --git a/src/t_string.c b/src/t_string.c
index 4be758e65..93910e5a9 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -167,13 +167,12 @@ int getGenericCommand(client *c) {
     if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) == NULL)
         return C_OK;
 
-    if (o->type != OBJ_STRING) {
-        addReply(c,shared.wrongtypeerr);
+    if (checkType(c,o,OBJ_STRING)) {
         return C_ERR;
-    } else {
-        addReplyBulk(c,o);
-        return C_OK;
     }
+
+    addReplyBulk(c,o);
+    return C_OK;
 }
 
 void getCommand(client *c) {
@@ -348,7 +347,7 @@ void incrDecrCommand(client *c, long long incr) {
     robj *o, *new;
 
     o = lookupKeyWrite(c->db,c->argv[1]);
-    if (o != NULL && checkType(c,o,OBJ_STRING)) return;
+    if (checkType(c,o,OBJ_STRING)) return;
     if (getLongLongFromObjectOrReply(c,o,&value,NULL) != C_OK) return;
 
     oldvalue = value;
@@ -408,7 +407,7 @@ void incrbyfloatCommand(client *c) {
     robj *o, *new, *aux1, *aux2;
 
     o = lookupKeyWrite(c->db,c->argv[1]);
-    if (o != NULL && checkType(c,o,OBJ_STRING)) return;
+    if (checkType(c,o,OBJ_STRING)) return;
     if (getLongDoubleFromObjectOrReply(c,o,&value,NULL) != C_OK ||
         getLongDoubleFromObjectOrReply(c,c->argv[2],&incr,NULL) != C_OK)
         return;
diff --git a/src/t_zset.c b/src/t_zset.c
index 87a5a882e..3b716bb99 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -1598,6 +1598,7 @@ void zaddGenericCommand(client *c, int flags) {
 
     /* Lookup the key and create the sorted set if does not exist. */
     zobj = lookupKeyWrite(c->db,key);
+    if (checkType(c,zobj,OBJ_ZSET)) goto cleanup;
     if (zobj == NULL) {
         if (xx) goto reply_to_client; /* No key + XX option: nothing to do. */
         if (server.zset_max_ziplist_entries == 0 ||
@@ -1608,11 +1609,6 @@ void zaddGenericCommand(client *c, int flags) {
             zobj = createZsetZiplistObject();
         }
         dbAdd(c->db,key,zobj);
-    } else {
-        if (zobj->type != OBJ_ZSET) {
-            addReply(c,shared.wrongtypeerr);
-            goto cleanup;
-        }
     }
 
     for (j = 0; j < elements; j++) {
@@ -3087,7 +3083,7 @@ void zmscoreCommand(client *c) {
     robj *zobj;
     double score;
     zobj = lookupKeyRead(c->db,key);
-    if (zobj != NULL && checkType(c,zobj,OBJ_ZSET)) return;
+    if (checkType(c,zobj,OBJ_ZSET)) return;
 
     addReplyArrayLen(c,c->argc - 2);
     for (int j = 2; j < c->argc; j++) {
@@ -3280,20 +3276,16 @@ void blockingGenericZpopCommand(client *c, int where) {
 
     for (j = 1; j < c->argc-1; j++) {
         o = lookupKeyWrite(c->db,c->argv[j]);
+        if (checkType(c,o,OBJ_ZSET)) return;
         if (o != NULL) {
-            if (o->type != OBJ_ZSET) {
-                addReply(c,shared.wrongtypeerr);
+            if (zsetLength(o) != 0) {
+                /* Non empty zset, this is like a normal ZPOP[MIN|MAX]. */
+                genericZpopCommand(c,&c->argv[j],1,where,1,NULL);
+                /* Replicate it as an ZPOP[MIN|MAX] instead of BZPOP[MIN|MAX]. */
+                rewriteClientCommandVector(c,2,
+                    where == ZSET_MAX ? shared.zpopmax : shared.zpopmin,
+                    c->argv[j]);
                 return;
-            } else {
-                if (zsetLength(o) != 0) {
-                    /* Non empty zset, this is like a normal ZPOP[MIN|MAX]. */
-                    genericZpopCommand(c,&c->argv[j],1,where,1,NULL);
-                    /* Replicate it as an ZPOP[MIN|MAX] instead of BZPOP[MIN|MAX]. */
-                    rewriteClientCommandVector(c,2,
-                        where == ZSET_MAX ? shared.zpopmax : shared.zpopmin,
-                        c->argv[j]);
-                    return;
-                }
             }
         }
     }

From 79c506ebf07db32bbcf84ede9d72dd9eb04e9812 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Tue, 11 Aug 2020 20:16:41 -0700
Subject: [PATCH 0576/1098] Fixed timer warning (#5953)

---
 src/modules/hellotimer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/modules/hellotimer.c b/src/modules/hellotimer.c
index 57b111b7c..f6700df26 100644
--- a/src/modules/hellotimer.c
+++ b/src/modules/hellotimer.c
@@ -40,7 +40,7 @@
 /* Timer callback. */
 void timerHandler(RedisModuleCtx *ctx, void *data) {
     REDISMODULE_NOT_USED(ctx);
-    printf("Fired %s!\n", data);
+    printf("Fired %s!\n", (char *)data);
     RedisModule_Free(data);
 }
 

From f3df3ec1346565d96773aba9a4d315d614aeca4b Mon Sep 17 00:00:00 2001
From: HarveyLiu <harvey_liuhw@163.com>
Date: Wed, 12 Aug 2020 15:23:55 +0800
Subject: [PATCH 0577/1098] fix misleading typo hasActiveChildProcess doc
 comment (#7588)

and a misspell in rax.c
---
 src/rax.c    | 2 +-
 src/server.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/rax.c b/src/rax.c
index c8a1fb6b4..0ca915746 100644
--- a/src/rax.c
+++ b/src/rax.c
@@ -554,7 +554,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void **
      *
      * Splitting a compressed node have a few possible cases.
      * Imagine that the node 'h' we are currently at is a compressed
-     * node contaning the string "ANNIBALE" (it means that it represents
+     * node containing the string "ANNIBALE" (it means that it represents
      * nodes A -> N -> N -> I -> B -> A -> L -> E with the only child
      * pointer of this node pointing at the 'E' node, because remember that
      * we have characters at the edges of the graph, not inside the nodes
diff --git a/src/server.c b/src/server.c
index 162dfa346..5a9d55c0c 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1479,7 +1479,7 @@ void updateDictResizePolicy(void) {
         dictDisableResize();
 }
 
-/* Return true if there are no active children processes doing RDB saving,
+/* Return true if there are active children processes doing RDB saving,
  * AOF rewriting, or some side process spawned by a loaded module. */
 int hasActiveChildProcess() {
     return server.rdb_child_pid != -1 ||

From 2530dc0ebd8be8d792f4673073401377cd5bdc42 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 12 Aug 2020 17:58:56 +0300
Subject: [PATCH 0578/1098] Add oom-score-adj configuration option to control
 Linux OOM killer. (#1690)

Add Linux kernel OOM killer control option.

This adds the ability to control the Linux OOM killer oom_score_adj
parameter for all Redis processes, depending on the process role (i.e.
master, replica, background child).

A oom-score-adj global boolean flag control this feature. In addition,
specific values can be configured using oom-score-adj-values if
additional tuning is required.
---
 redis.conf                   |  26 ++++++++
 src/config.c                 | 119 +++++++++++++++++++++++++++++++++++
 src/config.h                 |   1 +
 src/replication.c            |   6 ++
 src/server.c                 |  60 ++++++++++++++++++
 src/server.h                 |  12 ++++
 tests/test_helper.tcl        |   1 +
 tests/unit/oom-score-adj.tcl |  81 ++++++++++++++++++++++++
 8 files changed, 306 insertions(+)
 create mode 100644 tests/unit/oom-score-adj.tcl

diff --git a/redis.conf b/redis.conf
index b8134e8c4..35ee752e1 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1049,6 +1049,32 @@ lazyfree-lazy-user-del no
 # --threads option to match the number of Redis theads, otherwise you'll not
 # be able to notice the improvements.
 
+############################ KERNEL OOM CONTROL ##############################
+
+# On Linux, it is possible to hint the kernel OOM killer on what processes
+# should be killed first when out of memory.
+#
+# Enabling this feature makes Redis actively control the oom_score_adj value
+# for all its processes, depending on their role. The default scores will
+# attempt to have background child processes killed before all others, and
+# replicas killed before masters.
+
+oom-score-adj no
+
+# When oom-score-adj is used, this directive controls the specific values used
+# for master, replica and background child processes. Values range -1000 to
+# 1000 (higher means more likely to be killed).
+#
+# Unprivileged processes (not root, and without CAP_SYS_RESOURCE capabilities)
+# can freely increase their value, but not decrease it below its initial
+# settings.
+#
+# Values are used relative to the initial value of oom_score_adj when the server
+# starts. Because typically the initial value is 0, they will often match the
+# absolute values.
+
+oom-score-adj-values 0 200 800
+
 ############################## APPEND ONLY MODE ###############################
 
 # By default Redis asynchronously dumps the dataset on disk. This mode is
diff --git a/src/config.c b/src/config.c
index 8a1b6d934..866665a68 100644
--- a/src/config.c
+++ b/src/config.c
@@ -111,6 +111,9 @@ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = {
     {1024*1024*32, 1024*1024*8, 60}  /* pubsub */
 };
 
+/* OOM Score defaults */
+int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT] = { 0, 200, 800 };
+
 /* Generic config infrastructure function pointers
  * int is_valid_fn(val, err)
  *     Return 1 when val is valid, and 0 when invalid.
@@ -286,6 +289,59 @@ void queueLoadModule(sds path, sds *argv, int argc) {
     listAddNodeTail(server.loadmodule_queue,loadmod);
 }
 
+/* Parse an array of CONFIG_OOM_COUNT sds strings, validate and populate
+ * server.oom_score_adj_values if valid.
+ */
+
+static int updateOOMScoreAdjValues(sds *args, char **err) {
+    int i;
+    int values[CONFIG_OOM_COUNT];
+
+    for (i = 0; i < CONFIG_OOM_COUNT; i++) {
+        char *eptr;
+        long long val = strtoll(args[i], &eptr, 10);
+
+        if (*eptr != '\0' || val < -1000 || val > 1000) {
+            if (err) *err = "Invalid oom-score-adj-values, elements must be between -1000 and 1000.";
+            return C_ERR;
+        }
+
+        values[i] = val;
+    }
+
+    /* Verify that the values make sense. If they don't omit a warning but
+     * keep the configuration, which may still be valid for privileged processes.
+     */
+
+    if (values[CONFIG_OOM_REPLICA] < values[CONFIG_OOM_MASTER] ||
+        values[CONFIG_OOM_BGCHILD] < values[CONFIG_OOM_REPLICA]) {
+            serverLog(LOG_WARNING,
+                    "The oom-score-adj-values configuration may not work for non-privileged processes! "
+                    "Please consult the documentation.");
+    }
+
+    /* Store values, retain previous config for rollback in case we fail. */
+    int old_values[CONFIG_OOM_COUNT];
+    for (i = 0; i < CONFIG_OOM_COUNT; i++) {
+        old_values[i] = server.oom_score_adj_values[i];
+        server.oom_score_adj_values[i] = values[i];
+    }
+
+    /* Update */
+    if (setOOMScoreAdj(-1) == C_ERR) {
+        /* Roll back */
+        for (i = 0; i < CONFIG_OOM_COUNT; i++)
+            server.oom_score_adj_values[i] = old_values[i];
+
+        if (err)
+            *err = "Failed to apply oom-score-adj-values configuration, check server logs.";
+
+        return C_ERR;
+    }
+
+    return C_OK;
+}
+
 void initConfigValues() {
     for (standardConfig *config = configs; config->name != NULL; config++) {
         config->interface.init(config->data);
@@ -479,6 +535,8 @@ void loadServerConfigFromString(char *config) {
             server.client_obuf_limits[class].hard_limit_bytes = hard;
             server.client_obuf_limits[class].soft_limit_bytes = soft;
             server.client_obuf_limits[class].soft_limit_seconds = soft_seconds;
+        } else if (!strcasecmp(argv[0],"oom-score-adj-values") && argc == 1 + CONFIG_OOM_COUNT) {
+            if (updateOOMScoreAdjValues(&argv[1], &err) == C_ERR) goto loaderr;
         } else if (!strcasecmp(argv[0],"notify-keyspace-events") && argc == 2) {
             int flags = keyspaceEventsStringToFlags(argv[1]);
 
@@ -728,6 +786,17 @@ void configSetCommand(client *c) {
             server.client_obuf_limits[class].soft_limit_seconds = soft_seconds;
         }
         sdsfreesplitres(v,vlen);
+    } config_set_special_field("oom-score-adj-values") {
+        int vlen;
+        int success = 1;
+
+        sds *v = sdssplitlen(o->ptr, sdslen(o->ptr), " ", 1, &vlen);
+        if (vlen != CONFIG_OOM_COUNT || updateOOMScoreAdjValues(v, &errstr) == C_ERR)
+            success = 0;
+
+        sdsfreesplitres(v, vlen);
+        if (!success)
+            goto badfmt;
     } config_set_special_field("notify-keyspace-events") {
         int flags = keyspaceEventsStringToFlags(o->ptr);
 
@@ -923,6 +992,22 @@ void configGetCommand(client *c) {
         matches++;
     }
 
+    if (stringmatch(pattern,"oom-score-adj-values",0)) {
+        sds buf = sdsempty();
+        int j;
+
+        for (j = 0; j < CONFIG_OOM_COUNT; j++) {
+            buf = sdscatprintf(buf,"%d", server.oom_score_adj_values[j]);
+            if (j != CONFIG_OOM_COUNT-1)
+                buf = sdscatlen(buf," ",1);
+        }
+
+        addReplyBulkCString(c,"oom-score-adj-values");
+        addReplyBulkCString(c,buf);
+        sdsfree(buf);
+        matches++;
+    }
+
     setDeferredMapLen(c,replylen,matches);
 }
 
@@ -1330,6 +1415,25 @@ void rewriteConfigClientoutputbufferlimitOption(struct rewriteConfigState *state
     }
 }
 
+/* Rewrite the oom-score-adj-values option. */
+void rewriteConfigOOMScoreAdjValuesOption(struct rewriteConfigState *state) {
+    int force = 0;
+    int j;
+    char *option = "oom-score-adj-values";
+    sds line;
+
+    line = sdsempty();
+    for (j = 0; j < CONFIG_OOM_COUNT; j++) {
+        if (server.oom_score_adj_values[j] != configOOMScoreAdjValuesDefaults[j])
+            force = 1;
+
+        line = sdscatprintf(line, "%d", server.oom_score_adj_values[j]);
+        if (j+1 != CONFIG_OOM_COUNT)
+            line = sdscatlen(line, " ", 1);
+    }
+    rewriteConfigRewriteLine(state,option,line,force);
+}
+
 /* Rewrite the bind option. */
 void rewriteConfigBindOption(struct rewriteConfigState *state) {
     int force = 1;
@@ -1528,6 +1632,7 @@ int rewriteConfig(char *path) {
     rewriteConfigStringOption(state,"cluster-config-file",server.cluster_configfile,CONFIG_DEFAULT_CLUSTER_CONFIG_FILE);
     rewriteConfigNotifykeyspaceeventsOption(state);
     rewriteConfigClientoutputbufferlimitOption(state);
+    rewriteConfigOOMScoreAdjValuesOption(state);
 
     /* Rewrite Sentinel config if in Sentinel mode. */
     if (server.sentinel_mode) rewriteConfigSentinelOption(state);
@@ -2082,6 +2187,19 @@ static int updateMaxclients(long long val, long long prev, char **err) {
     return 1;
 }
 
+static int updateOOMScoreAdj(int val, int prev, char **err) {
+    UNUSED(prev);
+
+    if (val) {
+        if (setOOMScoreAdj(-1) == C_ERR) {
+            *err = "Failed to set current oom_score_adj. Check server logs.";
+            return 0;
+        }
+    }
+
+    return 1;
+}
+
 #ifdef USE_OPENSSL
 static int updateTlsCfg(char *val, char *prev, char **err) {
     UNUSED(val);
@@ -2146,6 +2264,7 @@ standardConfig configs[] = {
     createBoolConfig("crash-log-enabled", NULL, MODIFIABLE_CONFIG, server.crashlog_enabled, 1, NULL, updateSighandlerEnabled),
     createBoolConfig("crash-memcheck-enabled", NULL, MODIFIABLE_CONFIG, server.memcheck_enabled, 1, NULL, NULL),
     createBoolConfig("use-exit-on-panic", NULL, MODIFIABLE_CONFIG, server.use_exit_on_panic, 0, NULL, NULL),
+    createBoolConfig("oom-score-adj", NULL, MODIFIABLE_CONFIG, server.oom_score_adj, 0, NULL, updateOOMScoreAdj),
 
     /* String Configs */
     createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, "", NULL, NULL),
diff --git a/src/config.h b/src/config.h
index 0fcc42972..e807b9330 100644
--- a/src/config.h
+++ b/src/config.h
@@ -54,6 +54,7 @@
 #define HAVE_PROC_MAPS 1
 #define HAVE_PROC_SMAPS 1
 #define HAVE_PROC_SOMAXCONN 1
+#define HAVE_PROC_OOM_SCORE_ADJ 1
 #endif
 
 /* Test for task_info() */
diff --git a/src/replication.c b/src/replication.c
index 05d450558..846f3a26b 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2497,6 +2497,9 @@ void replicationSetMaster(char *ip, int port) {
     server.masterhost = sdsnew(ip);
     server.masterport = port;
 
+    /* Update oom_score_adj */
+    setOOMScoreAdj(-1);
+
     /* Force our slaves to resync with us as well. They may hopefully be able
      * to partially resync with us, but we can notify the replid change. */
     disconnectSlaves();
@@ -2564,6 +2567,9 @@ void replicationUnsetMaster(void) {
      * master switch. */
     server.slaveseldb = -1;
 
+    /* Update oom_score_adj */
+    setOOMScoreAdj(-1);
+
     /* Once we turn from slave to master, we consider the starting time without
      * slaves (that is used to count the replication backlog time to live) as
      * starting from now. Otherwise the backlog will be freed after a
diff --git a/src/server.c b/src/server.c
index 5a9d55c0c..caf8d302d 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2422,6 +2422,10 @@ void initServerConfig(void) {
     for (j = 0; j < CLIENT_TYPE_OBUF_COUNT; j++)
         server.client_obuf_limits[j] = clientBufferLimitsDefaults[j];
 
+    /* Linux OOM Score config */
+    for (j = 0; j < CONFIG_OOM_COUNT; j++)
+        server.oom_score_adj_values[j] = configOOMScoreAdjValuesDefaults[j];
+
     /* Double constants initialization */
     R_Zero = 0.0;
     R_PosInf = 1.0/R_Zero;
@@ -2527,6 +2531,58 @@ int restartServer(int flags, mstime_t delay) {
     return C_ERR; /* Never reached. */
 }
 
+static void readOOMScoreAdj(void) {
+#ifdef HAVE_PROC_OOM_SCORE_ADJ
+    char buf[64];
+    int fd = open("/proc/self/oom_score_adj", O_RDONLY);
+
+    if (fd < 0) return;
+    if (read(fd, buf, sizeof(buf)) > 0)
+        server.oom_score_adj_base = atoi(buf);
+    close(fd);
+#endif
+}
+
+/* This function will configure the current process's oom_score_adj according
+ * to user specified configuration. This is currently implemented on Linux
+ * only.
+ *
+ * A process_class value of -1 implies OOM_CONFIG_MASTER or OOM_CONFIG_REPLICA,
+ * depending on current role.
+ */
+int setOOMScoreAdj(int process_class) {
+    int fd;
+    int val;
+    char buf[64];
+
+    if (!server.oom_score_adj) return C_OK;
+    if (process_class == -1)
+        process_class = (server.masterhost ? CONFIG_OOM_REPLICA : CONFIG_OOM_MASTER);
+
+    serverAssert(process_class >= 0 && process_class < CONFIG_OOM_COUNT);
+
+#ifdef HAVE_PROC_OOM_SCORE_ADJ
+    val = server.oom_score_adj_base + server.oom_score_adj_values[process_class];
+    if (val > 1000) val = 1000;
+    if (val < -1000) val = -1000;
+
+    snprintf(buf, sizeof(buf) - 1, "%d\n", val);
+
+    fd = open("/proc/self/oom_score_adj", O_WRONLY);
+    if (fd < 0 || write(fd, buf, strlen(buf)) < 0) {
+        serverLog(LOG_WARNING, "Unable to write oom_score_adj: %s", strerror(errno));
+        if (fd != -1) close(fd);
+        return C_ERR;
+    }
+
+    close(fd);
+    return C_OK;
+#else
+    /* Unsupported */
+    return C_ERR;
+#endif
+}
+
 /* This function will try to raise the max number of open files accordingly to
  * the configured max number of clients. It also reserves a number of file
  * descriptors (CONFIG_MIN_RESERVED_FDS) for extra operations of
@@ -4866,6 +4922,7 @@ int redisFork() {
     long long start = ustime();
     if ((childpid = fork()) == 0) {
         /* Child */
+        setOOMScoreAdj(CONFIG_OOM_BGCHILD);
         closeListeningSockets(0);
         setupChildSignalHandlers();
     } else {
@@ -5197,6 +5254,7 @@ int main(int argc, char **argv) {
     server.supervised = redisIsSupervised(server.supervised_mode);
     int background = server.daemonize && !server.supervised;
     if (background) daemonize();
+    readOOMScoreAdj();
 
     initServer();
     if (background || server.pidfile) createPidFile();
@@ -5250,6 +5308,8 @@ int main(int argc, char **argv) {
     }
 
     redisSetCpuAffinity(server.server_cpulist);
+    setOOMScoreAdj(-1);
+
     aeMain(server.el);
     aeDeleteEventLoop(server.el);
     return 0;
diff --git a/src/server.h b/src/server.h
index 7f1e7ea7b..4940b0c5b 100644
--- a/src/server.h
+++ b/src/server.h
@@ -150,6 +150,14 @@ typedef long long ustime_t; /* microsecond time type. */
  * in order to make sure of not over provisioning more than 128 fds. */
 #define CONFIG_FDSET_INCR (CONFIG_MIN_RESERVED_FDS+96)
 
+/* OOM Score Adjustment classes. */
+#define CONFIG_OOM_MASTER 0
+#define CONFIG_OOM_REPLICA 1
+#define CONFIG_OOM_BGCHILD 2
+#define CONFIG_OOM_COUNT 3
+
+extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
+
 /* Hash table parameters */
 #define HASHTABLE_MIN_FILL        10      /* Minimal hash table fill 10% */
 
@@ -1350,6 +1358,9 @@ struct redisServer {
     int lfu_log_factor;             /* LFU logarithmic counter factor. */
     int lfu_decay_time;             /* LFU counter decay factor. */
     long long proto_max_bulk_len;   /* Protocol bulk length maximum size. */
+    int oom_score_adj_base;         /* Base oom_score_adj value, as observed on startup */
+    int oom_score_adj_values[CONFIG_OOM_COUNT];   /* Linux oom_score_adj configuration */
+    int oom_score_adj;                            /* If true, oom_score_adj is managed */
     /* Blocked clients */
     unsigned int blocked_clients;   /* # of clients executing a blocking cmd.*/
     unsigned int blocked_clients_by_type[BLOCKED_NUM];
@@ -2016,6 +2027,7 @@ const char *evictPolicyToString(void);
 struct redisMemOverhead *getMemoryOverheadData(void);
 void freeMemoryOverheadData(struct redisMemOverhead *mh);
 void checkChildrenDone(void);
+int setOOMScoreAdj(int process_class);
 
 #define RESTART_SERVER_NONE 0
 #define RESTART_SERVER_GRACEFULLY (1<<0)     /* Do proper shutdown. */
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 7ce0d545e..d0f962762 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -68,6 +68,7 @@ set ::all_tests {
     unit/pendingquerybuf
     unit/tls
     unit/tracking
+    unit/oom-score-adj
 }
 # Index to the next test to run in the ::all_tests list.
 set ::next_test 0
diff --git a/tests/unit/oom-score-adj.tcl b/tests/unit/oom-score-adj.tcl
new file mode 100644
index 000000000..894a70fb2
--- /dev/null
+++ b/tests/unit/oom-score-adj.tcl
@@ -0,0 +1,81 @@
+set system_name [string tolower [exec uname -s]]
+set user_id [exec id -u]
+
+if {$system_name eq {linux}} {
+    start_server {tags {"oom-score-adj"}} {
+        proc get_oom_score_adj {{pid ""}} {
+            if {$pid == ""} {
+                set pid [srv 0 pid]
+            }
+            set fd [open "/proc/$pid/oom_score_adj" "r"]
+            set val [gets $fd]
+            close $fd
+
+            return $val
+        }
+
+        proc get_child_pid {} {
+            set pid [srv 0 pid]
+            set fd [open "|ps --ppid $pid -o pid -h" "r"]
+            set child_pid [string trim [read $fd]]
+            close $fd
+
+            return $child_pid
+        }
+
+        test {CONFIG SET oom-score-adj works as expected} {
+            set base [get_oom_score_adj]
+
+            # Enable oom-score-adj, check defaults
+            r config set oom-score-adj-values "10 20 30"
+            r config set oom-score-adj yes
+
+            assert {[get_oom_score_adj] == [expr $base + 10]}
+
+            # Modify current class
+            r config set oom-score-adj-values "15 20 30"
+            assert {[get_oom_score_adj] == [expr $base + 15]}
+
+            # Check replica class
+            r replicaof localhost 1
+            assert {[get_oom_score_adj] == [expr $base + 20]}
+            r replicaof no one
+            assert {[get_oom_score_adj] == [expr $base + 15]}
+
+            # Check child process
+            r set key-a value-a
+            r config set rdb-key-save-delay 100000
+            r bgsave
+
+            set child_pid [get_child_pid]
+            assert {[get_oom_score_adj $child_pid] == [expr $base + 30]}
+        }
+
+        # Failed oom-score-adj tests can only run unprivileged
+        if {$user_id != 0} {
+            test {CONFIG SET oom-score-adj handles configuration failures} {
+                # Bad config
+                r config set oom-score-adj no
+                r config set oom-score-adj-values "-1000 -1000 -1000"
+
+                # Make sure it fails
+                catch {r config set oom-score-adj yes} e
+                assert_match {*Failed to set*} $e
+
+                # Make sure it remains off
+                assert {[r config get oom-score-adj] == "oom-score-adj no"}
+
+                # Fix config
+                r config set oom-score-adj-values "0 100 100"
+                r config set oom-score-adj yes
+
+                # Make sure it fails
+                catch {r config set oom-score-adj-values "-1000 -1000 -1000"} e
+                assert_match {*Failed*} $e
+
+                # Make sure previous values remain
+                assert {[r config get oom-score-adj-values] == {oom-score-adj-values {0 100 100}}}
+            }
+        }
+    }
+}

From 47637bea6d12d899f69a6b384ee7b024177006de Mon Sep 17 00:00:00 2001
From: RemRain <chenyaosf@gmail.com>
Date: Thu, 13 Aug 2020 04:15:58 +0800
Subject: [PATCH 0579/1098] Set the initial seed for random() (#5679)

---
 src/server.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/server.c b/src/server.c
index caf8d302d..5c083449b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5135,6 +5135,7 @@ int main(int argc, char **argv) {
     tzset(); /* Populates 'timezone' global. */
     zmalloc_set_oom_handler(redisOutOfMemoryHandler);
     srand(time(NULL)^getpid());
+    srandom(time(NULL)^getpid());
     gettimeofday(&tv,NULL);
     crc64_init();
 

From d10b2f3173ebfcc673d4ec36efef512802e3a535 Mon Sep 17 00:00:00 2001
From: caozb <1162650653@qq.com>
Date: Thu, 13 Aug 2020 19:30:51 +0800
Subject: [PATCH 0580/1098] wait command optimization (#7333)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Client that issued WAIT last will most likely have the highest replication offset, so imagine a probably common case where all clients are waiting for the same number of replicas. we prefer the loop to start from the last client (one waiting for the highest offset), so that the optimization in the function will call replicationCountAcksByOffset for each client until it found a good one, and stop calling it for the rest of the clients.
the way the loop was implemented would mean that in such case it is likely to call replicationCountAcksByOffset for all clients.

Note: the change from > to >= is not directly related to the above.

Co-authored-by: 曹正斌 <caozb@jiedaibao.com>
---
 src/replication.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index 846f3a26b..1e1be345c 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -3068,7 +3068,7 @@ void waitCommand(client *c) {
     c->bpop.timeout = timeout;
     c->bpop.reploffset = offset;
     c->bpop.numreplicas = numreplicas;
-    listAddNodeTail(server.clients_waiting_acks,c);
+    listAddNodeHead(server.clients_waiting_acks,c);
     blockClient(c,BLOCKED_WAIT);
 
     /* Make sure that the server will send an ACK request to all the slaves
@@ -3103,8 +3103,8 @@ void processClientsWaitingReplicas(void) {
          * offset and number of replicas, we remember it so the next client
          * may be unblocked without calling replicationCountAcksByOffset()
          * if the requested offset / replicas were equal or less. */
-        if (last_offset && last_offset > c->bpop.reploffset &&
-                           last_numreplicas > c->bpop.numreplicas)
+        if (last_offset && last_offset >= c->bpop.reploffset &&
+                           last_numreplicas >= c->bpop.numreplicas)
         {
             unblockClient(c);
             addReplyLongLong(c,last_numreplicas);

From 11cd983d58199b6ac7fa54049734457bd767a0b5 Mon Sep 17 00:00:00 2001
From: Nathan Scott <natoscott@users.noreply.github.com>
Date: Fri, 14 Aug 2020 21:45:34 +1000
Subject: [PATCH 0581/1098] Annotate module API functions in redismodule.h for
 use with -fno-common (#6900)

In order to keep the redismodule.h self-contained but still usable with
gcc v10 and later, annotate each API function tentative definition with
the __common__ attribute.  This avoids the 'multiple definition' errors
modules will otherwise see for all API functions at link time.

Further details at gcc.gnu.org/gcc-10/porting_to.html

Turn the existing __attribute__ ((unused)), ((__common__)) and ((print))
annotations into conditional macros for any compilers not accepting this
syntax.  These macros only expand to API annotations under gcc.

Provide a pre- and post- macro for every API function, so that they can
be defined differently by the file that includes redismodule.h.

Removing REDISMODULE_API_FUNC in the interest of keeping the function
declarations readable.

Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/redismodule.h | 506 ++++++++++++++++++++++++----------------------
 1 file changed, 265 insertions(+), 241 deletions(-)

diff --git a/src/redismodule.h b/src/redismodule.h
index 5f828b9e3..460fdd480 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -380,6 +380,31 @@ typedef struct RedisModuleLoadingProgressInfo {
 
 typedef long long mstime_t;
 
+/* Macro definitions specific to individual compilers */
+#ifndef REDISMODULE_ATTR_UNUSED
+#    ifdef __GNUC__
+#        define REDISMODULE_ATTR_UNUSED __attribute__((unused))
+#    else
+#        define REDISMODULE_ATTR_UNUSED
+#    endif
+#endif
+
+#ifndef REDISMODULE_ATTR_PRINTF
+#    ifdef __GNUC__
+#        define REDISMODULE_ATTR_PRINTF(idx,cnt) __attribute__((format(printf,idx,cnt)))
+#    else
+#        define REDISMODULE_ATTR_PRINTF(idx,cnt)
+#    endif
+#endif
+
+#ifndef REDISMODULE_ATTR_COMMON
+#    if defined(__GNUC__) && !defined(__clang__)
+#        define REDISMODULE_ATTR_COMMON __attribute__((__common__))
+#    else
+#        define REDISMODULE_ATTR_COMMON
+#    endif
+#endif
+
 /* Incomplete structures for compiler checks but opaque access. */
 typedef struct RedisModuleCtx RedisModuleCtx;
 typedef struct RedisModuleKey RedisModuleKey;
@@ -436,257 +461,256 @@ typedef struct RedisModuleTypeMethods {
 #define REDISMODULE_GET_API(name) \
     RedisModule_GetApi("RedisModule_" #name, ((void **)&RedisModule_ ## name))
 
-#define REDISMODULE_API_FUNC(x) (*x)
-
-
-void *REDISMODULE_API_FUNC(RedisModule_Alloc)(size_t bytes);
-void *REDISMODULE_API_FUNC(RedisModule_Realloc)(void *ptr, size_t bytes);
-void REDISMODULE_API_FUNC(RedisModule_Free)(void *ptr);
-void *REDISMODULE_API_FUNC(RedisModule_Calloc)(size_t nmemb, size_t size);
-char *REDISMODULE_API_FUNC(RedisModule_Strdup)(const char *str);
-int REDISMODULE_API_FUNC(RedisModule_GetApi)(const char *, void *);
-int REDISMODULE_API_FUNC(RedisModule_CreateCommand)(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep);
-void REDISMODULE_API_FUNC(RedisModule_SetModuleAttribs)(RedisModuleCtx *ctx, const char *name, int ver, int apiver);
-int REDISMODULE_API_FUNC(RedisModule_IsModuleNameBusy)(const char *name);
-int REDISMODULE_API_FUNC(RedisModule_WrongArity)(RedisModuleCtx *ctx);
-int REDISMODULE_API_FUNC(RedisModule_ReplyWithLongLong)(RedisModuleCtx *ctx, long long ll);
-int REDISMODULE_API_FUNC(RedisModule_GetSelectedDb)(RedisModuleCtx *ctx);
-int REDISMODULE_API_FUNC(RedisModule_SelectDb)(RedisModuleCtx *ctx, int newid);
-void *REDISMODULE_API_FUNC(RedisModule_OpenKey)(RedisModuleCtx *ctx, RedisModuleString *keyname, int mode);
-void REDISMODULE_API_FUNC(RedisModule_CloseKey)(RedisModuleKey *kp);
-int REDISMODULE_API_FUNC(RedisModule_KeyType)(RedisModuleKey *kp);
-size_t REDISMODULE_API_FUNC(RedisModule_ValueLength)(RedisModuleKey *kp);
-int REDISMODULE_API_FUNC(RedisModule_ListPush)(RedisModuleKey *kp, int where, RedisModuleString *ele);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_ListPop)(RedisModuleKey *key, int where);
-RedisModuleCallReply *REDISMODULE_API_FUNC(RedisModule_Call)(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...);
-const char *REDISMODULE_API_FUNC(RedisModule_CallReplyProto)(RedisModuleCallReply *reply, size_t *len);
-void REDISMODULE_API_FUNC(RedisModule_FreeCallReply)(RedisModuleCallReply *reply);
-int REDISMODULE_API_FUNC(RedisModule_CallReplyType)(RedisModuleCallReply *reply);
-long long REDISMODULE_API_FUNC(RedisModule_CallReplyInteger)(RedisModuleCallReply *reply);
-size_t REDISMODULE_API_FUNC(RedisModule_CallReplyLength)(RedisModuleCallReply *reply);
-RedisModuleCallReply *REDISMODULE_API_FUNC(RedisModule_CallReplyArrayElement)(RedisModuleCallReply *reply, size_t idx);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateString)(RedisModuleCtx *ctx, const char *ptr, size_t len);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongLong)(RedisModuleCtx *ctx, long long ll);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromDouble)(RedisModuleCtx *ctx, double d);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongDouble)(RedisModuleCtx *ctx, long double ld, int humanfriendly);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromString)(RedisModuleCtx *ctx, const RedisModuleString *str);
-#ifdef __GNUC__
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...) __attribute__ ((format (printf, 2, 3)));
-#else
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...);
+/* Default API declaration prefix (not 'extern' for backwards compatibility) */
+#ifndef REDISMODULE_API
+#define REDISMODULE_API
 #endif
-void REDISMODULE_API_FUNC(RedisModule_FreeString)(RedisModuleCtx *ctx, RedisModuleString *str);
-const char *REDISMODULE_API_FUNC(RedisModule_StringPtrLen)(const RedisModuleString *str, size_t *len);
-int REDISMODULE_API_FUNC(RedisModule_ReplyWithError)(RedisModuleCtx *ctx, const char *err);
-int REDISMODULE_API_FUNC(RedisModule_ReplyWithSimpleString)(RedisModuleCtx *ctx, const char *msg);
-int REDISMODULE_API_FUNC(RedisModule_ReplyWithArray)(RedisModuleCtx *ctx, long len);
-int REDISMODULE_API_FUNC(RedisModule_ReplyWithNullArray)(RedisModuleCtx *ctx);
-int REDISMODULE_API_FUNC(RedisModule_ReplyWithEmptyArray)(RedisModuleCtx *ctx);
-void REDISMODULE_API_FUNC(RedisModule_ReplySetArrayLength)(RedisModuleCtx *ctx, long len);
-int REDISMODULE_API_FUNC(RedisModule_ReplyWithStringBuffer)(RedisModuleCtx *ctx, const char *buf, size_t len);
-int REDISMODULE_API_FUNC(RedisModule_ReplyWithCString)(RedisModuleCtx *ctx, const char *buf);
-int REDISMODULE_API_FUNC(RedisModule_ReplyWithString)(RedisModuleCtx *ctx, RedisModuleString *str);
-int REDISMODULE_API_FUNC(RedisModule_ReplyWithEmptyString)(RedisModuleCtx *ctx);
-int REDISMODULE_API_FUNC(RedisModule_ReplyWithVerbatimString)(RedisModuleCtx *ctx, const char *buf, size_t len);
-int REDISMODULE_API_FUNC(RedisModule_ReplyWithNull)(RedisModuleCtx *ctx);
-int REDISMODULE_API_FUNC(RedisModule_ReplyWithDouble)(RedisModuleCtx *ctx, double d);
-int REDISMODULE_API_FUNC(RedisModule_ReplyWithLongDouble)(RedisModuleCtx *ctx, long double d);
-int REDISMODULE_API_FUNC(RedisModule_ReplyWithCallReply)(RedisModuleCtx *ctx, RedisModuleCallReply *reply);
-int REDISMODULE_API_FUNC(RedisModule_StringToLongLong)(const RedisModuleString *str, long long *ll);
-int REDISMODULE_API_FUNC(RedisModule_StringToDouble)(const RedisModuleString *str, double *d);
-int REDISMODULE_API_FUNC(RedisModule_StringToLongDouble)(const RedisModuleString *str, long double *d);
-void REDISMODULE_API_FUNC(RedisModule_AutoMemory)(RedisModuleCtx *ctx);
-int REDISMODULE_API_FUNC(RedisModule_Replicate)(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...);
-int REDISMODULE_API_FUNC(RedisModule_ReplicateVerbatim)(RedisModuleCtx *ctx);
-const char *REDISMODULE_API_FUNC(RedisModule_CallReplyStringPtr)(RedisModuleCallReply *reply, size_t *len);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromCallReply)(RedisModuleCallReply *reply);
-int REDISMODULE_API_FUNC(RedisModule_DeleteKey)(RedisModuleKey *key);
-int REDISMODULE_API_FUNC(RedisModule_UnlinkKey)(RedisModuleKey *key);
-int REDISMODULE_API_FUNC(RedisModule_StringSet)(RedisModuleKey *key, RedisModuleString *str);
-char *REDISMODULE_API_FUNC(RedisModule_StringDMA)(RedisModuleKey *key, size_t *len, int mode);
-int REDISMODULE_API_FUNC(RedisModule_StringTruncate)(RedisModuleKey *key, size_t newlen);
-mstime_t REDISMODULE_API_FUNC(RedisModule_GetExpire)(RedisModuleKey *key);
-int REDISMODULE_API_FUNC(RedisModule_SetExpire)(RedisModuleKey *key, mstime_t expire);
-void REDISMODULE_API_FUNC(RedisModule_ResetDataset)(int restart_aof, int async);
-unsigned long long REDISMODULE_API_FUNC(RedisModule_DbSize)(RedisModuleCtx *ctx);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_RandomKey)(RedisModuleCtx *ctx);
-int REDISMODULE_API_FUNC(RedisModule_ZsetAdd)(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr);
-int REDISMODULE_API_FUNC(RedisModule_ZsetIncrby)(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr, double *newscore);
-int REDISMODULE_API_FUNC(RedisModule_ZsetScore)(RedisModuleKey *key, RedisModuleString *ele, double *score);
-int REDISMODULE_API_FUNC(RedisModule_ZsetRem)(RedisModuleKey *key, RedisModuleString *ele, int *deleted);
-void REDISMODULE_API_FUNC(RedisModule_ZsetRangeStop)(RedisModuleKey *key);
-int REDISMODULE_API_FUNC(RedisModule_ZsetFirstInScoreRange)(RedisModuleKey *key, double min, double max, int minex, int maxex);
-int REDISMODULE_API_FUNC(RedisModule_ZsetLastInScoreRange)(RedisModuleKey *key, double min, double max, int minex, int maxex);
-int REDISMODULE_API_FUNC(RedisModule_ZsetFirstInLexRange)(RedisModuleKey *key, RedisModuleString *min, RedisModuleString *max);
-int REDISMODULE_API_FUNC(RedisModule_ZsetLastInLexRange)(RedisModuleKey *key, RedisModuleString *min, RedisModuleString *max);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_ZsetRangeCurrentElement)(RedisModuleKey *key, double *score);
-int REDISMODULE_API_FUNC(RedisModule_ZsetRangeNext)(RedisModuleKey *key);
-int REDISMODULE_API_FUNC(RedisModule_ZsetRangePrev)(RedisModuleKey *key);
-int REDISMODULE_API_FUNC(RedisModule_ZsetRangeEndReached)(RedisModuleKey *key);
-int REDISMODULE_API_FUNC(RedisModule_HashSet)(RedisModuleKey *key, int flags, ...);
-int REDISMODULE_API_FUNC(RedisModule_HashGet)(RedisModuleKey *key, int flags, ...);
-int REDISMODULE_API_FUNC(RedisModule_IsKeysPositionRequest)(RedisModuleCtx *ctx);
-void REDISMODULE_API_FUNC(RedisModule_KeyAtPos)(RedisModuleCtx *ctx, int pos);
-unsigned long long REDISMODULE_API_FUNC(RedisModule_GetClientId)(RedisModuleCtx *ctx);
-int REDISMODULE_API_FUNC(RedisModule_GetClientInfoById)(void *ci, uint64_t id);
-int REDISMODULE_API_FUNC(RedisModule_PublishMessage)(RedisModuleCtx *ctx, RedisModuleString *channel, RedisModuleString *message);
-int REDISMODULE_API_FUNC(RedisModule_GetContextFlags)(RedisModuleCtx *ctx);
-int REDISMODULE_API_FUNC(RedisModule_AvoidReplicaTraffic)();
-void *REDISMODULE_API_FUNC(RedisModule_PoolAlloc)(RedisModuleCtx *ctx, size_t bytes);
-RedisModuleType *REDISMODULE_API_FUNC(RedisModule_CreateDataType)(RedisModuleCtx *ctx, const char *name, int encver, RedisModuleTypeMethods *typemethods);
-int REDISMODULE_API_FUNC(RedisModule_ModuleTypeSetValue)(RedisModuleKey *key, RedisModuleType *mt, void *value);
-int REDISMODULE_API_FUNC(RedisModule_ModuleTypeReplaceValue)(RedisModuleKey *key, RedisModuleType *mt, void *new_value, void **old_value);
-RedisModuleType *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetType)(RedisModuleKey *key);
-void *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetValue)(RedisModuleKey *key);
-int REDISMODULE_API_FUNC(RedisModule_IsIOError)(RedisModuleIO *io);
-void REDISMODULE_API_FUNC(RedisModule_SetModuleOptions)(RedisModuleCtx *ctx, int options);
-int REDISMODULE_API_FUNC(RedisModule_SignalModifiedKey)(RedisModuleCtx *ctx, RedisModuleString *keyname);
-void REDISMODULE_API_FUNC(RedisModule_SaveUnsigned)(RedisModuleIO *io, uint64_t value);
-uint64_t REDISMODULE_API_FUNC(RedisModule_LoadUnsigned)(RedisModuleIO *io);
-void REDISMODULE_API_FUNC(RedisModule_SaveSigned)(RedisModuleIO *io, int64_t value);
-int64_t REDISMODULE_API_FUNC(RedisModule_LoadSigned)(RedisModuleIO *io);
-void REDISMODULE_API_FUNC(RedisModule_EmitAOF)(RedisModuleIO *io, const char *cmdname, const char *fmt, ...);
-void REDISMODULE_API_FUNC(RedisModule_SaveString)(RedisModuleIO *io, RedisModuleString *s);
-void REDISMODULE_API_FUNC(RedisModule_SaveStringBuffer)(RedisModuleIO *io, const char *str, size_t len);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_LoadString)(RedisModuleIO *io);
-char *REDISMODULE_API_FUNC(RedisModule_LoadStringBuffer)(RedisModuleIO *io, size_t *lenptr);
-void REDISMODULE_API_FUNC(RedisModule_SaveDouble)(RedisModuleIO *io, double value);
-double REDISMODULE_API_FUNC(RedisModule_LoadDouble)(RedisModuleIO *io);
-void REDISMODULE_API_FUNC(RedisModule_SaveFloat)(RedisModuleIO *io, float value);
-float REDISMODULE_API_FUNC(RedisModule_LoadFloat)(RedisModuleIO *io);
-void REDISMODULE_API_FUNC(RedisModule_SaveLongDouble)(RedisModuleIO *io, long double value);
-long double REDISMODULE_API_FUNC(RedisModule_LoadLongDouble)(RedisModuleIO *io);
-void *REDISMODULE_API_FUNC(RedisModule_LoadDataTypeFromString)(const RedisModuleString *str, const RedisModuleType *mt);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_SaveDataTypeToString)(RedisModuleCtx *ctx, void *data, const RedisModuleType *mt);
-#ifdef __GNUC__
-void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...) __attribute__ ((format (printf, 3, 4)));
-void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...) __attribute__ ((format (printf, 3, 4)));
-#else
-void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...);
-void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...);
+
+/* Default API declaration suffix (compiler attributes) */
+#ifndef REDISMODULE_ATTR
+#define REDISMODULE_ATTR REDISMODULE_ATTR_COMMON
 #endif
-void REDISMODULE_API_FUNC(RedisModule__Assert)(const char *estr, const char *file, int line);
-void REDISMODULE_API_FUNC(RedisModule_LatencyAddSample)(const char *event, mstime_t latency);
-int REDISMODULE_API_FUNC(RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len);
-void REDISMODULE_API_FUNC(RedisModule_RetainString)(RedisModuleCtx *ctx, RedisModuleString *str);
-RedisModuleString* REDISMODULE_API_FUNC(RedisModule_HoldString)(RedisModuleCtx *ctx, RedisModuleString *str);
-int REDISMODULE_API_FUNC(RedisModule_StringCompare)(RedisModuleString *a, RedisModuleString *b);
-RedisModuleCtx *REDISMODULE_API_FUNC(RedisModule_GetContextFromIO)(RedisModuleIO *io);
-const RedisModuleString *REDISMODULE_API_FUNC(RedisModule_GetKeyNameFromIO)(RedisModuleIO *io);
-const RedisModuleString *REDISMODULE_API_FUNC(RedisModule_GetKeyNameFromModuleKey)(RedisModuleKey *key);
-long long REDISMODULE_API_FUNC(RedisModule_Milliseconds)(void);
-void REDISMODULE_API_FUNC(RedisModule_DigestAddStringBuffer)(RedisModuleDigest *md, unsigned char *ele, size_t len);
-void REDISMODULE_API_FUNC(RedisModule_DigestAddLongLong)(RedisModuleDigest *md, long long ele);
-void REDISMODULE_API_FUNC(RedisModule_DigestEndSequence)(RedisModuleDigest *md);
-RedisModuleDict *REDISMODULE_API_FUNC(RedisModule_CreateDict)(RedisModuleCtx *ctx);
-void REDISMODULE_API_FUNC(RedisModule_FreeDict)(RedisModuleCtx *ctx, RedisModuleDict *d);
-uint64_t REDISMODULE_API_FUNC(RedisModule_DictSize)(RedisModuleDict *d);
-int REDISMODULE_API_FUNC(RedisModule_DictSetC)(RedisModuleDict *d, void *key, size_t keylen, void *ptr);
-int REDISMODULE_API_FUNC(RedisModule_DictReplaceC)(RedisModuleDict *d, void *key, size_t keylen, void *ptr);
-int REDISMODULE_API_FUNC(RedisModule_DictSet)(RedisModuleDict *d, RedisModuleString *key, void *ptr);
-int REDISMODULE_API_FUNC(RedisModule_DictReplace)(RedisModuleDict *d, RedisModuleString *key, void *ptr);
-void *REDISMODULE_API_FUNC(RedisModule_DictGetC)(RedisModuleDict *d, void *key, size_t keylen, int *nokey);
-void *REDISMODULE_API_FUNC(RedisModule_DictGet)(RedisModuleDict *d, RedisModuleString *key, int *nokey);
-int REDISMODULE_API_FUNC(RedisModule_DictDelC)(RedisModuleDict *d, void *key, size_t keylen, void *oldval);
-int REDISMODULE_API_FUNC(RedisModule_DictDel)(RedisModuleDict *d, RedisModuleString *key, void *oldval);
-RedisModuleDictIter *REDISMODULE_API_FUNC(RedisModule_DictIteratorStartC)(RedisModuleDict *d, const char *op, void *key, size_t keylen);
-RedisModuleDictIter *REDISMODULE_API_FUNC(RedisModule_DictIteratorStart)(RedisModuleDict *d, const char *op, RedisModuleString *key);
-void REDISMODULE_API_FUNC(RedisModule_DictIteratorStop)(RedisModuleDictIter *di);
-int REDISMODULE_API_FUNC(RedisModule_DictIteratorReseekC)(RedisModuleDictIter *di, const char *op, void *key, size_t keylen);
-int REDISMODULE_API_FUNC(RedisModule_DictIteratorReseek)(RedisModuleDictIter *di, const char *op, RedisModuleString *key);
-void *REDISMODULE_API_FUNC(RedisModule_DictNextC)(RedisModuleDictIter *di, size_t *keylen, void **dataptr);
-void *REDISMODULE_API_FUNC(RedisModule_DictPrevC)(RedisModuleDictIter *di, size_t *keylen, void **dataptr);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_DictNext)(RedisModuleCtx *ctx, RedisModuleDictIter *di, void **dataptr);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_DictPrev)(RedisModuleCtx *ctx, RedisModuleDictIter *di, void **dataptr);
-int REDISMODULE_API_FUNC(RedisModule_DictCompareC)(RedisModuleDictIter *di, const char *op, void *key, size_t keylen);
-int REDISMODULE_API_FUNC(RedisModule_DictCompare)(RedisModuleDictIter *di, const char *op, RedisModuleString *key);
-int REDISMODULE_API_FUNC(RedisModule_RegisterInfoFunc)(RedisModuleCtx *ctx, RedisModuleInfoFunc cb);
-int REDISMODULE_API_FUNC(RedisModule_InfoAddSection)(RedisModuleInfoCtx *ctx, char *name);
-int REDISMODULE_API_FUNC(RedisModule_InfoBeginDictField)(RedisModuleInfoCtx *ctx, char *name);
-int REDISMODULE_API_FUNC(RedisModule_InfoEndDictField)(RedisModuleInfoCtx *ctx);
-int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldString)(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value);
-int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldCString)(RedisModuleInfoCtx *ctx, char *field, char *value);
-int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldDouble)(RedisModuleInfoCtx *ctx, char *field, double value);
-int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldLongLong)(RedisModuleInfoCtx *ctx, char *field, long long value);
-int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldULongLong)(RedisModuleInfoCtx *ctx, char *field, unsigned long long value);
-RedisModuleServerInfoData *REDISMODULE_API_FUNC(RedisModule_GetServerInfo)(RedisModuleCtx *ctx, const char *section);
-void REDISMODULE_API_FUNC(RedisModule_FreeServerInfo)(RedisModuleCtx *ctx, RedisModuleServerInfoData *data);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_ServerInfoGetField)(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field);
-const char *REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldC)(RedisModuleServerInfoData *data, const char* field);
-long long REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldSigned)(RedisModuleServerInfoData *data, const char* field, int *out_err);
-unsigned long long REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldUnsigned)(RedisModuleServerInfoData *data, const char* field, int *out_err);
-double REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldDouble)(RedisModuleServerInfoData *data, const char* field, int *out_err);
-int REDISMODULE_API_FUNC(RedisModule_SubscribeToServerEvent)(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback);
-int REDISMODULE_API_FUNC(RedisModule_SetLRU)(RedisModuleKey *key, mstime_t lru_idle);
-int REDISMODULE_API_FUNC(RedisModule_GetLRU)(RedisModuleKey *key, mstime_t *lru_idle);
-int REDISMODULE_API_FUNC(RedisModule_SetLFU)(RedisModuleKey *key, long long lfu_freq);
-int REDISMODULE_API_FUNC(RedisModule_GetLFU)(RedisModuleKey *key, long long *lfu_freq);
-RedisModuleBlockedClient *REDISMODULE_API_FUNC(RedisModule_BlockClientOnKeys)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata);
-void REDISMODULE_API_FUNC(RedisModule_SignalKeyAsReady)(RedisModuleCtx *ctx, RedisModuleString *key);
-RedisModuleString *REDISMODULE_API_FUNC(RedisModule_GetBlockedClientReadyKey)(RedisModuleCtx *ctx);
-RedisModuleScanCursor *REDISMODULE_API_FUNC(RedisModule_ScanCursorCreate)();
-void REDISMODULE_API_FUNC(RedisModule_ScanCursorRestart)(RedisModuleScanCursor *cursor);
-void REDISMODULE_API_FUNC(RedisModule_ScanCursorDestroy)(RedisModuleScanCursor *cursor);
-int REDISMODULE_API_FUNC(RedisModule_Scan)(RedisModuleCtx *ctx, RedisModuleScanCursor *cursor, RedisModuleScanCB fn, void *privdata);
-int REDISMODULE_API_FUNC(RedisModule_ScanKey)(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleScanKeyCB fn, void *privdata);
+
+REDISMODULE_API void * (*RedisModule_Alloc)(size_t bytes) REDISMODULE_ATTR;
+REDISMODULE_API void * (*RedisModule_Realloc)(void *ptr, size_t bytes) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_Free)(void *ptr) REDISMODULE_ATTR;
+REDISMODULE_API void * (*RedisModule_Calloc)(size_t nmemb, size_t size) REDISMODULE_ATTR;
+REDISMODULE_API char * (*RedisModule_Strdup)(const char *str) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_GetApi)(const char *, void *) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_CreateCommand)(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_SetModuleAttribs)(RedisModuleCtx *ctx, const char *name, int ver, int apiver) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_IsModuleNameBusy)(const char *name) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_WrongArity)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplyWithLongLong)(RedisModuleCtx *ctx, long long ll) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_GetSelectedDb)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_SelectDb)(RedisModuleCtx *ctx, int newid) REDISMODULE_ATTR;
+REDISMODULE_API void * (*RedisModule_OpenKey)(RedisModuleCtx *ctx, RedisModuleString *keyname, int mode) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_CloseKey)(RedisModuleKey *kp) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_KeyType)(RedisModuleKey *kp) REDISMODULE_ATTR;
+REDISMODULE_API size_t (*RedisModule_ValueLength)(RedisModuleKey *kp) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ListPush)(RedisModuleKey *kp, int where, RedisModuleString *ele) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_ListPop)(RedisModuleKey *key, int where) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleCallReply * (*RedisModule_Call)(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) REDISMODULE_ATTR;
+REDISMODULE_API const char * (*RedisModule_CallReplyProto)(RedisModuleCallReply *reply, size_t *len) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_FreeCallReply)(RedisModuleCallReply *reply) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_CallReplyType)(RedisModuleCallReply *reply) REDISMODULE_ATTR;
+REDISMODULE_API long long (*RedisModule_CallReplyInteger)(RedisModuleCallReply *reply) REDISMODULE_ATTR;
+REDISMODULE_API size_t (*RedisModule_CallReplyLength)(RedisModuleCallReply *reply) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleCallReply * (*RedisModule_CallReplyArrayElement)(RedisModuleCallReply *reply, size_t idx) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_CreateString)(RedisModuleCtx *ctx, const char *ptr, size_t len) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_CreateStringFromLongLong)(RedisModuleCtx *ctx, long long ll) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_CreateStringFromDouble)(RedisModuleCtx *ctx, double d) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_CreateStringFromLongDouble)(RedisModuleCtx *ctx, long double ld, int humanfriendly) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_CreateStringFromString)(RedisModuleCtx *ctx, const RedisModuleString *str) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...) REDISMODULE_ATTR_PRINTF(2,3) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_FreeString)(RedisModuleCtx *ctx, RedisModuleString *str) REDISMODULE_ATTR;
+REDISMODULE_API const char * (*RedisModule_StringPtrLen)(const RedisModuleString *str, size_t *len) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplyWithError)(RedisModuleCtx *ctx, const char *err) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplyWithSimpleString)(RedisModuleCtx *ctx, const char *msg) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplyWithArray)(RedisModuleCtx *ctx, long len) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplyWithNullArray)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplyWithEmptyArray)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_ReplySetArrayLength)(RedisModuleCtx *ctx, long len) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplyWithStringBuffer)(RedisModuleCtx *ctx, const char *buf, size_t len) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplyWithCString)(RedisModuleCtx *ctx, const char *buf) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplyWithString)(RedisModuleCtx *ctx, RedisModuleString *str) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplyWithEmptyString)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplyWithVerbatimString)(RedisModuleCtx *ctx, const char *buf, size_t len) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplyWithNull)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplyWithDouble)(RedisModuleCtx *ctx, double d) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplyWithLongDouble)(RedisModuleCtx *ctx, long double d) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplyWithCallReply)(RedisModuleCtx *ctx, RedisModuleCallReply *reply) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StringToLongLong)(const RedisModuleString *str, long long *ll) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StringToDouble)(const RedisModuleString *str, double *d) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StringToLongDouble)(const RedisModuleString *str, long double *d) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_AutoMemory)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_Replicate)(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ReplicateVerbatim)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API const char * (*RedisModule_CallReplyStringPtr)(RedisModuleCallReply *reply, size_t *len) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_CreateStringFromCallReply)(RedisModuleCallReply *reply) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_DeleteKey)(RedisModuleKey *key) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_UnlinkKey)(RedisModuleKey *key) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StringSet)(RedisModuleKey *key, RedisModuleString *str) REDISMODULE_ATTR;
+REDISMODULE_API char * (*RedisModule_StringDMA)(RedisModuleKey *key, size_t *len, int mode) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StringTruncate)(RedisModuleKey *key, size_t newlen) REDISMODULE_ATTR;
+REDISMODULE_API mstime_t (*RedisModule_GetExpire)(RedisModuleKey *key) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_SetExpire)(RedisModuleKey *key, mstime_t expire) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_ResetDataset)(int restart_aof, int async) REDISMODULE_ATTR;
+REDISMODULE_API unsigned long long (*RedisModule_DbSize)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_RandomKey)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ZsetAdd)(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ZsetIncrby)(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr, double *newscore) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ZsetScore)(RedisModuleKey *key, RedisModuleString *ele, double *score) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ZsetRem)(RedisModuleKey *key, RedisModuleString *ele, int *deleted) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_ZsetRangeStop)(RedisModuleKey *key) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ZsetFirstInScoreRange)(RedisModuleKey *key, double min, double max, int minex, int maxex) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ZsetLastInScoreRange)(RedisModuleKey *key, double min, double max, int minex, int maxex) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ZsetFirstInLexRange)(RedisModuleKey *key, RedisModuleString *min, RedisModuleString *max) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ZsetLastInLexRange)(RedisModuleKey *key, RedisModuleString *min, RedisModuleString *max) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_ZsetRangeCurrentElement)(RedisModuleKey *key, double *score) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ZsetRangeNext)(RedisModuleKey *key) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ZsetRangePrev)(RedisModuleKey *key) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ZsetRangeEndReached)(RedisModuleKey *key) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_HashSet)(RedisModuleKey *key, int flags, ...) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_HashGet)(RedisModuleKey *key, int flags, ...) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_IsKeysPositionRequest)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_KeyAtPos)(RedisModuleCtx *ctx, int pos) REDISMODULE_ATTR;
+REDISMODULE_API unsigned long long (*RedisModule_GetClientId)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_GetClientInfoById)(void *ci, uint64_t id) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_PublishMessage)(RedisModuleCtx *ctx, RedisModuleString *channel, RedisModuleString *message) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_GetContextFlags)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_AvoidReplicaTraffic)() REDISMODULE_ATTR;
+REDISMODULE_API void * (*RedisModule_PoolAlloc)(RedisModuleCtx *ctx, size_t bytes) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleType * (*RedisModule_CreateDataType)(RedisModuleCtx *ctx, const char *name, int encver, RedisModuleTypeMethods *typemethods) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ModuleTypeSetValue)(RedisModuleKey *key, RedisModuleType *mt, void *value) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ModuleTypeReplaceValue)(RedisModuleKey *key, RedisModuleType *mt, void *new_value, void **old_value) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleType * (*RedisModule_ModuleTypeGetType)(RedisModuleKey *key) REDISMODULE_ATTR;
+REDISMODULE_API void * (*RedisModule_ModuleTypeGetValue)(RedisModuleKey *key) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_IsIOError)(RedisModuleIO *io) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_SetModuleOptions)(RedisModuleCtx *ctx, int options) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_SignalModifiedKey)(RedisModuleCtx *ctx, RedisModuleString *keyname) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_SaveUnsigned)(RedisModuleIO *io, uint64_t value) REDISMODULE_ATTR;
+REDISMODULE_API uint64_t (*RedisModule_LoadUnsigned)(RedisModuleIO *io) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_SaveSigned)(RedisModuleIO *io, int64_t value) REDISMODULE_ATTR;
+REDISMODULE_API int64_t (*RedisModule_LoadSigned)(RedisModuleIO *io) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_EmitAOF)(RedisModuleIO *io, const char *cmdname, const char *fmt, ...) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_SaveString)(RedisModuleIO *io, RedisModuleString *s) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_SaveStringBuffer)(RedisModuleIO *io, const char *str, size_t len) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_LoadString)(RedisModuleIO *io) REDISMODULE_ATTR;
+REDISMODULE_API char * (*RedisModule_LoadStringBuffer)(RedisModuleIO *io, size_t *lenptr) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_SaveDouble)(RedisModuleIO *io, double value) REDISMODULE_ATTR;
+REDISMODULE_API double (*RedisModule_LoadDouble)(RedisModuleIO *io) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_SaveFloat)(RedisModuleIO *io, float value) REDISMODULE_ATTR;
+REDISMODULE_API float (*RedisModule_LoadFloat)(RedisModuleIO *io) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_SaveLongDouble)(RedisModuleIO *io, long double value) REDISMODULE_ATTR;
+REDISMODULE_API long double (*RedisModule_LoadLongDouble)(RedisModuleIO *io) REDISMODULE_ATTR;
+REDISMODULE_API void * (*RedisModule_LoadDataTypeFromString)(const RedisModuleString *str, const RedisModuleType *mt) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_SaveDataTypeToString)(RedisModuleCtx *ctx, void *data, const RedisModuleType *mt) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...) REDISMODULE_ATTR REDISMODULE_ATTR_PRINTF(3,4);
+REDISMODULE_API void (*RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...) REDISMODULE_ATTR REDISMODULE_ATTR_PRINTF(3,4);
+REDISMODULE_API void (*RedisModule__Assert)(const char *estr, const char *file, int line) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_LatencyAddSample)(const char *event, mstime_t latency) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_RetainString)(RedisModuleCtx *ctx, RedisModuleString *str) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_HoldString)(RedisModuleCtx *ctx, RedisModuleString *str) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StringCompare)(RedisModuleString *a, RedisModuleString *b) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleCtx * (*RedisModule_GetContextFromIO)(RedisModuleIO *io) REDISMODULE_ATTR;
+REDISMODULE_API const RedisModuleString * (*RedisModule_GetKeyNameFromIO)(RedisModuleIO *io) REDISMODULE_ATTR;
+REDISMODULE_API const RedisModuleString * (*RedisModule_GetKeyNameFromModuleKey)(RedisModuleKey *key) REDISMODULE_ATTR;
+REDISMODULE_API long long (*RedisModule_Milliseconds)(void) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_DigestAddStringBuffer)(RedisModuleDigest *md, unsigned char *ele, size_t len) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_DigestAddLongLong)(RedisModuleDigest *md, long long ele) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_DigestEndSequence)(RedisModuleDigest *md) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleDict * (*RedisModule_CreateDict)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_FreeDict)(RedisModuleCtx *ctx, RedisModuleDict *d) REDISMODULE_ATTR;
+REDISMODULE_API uint64_t (*RedisModule_DictSize)(RedisModuleDict *d) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_DictSetC)(RedisModuleDict *d, void *key, size_t keylen, void *ptr) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_DictReplaceC)(RedisModuleDict *d, void *key, size_t keylen, void *ptr) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_DictSet)(RedisModuleDict *d, RedisModuleString *key, void *ptr) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_DictReplace)(RedisModuleDict *d, RedisModuleString *key, void *ptr) REDISMODULE_ATTR;
+REDISMODULE_API void * (*RedisModule_DictGetC)(RedisModuleDict *d, void *key, size_t keylen, int *nokey) REDISMODULE_ATTR;
+REDISMODULE_API void * (*RedisModule_DictGet)(RedisModuleDict *d, RedisModuleString *key, int *nokey) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_DictDelC)(RedisModuleDict *d, void *key, size_t keylen, void *oldval) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_DictDel)(RedisModuleDict *d, RedisModuleString *key, void *oldval) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleDictIter * (*RedisModule_DictIteratorStartC)(RedisModuleDict *d, const char *op, void *key, size_t keylen) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleDictIter * (*RedisModule_DictIteratorStart)(RedisModuleDict *d, const char *op, RedisModuleString *key) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_DictIteratorStop)(RedisModuleDictIter *di) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_DictIteratorReseekC)(RedisModuleDictIter *di, const char *op, void *key, size_t keylen) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_DictIteratorReseek)(RedisModuleDictIter *di, const char *op, RedisModuleString *key) REDISMODULE_ATTR;
+REDISMODULE_API void * (*RedisModule_DictNextC)(RedisModuleDictIter *di, size_t *keylen, void **dataptr) REDISMODULE_ATTR;
+REDISMODULE_API void * (*RedisModule_DictPrevC)(RedisModuleDictIter *di, size_t *keylen, void **dataptr) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_DictNext)(RedisModuleCtx *ctx, RedisModuleDictIter *di, void **dataptr) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_DictPrev)(RedisModuleCtx *ctx, RedisModuleDictIter *di, void **dataptr) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_DictCompareC)(RedisModuleDictIter *di, const char *op, void *key, size_t keylen) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_DictCompare)(RedisModuleDictIter *di, const char *op, RedisModuleString *key) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_RegisterInfoFunc)(RedisModuleCtx *ctx, RedisModuleInfoFunc cb) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_InfoAddSection)(RedisModuleInfoCtx *ctx, char *name) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_InfoBeginDictField)(RedisModuleInfoCtx *ctx, char *name) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_InfoEndDictField)(RedisModuleInfoCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_InfoAddFieldString)(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_InfoAddFieldCString)(RedisModuleInfoCtx *ctx, char *field, char *value) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_InfoAddFieldDouble)(RedisModuleInfoCtx *ctx, char *field, double value) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_InfoAddFieldLongLong)(RedisModuleInfoCtx *ctx, char *field, long long value) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_InfoAddFieldULongLong)(RedisModuleInfoCtx *ctx, char *field, unsigned long long value) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleServerInfoData * (*RedisModule_GetServerInfo)(RedisModuleCtx *ctx, const char *section) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_FreeServerInfo)(RedisModuleCtx *ctx, RedisModuleServerInfoData *data) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_ServerInfoGetField)(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field) REDISMODULE_ATTR;
+REDISMODULE_API const char * (*RedisModule_ServerInfoGetFieldC)(RedisModuleServerInfoData *data, const char* field) REDISMODULE_ATTR;
+REDISMODULE_API long long (*RedisModule_ServerInfoGetFieldSigned)(RedisModuleServerInfoData *data, const char* field, int *out_err) REDISMODULE_ATTR;
+REDISMODULE_API unsigned long long (*RedisModule_ServerInfoGetFieldUnsigned)(RedisModuleServerInfoData *data, const char* field, int *out_err) REDISMODULE_ATTR;
+REDISMODULE_API double (*RedisModule_ServerInfoGetFieldDouble)(RedisModuleServerInfoData *data, const char* field, int *out_err) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_SubscribeToServerEvent)(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_SetLRU)(RedisModuleKey *key, mstime_t lru_idle) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_GetLRU)(RedisModuleKey *key, mstime_t *lru_idle) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_SetLFU)(RedisModuleKey *key, long long lfu_freq) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_GetLFU)(RedisModuleKey *key, long long *lfu_freq) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleBlockedClient * (*RedisModule_BlockClientOnKeys)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_SignalKeyAsReady)(RedisModuleCtx *ctx, RedisModuleString *key) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_GetBlockedClientReadyKey)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleScanCursor * (*RedisModule_ScanCursorCreate)() REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_ScanCursorRestart)(RedisModuleScanCursor *cursor) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_ScanCursorDestroy)(RedisModuleScanCursor *cursor) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_Scan)(RedisModuleCtx *ctx, RedisModuleScanCursor *cursor, RedisModuleScanCB fn, void *privdata) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ScanKey)(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleScanKeyCB fn, void *privdata) REDISMODULE_ATTR;
+
 /* Experimental APIs */
 #ifdef REDISMODULE_EXPERIMENTAL_API
 #define REDISMODULE_EXPERIMENTAL_API_VERSION 3
-RedisModuleBlockedClient *REDISMODULE_API_FUNC(RedisModule_BlockClient)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms);
-int REDISMODULE_API_FUNC(RedisModule_UnblockClient)(RedisModuleBlockedClient *bc, void *privdata);
-int REDISMODULE_API_FUNC(RedisModule_IsBlockedReplyRequest)(RedisModuleCtx *ctx);
-int REDISMODULE_API_FUNC(RedisModule_IsBlockedTimeoutRequest)(RedisModuleCtx *ctx);
-void *REDISMODULE_API_FUNC(RedisModule_GetBlockedClientPrivateData)(RedisModuleCtx *ctx);
-RedisModuleBlockedClient *REDISMODULE_API_FUNC(RedisModule_GetBlockedClientHandle)(RedisModuleCtx *ctx);
-int REDISMODULE_API_FUNC(RedisModule_AbortBlock)(RedisModuleBlockedClient *bc);
-RedisModuleCtx *REDISMODULE_API_FUNC(RedisModule_GetThreadSafeContext)(RedisModuleBlockedClient *bc);
-void REDISMODULE_API_FUNC(RedisModule_FreeThreadSafeContext)(RedisModuleCtx *ctx);
-void REDISMODULE_API_FUNC(RedisModule_ThreadSafeContextLock)(RedisModuleCtx *ctx);
-void REDISMODULE_API_FUNC(RedisModule_ThreadSafeContextUnlock)(RedisModuleCtx *ctx);
-int REDISMODULE_API_FUNC(RedisModule_SubscribeToKeyspaceEvents)(RedisModuleCtx *ctx, int types, RedisModuleNotificationFunc cb);
-int REDISMODULE_API_FUNC(RedisModule_NotifyKeyspaceEvent)(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key);
-int REDISMODULE_API_FUNC(RedisModule_GetNotifyKeyspaceEvents)();
-int REDISMODULE_API_FUNC(RedisModule_BlockedClientDisconnected)(RedisModuleCtx *ctx);
-void REDISMODULE_API_FUNC(RedisModule_RegisterClusterMessageReceiver)(RedisModuleCtx *ctx, uint8_t type, RedisModuleClusterMessageReceiver callback);
-int REDISMODULE_API_FUNC(RedisModule_SendClusterMessage)(RedisModuleCtx *ctx, char *target_id, uint8_t type, unsigned char *msg, uint32_t len);
-int REDISMODULE_API_FUNC(RedisModule_GetClusterNodeInfo)(RedisModuleCtx *ctx, const char *id, char *ip, char *master_id, int *port, int *flags);
-char **REDISMODULE_API_FUNC(RedisModule_GetClusterNodesList)(RedisModuleCtx *ctx, size_t *numnodes);
-void REDISMODULE_API_FUNC(RedisModule_FreeClusterNodesList)(char **ids);
-RedisModuleTimerID REDISMODULE_API_FUNC(RedisModule_CreateTimer)(RedisModuleCtx *ctx, mstime_t period, RedisModuleTimerProc callback, void *data);
-int REDISMODULE_API_FUNC(RedisModule_StopTimer)(RedisModuleCtx *ctx, RedisModuleTimerID id, void **data);
-int REDISMODULE_API_FUNC(RedisModule_GetTimerInfo)(RedisModuleCtx *ctx, RedisModuleTimerID id, uint64_t *remaining, void **data);
-const char *REDISMODULE_API_FUNC(RedisModule_GetMyClusterID)(void);
-size_t REDISMODULE_API_FUNC(RedisModule_GetClusterSize)(void);
-void REDISMODULE_API_FUNC(RedisModule_GetRandomBytes)(unsigned char *dst, size_t len);
-void REDISMODULE_API_FUNC(RedisModule_GetRandomHexChars)(char *dst, size_t len);
-void REDISMODULE_API_FUNC(RedisModule_SetDisconnectCallback)(RedisModuleBlockedClient *bc, RedisModuleDisconnectFunc callback);
-void REDISMODULE_API_FUNC(RedisModule_SetClusterFlags)(RedisModuleCtx *ctx, uint64_t flags);
-int REDISMODULE_API_FUNC(RedisModule_ExportSharedAPI)(RedisModuleCtx *ctx, const char *apiname, void *func);
-void *REDISMODULE_API_FUNC(RedisModule_GetSharedAPI)(RedisModuleCtx *ctx, const char *apiname);
-RedisModuleCommandFilter *REDISMODULE_API_FUNC(RedisModule_RegisterCommandFilter)(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc cb, int flags);
-int REDISMODULE_API_FUNC(RedisModule_UnregisterCommandFilter)(RedisModuleCtx *ctx, RedisModuleCommandFilter *filter);
-int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgsCount)(RedisModuleCommandFilterCtx *fctx);
-const RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CommandFilterArgGet)(RedisModuleCommandFilterCtx *fctx, int pos);
-int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgInsert)(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg);
-int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgReplace)(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg);
-int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgDelete)(RedisModuleCommandFilterCtx *fctx, int pos);
-int REDISMODULE_API_FUNC(RedisModule_Fork)(RedisModuleForkDoneHandler cb, void *user_data);
-int REDISMODULE_API_FUNC(RedisModule_ExitFromChild)(int retcode);
-int REDISMODULE_API_FUNC(RedisModule_KillForkChild)(int child_pid);
-float REDISMODULE_API_FUNC(RedisModule_GetUsedMemoryRatio)();
-size_t REDISMODULE_API_FUNC(RedisModule_MallocSize)(void* ptr);
-RedisModuleUser *REDISMODULE_API_FUNC(RedisModule_CreateModuleUser)(const char *name);
-void REDISMODULE_API_FUNC(RedisModule_FreeModuleUser)(RedisModuleUser *user);
-int REDISMODULE_API_FUNC(RedisModule_SetModuleUserACL)(RedisModuleUser *user, const char* acl);
-int REDISMODULE_API_FUNC(RedisModule_AuthenticateClientWithACLUser)(RedisModuleCtx *ctx, const char *name, size_t len, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id);
-int REDISMODULE_API_FUNC(RedisModule_AuthenticateClientWithUser)(RedisModuleCtx *ctx, RedisModuleUser *user, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id);
-int REDISMODULE_API_FUNC(RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id);
+REDISMODULE_API RedisModuleBlockedClient * (*RedisModule_BlockClient)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_UnblockClient)(RedisModuleBlockedClient *bc, void *privdata) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_IsBlockedReplyRequest)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_IsBlockedTimeoutRequest)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API void * (*RedisModule_GetBlockedClientPrivateData)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleBlockedClient * (*RedisModule_GetBlockedClientHandle)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_AbortBlock)(RedisModuleBlockedClient *bc) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleCtx * (*RedisModule_GetThreadSafeContext)(RedisModuleBlockedClient *bc) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_FreeThreadSafeContext)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_ThreadSafeContextLock)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_ThreadSafeContextUnlock)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_SubscribeToKeyspaceEvents)(RedisModuleCtx *ctx, int types, RedisModuleNotificationFunc cb) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_NotifyKeyspaceEvent)(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_GetNotifyKeyspaceEvents)() REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_BlockedClientDisconnected)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_RegisterClusterMessageReceiver)(RedisModuleCtx *ctx, uint8_t type, RedisModuleClusterMessageReceiver callback) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_SendClusterMessage)(RedisModuleCtx *ctx, char *target_id, uint8_t type, unsigned char *msg, uint32_t len) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_GetClusterNodeInfo)(RedisModuleCtx *ctx, const char *id, char *ip, char *master_id, int *port, int *flags) REDISMODULE_ATTR;
+REDISMODULE_API char ** (*RedisModule_GetClusterNodesList)(RedisModuleCtx *ctx, size_t *numnodes) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_FreeClusterNodesList)(char **ids) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleTimerID (*RedisModule_CreateTimer)(RedisModuleCtx *ctx, mstime_t period, RedisModuleTimerProc callback, void *data) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StopTimer)(RedisModuleCtx *ctx, RedisModuleTimerID id, void **data) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_GetTimerInfo)(RedisModuleCtx *ctx, RedisModuleTimerID id, uint64_t *remaining, void **data) REDISMODULE_ATTR;
+REDISMODULE_API const char * (*RedisModule_GetMyClusterID)(void) REDISMODULE_ATTR;
+REDISMODULE_API size_t (*RedisModule_GetClusterSize)(void) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_GetRandomBytes)(unsigned char *dst, size_t len) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_GetRandomHexChars)(char *dst, size_t len) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_SetDisconnectCallback)(RedisModuleBlockedClient *bc, RedisModuleDisconnectFunc callback) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_SetClusterFlags)(RedisModuleCtx *ctx, uint64_t flags) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ExportSharedAPI)(RedisModuleCtx *ctx, const char *apiname, void *func) REDISMODULE_ATTR;
+REDISMODULE_API void * (*RedisModule_GetSharedAPI)(RedisModuleCtx *ctx, const char *apiname) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleCommandFilter * (*RedisModule_RegisterCommandFilter)(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc cb, int flags) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_UnregisterCommandFilter)(RedisModuleCtx *ctx, RedisModuleCommandFilter *filter) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_CommandFilterArgsCount)(RedisModuleCommandFilterCtx *fctx) REDISMODULE_ATTR;
+REDISMODULE_API const RedisModuleString * (*RedisModule_CommandFilterArgGet)(RedisModuleCommandFilterCtx *fctx, int pos) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_CommandFilterArgInsert)(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_CommandFilterArgReplace)(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_CommandFilterArgDelete)(RedisModuleCommandFilterCtx *fctx, int pos) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_Fork)(RedisModuleForkDoneHandler cb, void *user_data) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ExitFromChild)(int retcode) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_KillForkChild)(int child_pid) REDISMODULE_ATTR;
+REDISMODULE_API float (*RedisModule_GetUsedMemoryRatio)() REDISMODULE_ATTR;
+REDISMODULE_API size_t (*RedisModule_MallocSize)(void* ptr) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleUser * (*RedisModule_CreateModuleUser)(const char *name) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_FreeModuleUser)(RedisModuleUser *user) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_SetModuleUserACL)(RedisModuleUser *user, const char* acl) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_AuthenticateClientWithACLUser)(RedisModuleCtx *ctx, const char *name, size_t len, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_AuthenticateClientWithUser)(RedisModuleCtx *ctx, RedisModuleUser *user, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id) REDISMODULE_ATTR;
 #endif
 
 #define RedisModule_IsAOFClient(id) ((id) == CLIENT_ID_AOF)
 
 /* This is included inline inside each Redis module. */
-static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int apiver) __attribute__((unused));
+static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int apiver) REDISMODULE_ATTR_UNUSED;
 static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int apiver) {
     void *getapifuncptr = ((void**)ctx)[0];
     RedisModule_GetApi = (int (*)(const char *, void *)) (unsigned long)getapifuncptr;

From bffbbeaa9a1a6b8e81384297272cb0631502e8fd Mon Sep 17 00:00:00 2001
From: michael-grunder <michael.grunder@gmail.com>
Date: Sat, 15 Aug 2020 12:24:31 -0700
Subject: [PATCH 0582/1098] Squashed 'deps/hiredis/' changes from
 d5b4c69b7..00272d669

00272d669 Rename sds calls so they don't conflict in Redis.

git-subtree-dir: deps/hiredis
git-subtree-split: 00272d669b11e96b8311d9bfe167c117f8887dd6
---
 async.c     |  26 +-
 hiredis.c   |  92 +++---
 hiredis.h   |   6 +-
 net.c       |   2 +-
 read.c      |  22 +-
 sds.c       | 800 ++++++++++++++++++++++++++--------------------------
 sds.h       | 262 ++++++++---------
 sdsalloc.h  |   6 +-
 sdscompat.h |  94 ++++++
 ssl.c       |   2 +-
 test.c      |  10 +-
 11 files changed, 708 insertions(+), 614 deletions(-)
 create mode 100644 sdscompat.h

diff --git a/async.c b/async.c
index 020cb09b9..64ab601c9 100644
--- a/async.c
+++ b/async.c
@@ -54,7 +54,7 @@ void __redisSetError(redisContext *c, int type, const char *str);
 /* Functions managing dictionary of callbacks for pub/sub. */
 static unsigned int callbackHash(const void *key) {
     return dictGenHashFunction((const unsigned char *)key,
-                               sdslen((const sds)key));
+                               hi_sdslen((const hisds)key));
 }
 
 static void *callbackValDup(void *privdata, const void *src) {
@@ -73,15 +73,15 @@ static int callbackKeyCompare(void *privdata, const void *key1, const void *key2
     int l1, l2;
     ((void) privdata);
 
-    l1 = sdslen((const sds)key1);
-    l2 = sdslen((const sds)key2);
+    l1 = hi_sdslen((const hisds)key1);
+    l2 = hi_sdslen((const hisds)key2);
     if (l1 != l2) return 0;
     return memcmp(key1,key2,l1) == 0;
 }
 
 static void callbackKeyDestructor(void *privdata, void *key) {
     ((void) privdata);
-    sdsfree((sds)key);
+    hi_sdsfree((hisds)key);
 }
 
 static void callbackValDestructor(void *privdata, void *val) {
@@ -418,7 +418,7 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply,
     dictEntry *de;
     int pvariant;
     char *stype;
-    sds sname;
+    hisds sname;
 
     /* Custom reply functions are not supported for pub/sub. This will fail
      * very hard when they are used... */
@@ -435,7 +435,7 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply,
 
         /* Locate the right callback */
         assert(reply->element[1]->type == REDIS_REPLY_STRING);
-        sname = sdsnewlen(reply->element[1]->str,reply->element[1]->len);
+        sname = hi_sdsnewlen(reply->element[1]->str,reply->element[1]->len);
         if (sname == NULL)
             goto oom;
 
@@ -466,7 +466,7 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply,
                     c->flags &= ~REDIS_SUBSCRIBED;
             }
         }
-        sdsfree(sname);
+        hi_sdsfree(sname);
     } else {
         /* Shift callback for invalid commands. */
         __redisShiftCallback(&ac->sub.invalid,dstcb);
@@ -511,7 +511,7 @@ void redisProcessCallbacks(redisAsyncContext *ac) {
         if (reply == NULL) {
             /* When the connection is being disconnected and there are
              * no more replies, this is the cue to really disconnect. */
-            if (c->flags & REDIS_DISCONNECTING && sdslen(c->obuf) == 0
+            if (c->flags & REDIS_DISCONNECTING && hi_sdslen(c->obuf) == 0
                 && ac->replies.head == NULL) {
                 __redisAsyncDisconnect(ac);
                 return;
@@ -744,7 +744,7 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void
     const char *cstr, *astr;
     size_t clen, alen;
     const char *p;
-    sds sname;
+    hisds sname;
     int ret;
 
     /* Don't accept new commands when the connection is about to be closed. */
@@ -768,7 +768,7 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void
 
         /* Add every channel/pattern to the list of subscription callbacks. */
         while ((p = nextArgument(p,&astr,&alen)) != NULL) {
-            sname = sdsnewlen(astr,alen);
+            sname = hi_sdsnewlen(astr,alen);
             if (sname == NULL)
                 goto oom;
 
@@ -786,7 +786,7 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void
 
             ret = dictReplace(cbdict,sname,&cb);
 
-            if (ret == 0) sdsfree(sname);
+            if (ret == 0) hi_sdsfree(sname);
         }
     } else if (strncasecmp(cstr,"unsubscribe\r\n",13) == 0) {
         /* It is only useful to call (P)UNSUBSCRIBE when the context is
@@ -845,14 +845,14 @@ int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata
 }
 
 int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen) {
-    sds cmd;
+    hisds cmd;
     int len;
     int status;
     len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen);
     if (len < 0)
         return REDIS_ERR;
     status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
-    sdsfree(cmd);
+    hi_sdsfree(cmd);
     return status;
 }
 
diff --git a/hiredis.c b/hiredis.c
index a7fbf486f..38e7e950a 100644
--- a/hiredis.c
+++ b/hiredis.c
@@ -305,7 +305,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) {
     const char *c = format;
     char *cmd = NULL; /* final command */
     int pos; /* position in final command */
-    sds curarg, newarg; /* current argument */
+    hisds curarg, newarg; /* current argument */
     int touched = 0; /* was the current argument touched? */
     char **curargv = NULL, **newargv = NULL;
     int argc = 0;
@@ -318,7 +318,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) {
         return -1;
 
     /* Build the command string accordingly to protocol */
-    curarg = sdsempty();
+    curarg = hi_sdsempty();
     if (curarg == NULL)
         return -1;
 
@@ -330,15 +330,15 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) {
                     if (newargv == NULL) goto memory_err;
                     curargv = newargv;
                     curargv[argc++] = curarg;
-                    totlen += bulklen(sdslen(curarg));
+                    totlen += bulklen(hi_sdslen(curarg));
 
                     /* curarg is put in argv so it can be overwritten. */
-                    curarg = sdsempty();
+                    curarg = hi_sdsempty();
                     if (curarg == NULL) goto memory_err;
                     touched = 0;
                 }
             } else {
-                newarg = sdscatlen(curarg,c,1);
+                newarg = hi_sdscatlen(curarg,c,1);
                 if (newarg == NULL) goto memory_err;
                 curarg = newarg;
                 touched = 1;
@@ -355,16 +355,16 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) {
                 arg = va_arg(ap,char*);
                 size = strlen(arg);
                 if (size > 0)
-                    newarg = sdscatlen(curarg,arg,size);
+                    newarg = hi_sdscatlen(curarg,arg,size);
                 break;
             case 'b':
                 arg = va_arg(ap,char*);
                 size = va_arg(ap,size_t);
                 if (size > 0)
-                    newarg = sdscatlen(curarg,arg,size);
+                    newarg = hi_sdscatlen(curarg,arg,size);
                 break;
             case '%':
-                newarg = sdscat(curarg,"%");
+                newarg = hi_sdscat(curarg,"%");
                 break;
             default:
                 /* Try to detect printf format */
@@ -452,7 +452,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) {
                     if (_l < sizeof(_format)-2) {
                         memcpy(_format,c,_l);
                         _format[_l] = '\0';
-                        newarg = sdscatvprintf(curarg,_format,_cpy);
+                        newarg = hi_sdscatvprintf(curarg,_format,_cpy);
 
                         /* Update current position (note: outer blocks
                          * increment c twice so compensate here) */
@@ -479,9 +479,9 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) {
         if (newargv == NULL) goto memory_err;
         curargv = newargv;
         curargv[argc++] = curarg;
-        totlen += bulklen(sdslen(curarg));
+        totlen += bulklen(hi_sdslen(curarg));
     } else {
-        sdsfree(curarg);
+        hi_sdsfree(curarg);
     }
 
     /* Clear curarg because it was put in curargv or was free'd. */
@@ -496,10 +496,10 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) {
 
     pos = sprintf(cmd,"*%d\r\n",argc);
     for (j = 0; j < argc; j++) {
-        pos += sprintf(cmd+pos,"$%zu\r\n",sdslen(curargv[j]));
-        memcpy(cmd+pos,curargv[j],sdslen(curargv[j]));
-        pos += sdslen(curargv[j]);
-        sdsfree(curargv[j]);
+        pos += sprintf(cmd+pos,"$%zu\r\n",hi_sdslen(curargv[j]));
+        memcpy(cmd+pos,curargv[j],hi_sdslen(curargv[j]));
+        pos += hi_sdslen(curargv[j]);
+        hi_sdsfree(curargv[j]);
         cmd[pos++] = '\r';
         cmd[pos++] = '\n';
     }
@@ -521,11 +521,11 @@ memory_err:
 cleanup:
     if (curargv) {
         while(argc--)
-            sdsfree(curargv[argc]);
+            hi_sdsfree(curargv[argc]);
         hi_free(curargv);
     }
 
-    sdsfree(curarg);
+    hi_sdsfree(curarg);
     hi_free(cmd);
 
     return error_type;
@@ -558,16 +558,16 @@ int redisFormatCommand(char **target, const char *format, ...) {
     return len;
 }
 
-/* Format a command according to the Redis protocol using an sds string and
- * sdscatfmt for the processing of arguments. This function takes the
+/* Format a command according to the Redis protocol using an hisds string and
+ * hi_sdscatfmt for the processing of arguments. This function takes the
  * number of arguments, an array with arguments and an array with their
  * lengths. If the latter is set to NULL, strlen will be used to compute the
  * argument lengths.
  */
-int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv,
+int redisFormatSdsCommandArgv(hisds *target, int argc, const char **argv,
                               const size_t *argvlen)
 {
-    sds cmd, aux;
+    hisds cmd, aux;
     unsigned long long totlen;
     int j;
     size_t len;
@@ -584,36 +584,36 @@ int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv,
     }
 
     /* Use an SDS string for command construction */
-    cmd = sdsempty();
+    cmd = hi_sdsempty();
     if (cmd == NULL)
         return -1;
 
     /* We already know how much storage we need */
-    aux = sdsMakeRoomFor(cmd, totlen);
+    aux = hi_sdsMakeRoomFor(cmd, totlen);
     if (aux == NULL) {
-        sdsfree(cmd);
+        hi_sdsfree(cmd);
         return -1;
     }
 
     cmd = aux;
 
     /* Construct command */
-    cmd = sdscatfmt(cmd, "*%i\r\n", argc);
+    cmd = hi_sdscatfmt(cmd, "*%i\r\n", argc);
     for (j=0; j < argc; j++) {
         len = argvlen ? argvlen[j] : strlen(argv[j]);
-        cmd = sdscatfmt(cmd, "$%u\r\n", len);
-        cmd = sdscatlen(cmd, argv[j], len);
-        cmd = sdscatlen(cmd, "\r\n", sizeof("\r\n")-1);
+        cmd = hi_sdscatfmt(cmd, "$%u\r\n", len);
+        cmd = hi_sdscatlen(cmd, argv[j], len);
+        cmd = hi_sdscatlen(cmd, "\r\n", sizeof("\r\n")-1);
     }
 
-    assert(sdslen(cmd)==totlen);
+    assert(hi_sdslen(cmd)==totlen);
 
     *target = cmd;
     return totlen;
 }
 
-void redisFreeSdsCommand(sds cmd) {
-    sdsfree(cmd);
+void redisFreeSdsCommand(hisds cmd) {
+    hi_sdsfree(cmd);
 }
 
 /* Format a command according to the Redis protocol. This function takes the
@@ -697,7 +697,7 @@ static redisContext *redisContextInit(void) {
 
     c->funcs = &redisContextDefaultFuncs;
 
-    c->obuf = sdsempty();
+    c->obuf = hi_sdsempty();
     c->reader = redisReaderCreate();
     c->fd = REDIS_INVALID_FD;
 
@@ -714,7 +714,7 @@ void redisFree(redisContext *c) {
         return;
     redisNetClose(c);
 
-    sdsfree(c->obuf);
+    hi_sdsfree(c->obuf);
     redisReaderFree(c->reader);
     hi_free(c->tcp.host);
     hi_free(c->tcp.source_addr);
@@ -751,10 +751,10 @@ int redisReconnect(redisContext *c) {
 
     redisNetClose(c);
 
-    sdsfree(c->obuf);
+    hi_sdsfree(c->obuf);
     redisReaderFree(c->reader);
 
-    c->obuf = sdsempty();
+    c->obuf = hi_sdsempty();
     c->reader = redisReaderCreate();
 
     if (c->obuf == NULL || c->reader == NULL) {
@@ -965,22 +965,22 @@ int redisBufferWrite(redisContext *c, int *done) {
     if (c->err)
         return REDIS_ERR;
 
-    if (sdslen(c->obuf) > 0) {
+    if (hi_sdslen(c->obuf) > 0) {
         ssize_t nwritten = c->funcs->write(c);
         if (nwritten < 0) {
             return REDIS_ERR;
         } else if (nwritten > 0) {
-            if (nwritten == (ssize_t)sdslen(c->obuf)) {
-                sdsfree(c->obuf);
-                c->obuf = sdsempty();
+            if (nwritten == (ssize_t)hi_sdslen(c->obuf)) {
+                hi_sdsfree(c->obuf);
+                c->obuf = hi_sdsempty();
                 if (c->obuf == NULL)
                     goto oom;
             } else {
-                if (sdsrange(c->obuf,nwritten,-1) < 0) goto oom;
+                if (hi_sdsrange(c->obuf,nwritten,-1) < 0) goto oom;
             }
         }
     }
-    if (done != NULL) *done = (sdslen(c->obuf) == 0);
+    if (done != NULL) *done = (hi_sdslen(c->obuf) == 0);
     return REDIS_OK;
 
 oom:
@@ -1058,9 +1058,9 @@ int redisGetReply(redisContext *c, void **reply) {
  * the reply (or replies in pub/sub).
  */
 int __redisAppendCommand(redisContext *c, const char *cmd, size_t len) {
-    sds newbuf;
+    hisds newbuf;
 
-    newbuf = sdscatlen(c->obuf,cmd,len);
+    newbuf = hi_sdscatlen(c->obuf,cmd,len);
     if (newbuf == NULL) {
         __redisSetError(c,REDIS_ERR_OOM,"Out of memory");
         return REDIS_ERR;
@@ -1112,7 +1112,7 @@ int redisAppendCommand(redisContext *c, const char *format, ...) {
 }
 
 int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) {
-    sds cmd;
+    hisds cmd;
     int len;
 
     len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen);
@@ -1122,11 +1122,11 @@ int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const s
     }
 
     if (__redisAppendCommand(c,cmd,len) != REDIS_OK) {
-        sdsfree(cmd);
+        hi_sdsfree(cmd);
         return REDIS_ERR;
     }
 
-    sdsfree(cmd);
+    hi_sdsfree(cmd);
     return REDIS_OK;
 }
 
diff --git a/hiredis.h b/hiredis.h
index 0622aaab8..b597394d4 100644
--- a/hiredis.h
+++ b/hiredis.h
@@ -42,7 +42,7 @@ struct timeval; /* forward declaration */
 typedef long long ssize_t;
 #endif
 #include <stdint.h> /* uintXX_t, etc */
-#include "sds.h" /* for sds */
+#include "sds.h" /* for hisds */
 #include "alloc.h" /* for allocation wrappers */
 
 #define HIREDIS_MAJOR 1
@@ -128,9 +128,9 @@ void freeReplyObject(void *reply);
 int redisvFormatCommand(char **target, const char *format, va_list ap);
 int redisFormatCommand(char **target, const char *format, ...);
 int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen);
-int redisFormatSdsCommandArgv(sds *target, int argc, const char ** argv, const size_t *argvlen);
+int redisFormatSdsCommandArgv(hisds *target, int argc, const char ** argv, const size_t *argvlen);
 void redisFreeCommand(char *cmd);
-void redisFreeSdsCommand(sds cmd);
+void redisFreeSdsCommand(hisds cmd);
 
 enum redisConnectionType {
     REDIS_CONN_TCP,
diff --git a/net.c b/net.c
index c6b0e5d8e..88f9aff25 100644
--- a/net.c
+++ b/net.c
@@ -80,7 +80,7 @@ ssize_t redisNetRead(redisContext *c, char *buf, size_t bufcap) {
 }
 
 ssize_t redisNetWrite(redisContext *c) {
-    ssize_t nwritten = send(c->fd, c->obuf, sdslen(c->obuf), 0);
+    ssize_t nwritten = send(c->fd, c->obuf, hi_sdslen(c->obuf), 0);
     if (nwritten < 0) {
         if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) {
             /* Try again later */
diff --git a/read.c b/read.c
index 09524692b..682b9a6b9 100644
--- a/read.c
+++ b/read.c
@@ -59,7 +59,7 @@ static void __redisReaderSetError(redisReader *r, int type, const char *str) {
     }
 
     /* Clear input buffer on errors. */
-    sdsfree(r->buf);
+    hi_sdsfree(r->buf);
     r->buf = NULL;
     r->pos = r->len = 0;
 
@@ -609,7 +609,7 @@ redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) {
     if (r == NULL)
         return NULL;
 
-    r->buf = sdsempty();
+    r->buf = hi_sdsempty();
     if (r->buf == NULL)
         goto oom;
 
@@ -650,12 +650,12 @@ void redisReaderFree(redisReader *r) {
         hi_free(r->task);
     }
 
-    sdsfree(r->buf);
+    hi_sdsfree(r->buf);
     hi_free(r);
 }
 
 int redisReaderFeed(redisReader *r, const char *buf, size_t len) {
-    sds newbuf;
+    hisds newbuf;
 
     /* Return early when this reader is in an erroneous state. */
     if (r->err)
@@ -664,19 +664,19 @@ int redisReaderFeed(redisReader *r, const char *buf, size_t len) {
     /* Copy the provided buffer. */
     if (buf != NULL && len >= 1) {
         /* Destroy internal buffer when it is empty and is quite large. */
-        if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) {
-            sdsfree(r->buf);
-            r->buf = sdsempty();
+        if (r->len == 0 && r->maxbuf != 0 && hi_sdsavail(r->buf) > r->maxbuf) {
+            hi_sdsfree(r->buf);
+            r->buf = hi_sdsempty();
             if (r->buf == 0) goto oom;
 
             r->pos = 0;
         }
 
-        newbuf = sdscatlen(r->buf,buf,len);
+        newbuf = hi_sdscatlen(r->buf,buf,len);
         if (newbuf == NULL) goto oom;
 
         r->buf = newbuf;
-        r->len = sdslen(r->buf);
+        r->len = hi_sdslen(r->buf);
     }
 
     return REDIS_OK;
@@ -721,9 +721,9 @@ int redisReaderGetReply(redisReader *r, void **reply) {
     /* Discard part of the buffer when we've consumed at least 1k, to avoid
      * doing unnecessary calls to memmove() in sds.c. */
     if (r->pos >= 1024) {
-        if (sdsrange(r->buf,r->pos,-1) < 0) return REDIS_ERR;
+        if (hi_sdsrange(r->buf,r->pos,-1) < 0) return REDIS_ERR;
         r->pos = 0;
-        r->len = sdslen(r->buf);
+        r->len = hi_sdslen(r->buf);
     }
 
     /* Emit a reply when there is one. */
diff --git a/sds.c b/sds.c
index 49d2096b7..675e7649f 100644
--- a/sds.c
+++ b/sds.c
@@ -40,90 +40,90 @@
 #include "sds.h"
 #include "sdsalloc.h"
 
-static inline int sdsHdrSize(char type) {
-    switch(type&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
-            return sizeof(struct sdshdr5);
-        case SDS_TYPE_8:
-            return sizeof(struct sdshdr8);
-        case SDS_TYPE_16:
-            return sizeof(struct sdshdr16);
-        case SDS_TYPE_32:
-            return sizeof(struct sdshdr32);
-        case SDS_TYPE_64:
-            return sizeof(struct sdshdr64);
+static inline int hi_sdsHdrSize(char type) {
+    switch(type&HI_SDS_TYPE_MASK) {
+        case HI_SDS_TYPE_5:
+            return sizeof(struct hisdshdr5);
+        case HI_SDS_TYPE_8:
+            return sizeof(struct hisdshdr8);
+        case HI_SDS_TYPE_16:
+            return sizeof(struct hisdshdr16);
+        case HI_SDS_TYPE_32:
+            return sizeof(struct hisdshdr32);
+        case HI_SDS_TYPE_64:
+            return sizeof(struct hisdshdr64);
     }
     return 0;
 }
 
-static inline char sdsReqType(size_t string_size) {
+static inline char hi_sdsReqType(size_t string_size) {
     if (string_size < 32)
-        return SDS_TYPE_5;
+        return HI_SDS_TYPE_5;
     if (string_size < 0xff)
-        return SDS_TYPE_8;
+        return HI_SDS_TYPE_8;
     if (string_size < 0xffff)
-        return SDS_TYPE_16;
+        return HI_SDS_TYPE_16;
     if (string_size < 0xffffffff)
-        return SDS_TYPE_32;
-    return SDS_TYPE_64;
+        return HI_SDS_TYPE_32;
+    return HI_SDS_TYPE_64;
 }
 
-/* Create a new sds string with the content specified by the 'init' pointer
+/* Create a new hisds string with the content specified by the 'init' pointer
  * and 'initlen'.
  * If NULL is used for 'init' the string is initialized with zero bytes.
  *
- * The string is always null-termined (all the sds strings are, always) so
- * even if you create an sds string with:
+ * The string is always null-termined (all the hisds strings are, always) so
+ * even if you create an hisds string with:
  *
- * mystring = sdsnewlen("abc",3);
+ * mystring = hi_sdsnewlen("abc",3);
  *
  * You can print the string with printf() as there is an implicit \0 at the
  * end of the string. However the string is binary safe and can contain
- * \0 characters in the middle, as the length is stored in the sds header. */
-sds sdsnewlen(const void *init, size_t initlen) {
+ * \0 characters in the middle, as the length is stored in the hisds header. */
+hisds hi_sdsnewlen(const void *init, size_t initlen) {
     void *sh;
-    sds s;
-    char type = sdsReqType(initlen);
+    hisds s;
+    char type = hi_sdsReqType(initlen);
     /* Empty strings are usually created in order to append. Use type 8
      * since type 5 is not good at this. */
-    if (type == SDS_TYPE_5 && initlen == 0) type = SDS_TYPE_8;
-    int hdrlen = sdsHdrSize(type);
+    if (type == HI_SDS_TYPE_5 && initlen == 0) type = HI_SDS_TYPE_8;
+    int hdrlen = hi_sdsHdrSize(type);
     unsigned char *fp; /* flags pointer. */
 
-    sh = s_malloc(hdrlen+initlen+1);
+    sh = hi_s_malloc(hdrlen+initlen+1);
     if (sh == NULL) return NULL;
     if (!init)
         memset(sh, 0, hdrlen+initlen+1);
     s = (char*)sh+hdrlen;
     fp = ((unsigned char*)s)-1;
     switch(type) {
-        case SDS_TYPE_5: {
-            *fp = type | (initlen << SDS_TYPE_BITS);
+        case HI_SDS_TYPE_5: {
+            *fp = type | (initlen << HI_SDS_TYPE_BITS);
             break;
         }
-        case SDS_TYPE_8: {
-            SDS_HDR_VAR(8,s);
+        case HI_SDS_TYPE_8: {
+            HI_SDS_HDR_VAR(8,s);
             sh->len = initlen;
             sh->alloc = initlen;
             *fp = type;
             break;
         }
-        case SDS_TYPE_16: {
-            SDS_HDR_VAR(16,s);
+        case HI_SDS_TYPE_16: {
+            HI_SDS_HDR_VAR(16,s);
             sh->len = initlen;
             sh->alloc = initlen;
             *fp = type;
             break;
         }
-        case SDS_TYPE_32: {
-            SDS_HDR_VAR(32,s);
+        case HI_SDS_TYPE_32: {
+            HI_SDS_HDR_VAR(32,s);
             sh->len = initlen;
             sh->alloc = initlen;
             *fp = type;
             break;
         }
-        case SDS_TYPE_64: {
-            SDS_HDR_VAR(64,s);
+        case HI_SDS_TYPE_64: {
+            HI_SDS_HDR_VAR(64,s);
             sh->len = initlen;
             sh->alloc = initlen;
             *fp = type;
@@ -136,164 +136,164 @@ sds sdsnewlen(const void *init, size_t initlen) {
     return s;
 }
 
-/* Create an empty (zero length) sds string. Even in this case the string
+/* Create an empty (zero length) hisds string. Even in this case the string
  * always has an implicit null term. */
-sds sdsempty(void) {
-    return sdsnewlen("",0);
+hisds hi_sdsempty(void) {
+    return hi_sdsnewlen("",0);
 }
 
-/* Create a new sds string starting from a null terminated C string. */
-sds sdsnew(const char *init) {
+/* Create a new hisds string starting from a null terminated C string. */
+hisds hi_sdsnew(const char *init) {
     size_t initlen = (init == NULL) ? 0 : strlen(init);
-    return sdsnewlen(init, initlen);
+    return hi_sdsnewlen(init, initlen);
 }
 
-/* Duplicate an sds string. */
-sds sdsdup(const sds s) {
-    return sdsnewlen(s, sdslen(s));
+/* Duplicate an hisds string. */
+hisds hi_sdsdup(const hisds s) {
+    return hi_sdsnewlen(s, hi_sdslen(s));
 }
 
-/* Free an sds string. No operation is performed if 's' is NULL. */
-void sdsfree(sds s) {
+/* Free an hisds string. No operation is performed if 's' is NULL. */
+void hi_sdsfree(hisds s) {
     if (s == NULL) return;
-    s_free((char*)s-sdsHdrSize(s[-1]));
+    hi_s_free((char*)s-hi_sdsHdrSize(s[-1]));
 }
 
-/* Set the sds string length to the length as obtained with strlen(), so
+/* Set the hisds string length to the length as obtained with strlen(), so
  * considering as content only up to the first null term character.
  *
- * This function is useful when the sds string is hacked manually in some
+ * This function is useful when the hisds string is hacked manually in some
  * way, like in the following example:
  *
- * s = sdsnew("foobar");
+ * s = hi_sdsnew("foobar");
  * s[2] = '\0';
- * sdsupdatelen(s);
- * printf("%d\n", sdslen(s));
+ * hi_sdsupdatelen(s);
+ * printf("%d\n", hi_sdslen(s));
  *
- * The output will be "2", but if we comment out the call to sdsupdatelen()
+ * The output will be "2", but if we comment out the call to hi_sdsupdatelen()
  * the output will be "6" as the string was modified but the logical length
  * remains 6 bytes. */
-void sdsupdatelen(sds s) {
+void hi_sdsupdatelen(hisds s) {
     int reallen = strlen(s);
-    sdssetlen(s, reallen);
+    hi_sdssetlen(s, reallen);
 }
 
-/* Modify an sds string in-place to make it empty (zero length).
+/* Modify an hisds string in-place to make it empty (zero length).
  * However all the existing buffer is not discarded but set as free space
  * so that next append operations will not require allocations up to the
  * number of bytes previously available. */
-void sdsclear(sds s) {
-    sdssetlen(s, 0);
+void hi_sdsclear(hisds s) {
+    hi_sdssetlen(s, 0);
     s[0] = '\0';
 }
 
-/* Enlarge the free space at the end of the sds string so that the caller
+/* Enlarge the free space at the end of the hisds string so that the caller
  * is sure that after calling this function can overwrite up to addlen
  * bytes after the end of the string, plus one more byte for nul term.
  *
- * Note: this does not change the *length* of the sds string as returned
- * by sdslen(), but only the free buffer space we have. */
-sds sdsMakeRoomFor(sds s, size_t addlen) {
+ * Note: this does not change the *length* of the hisds string as returned
+ * by hi_sdslen(), but only the free buffer space we have. */
+hisds hi_sdsMakeRoomFor(hisds s, size_t addlen) {
     void *sh, *newsh;
-    size_t avail = sdsavail(s);
+    size_t avail = hi_sdsavail(s);
     size_t len, newlen;
-    char type, oldtype = s[-1] & SDS_TYPE_MASK;
+    char type, oldtype = s[-1] & HI_SDS_TYPE_MASK;
     int hdrlen;
 
     /* Return ASAP if there is enough space left. */
     if (avail >= addlen) return s;
 
-    len = sdslen(s);
-    sh = (char*)s-sdsHdrSize(oldtype);
+    len = hi_sdslen(s);
+    sh = (char*)s-hi_sdsHdrSize(oldtype);
     newlen = (len+addlen);
-    if (newlen < SDS_MAX_PREALLOC)
+    if (newlen < HI_SDS_MAX_PREALLOC)
         newlen *= 2;
     else
-        newlen += SDS_MAX_PREALLOC;
+        newlen += HI_SDS_MAX_PREALLOC;
 
-    type = sdsReqType(newlen);
+    type = hi_sdsReqType(newlen);
 
     /* Don't use type 5: the user is appending to the string and type 5 is
-     * not able to remember empty space, so sdsMakeRoomFor() must be called
+     * not able to remember empty space, so hi_sdsMakeRoomFor() must be called
      * at every appending operation. */
-    if (type == SDS_TYPE_5) type = SDS_TYPE_8;
+    if (type == HI_SDS_TYPE_5) type = HI_SDS_TYPE_8;
 
-    hdrlen = sdsHdrSize(type);
+    hdrlen = hi_sdsHdrSize(type);
     if (oldtype==type) {
-        newsh = s_realloc(sh, hdrlen+newlen+1);
+        newsh = hi_s_realloc(sh, hdrlen+newlen+1);
         if (newsh == NULL) return NULL;
         s = (char*)newsh+hdrlen;
     } else {
         /* Since the header size changes, need to move the string forward,
          * and can't use realloc */
-        newsh = s_malloc(hdrlen+newlen+1);
+        newsh = hi_s_malloc(hdrlen+newlen+1);
         if (newsh == NULL) return NULL;
         memcpy((char*)newsh+hdrlen, s, len+1);
-        s_free(sh);
+        hi_s_free(sh);
         s = (char*)newsh+hdrlen;
         s[-1] = type;
-        sdssetlen(s, len);
+        hi_sdssetlen(s, len);
     }
-    sdssetalloc(s, newlen);
+    hi_sdssetalloc(s, newlen);
     return s;
 }
 
-/* Reallocate the sds string so that it has no free space at the end. The
+/* Reallocate the hisds string so that it has no free space at the end. The
  * contained string remains not altered, but next concatenation operations
  * will require a reallocation.
  *
- * After the call, the passed sds string is no longer valid and all the
+ * After the call, the passed hisds string is no longer valid and all the
  * references must be substituted with the new pointer returned by the call. */
-sds sdsRemoveFreeSpace(sds s) {
+hisds hi_sdsRemoveFreeSpace(hisds s) {
     void *sh, *newsh;
-    char type, oldtype = s[-1] & SDS_TYPE_MASK;
+    char type, oldtype = s[-1] & HI_SDS_TYPE_MASK;
     int hdrlen;
-    size_t len = sdslen(s);
-    sh = (char*)s-sdsHdrSize(oldtype);
+    size_t len = hi_sdslen(s);
+    sh = (char*)s-hi_sdsHdrSize(oldtype);
 
-    type = sdsReqType(len);
-    hdrlen = sdsHdrSize(type);
+    type = hi_sdsReqType(len);
+    hdrlen = hi_sdsHdrSize(type);
     if (oldtype==type) {
-        newsh = s_realloc(sh, hdrlen+len+1);
+        newsh = hi_s_realloc(sh, hdrlen+len+1);
         if (newsh == NULL) return NULL;
         s = (char*)newsh+hdrlen;
     } else {
-        newsh = s_malloc(hdrlen+len+1);
+        newsh = hi_s_malloc(hdrlen+len+1);
         if (newsh == NULL) return NULL;
         memcpy((char*)newsh+hdrlen, s, len+1);
-        s_free(sh);
+        hi_s_free(sh);
         s = (char*)newsh+hdrlen;
         s[-1] = type;
-        sdssetlen(s, len);
+        hi_sdssetlen(s, len);
     }
-    sdssetalloc(s, len);
+    hi_sdssetalloc(s, len);
     return s;
 }
 
-/* Return the total size of the allocation of the specifed sds string,
+/* Return the total size of the allocation of the specifed hisds string,
  * including:
- * 1) The sds header before the pointer.
+ * 1) The hisds header before the pointer.
  * 2) The string.
  * 3) The free buffer at the end if any.
  * 4) The implicit null term.
  */
-size_t sdsAllocSize(sds s) {
-    size_t alloc = sdsalloc(s);
-    return sdsHdrSize(s[-1])+alloc+1;
+size_t hi_sdsAllocSize(hisds s) {
+    size_t alloc = hi_sdsalloc(s);
+    return hi_sdsHdrSize(s[-1])+alloc+1;
 }
 
 /* Return the pointer of the actual SDS allocation (normally SDS strings
  * are referenced by the start of the string buffer). */
-void *sdsAllocPtr(sds s) {
-    return (void*) (s-sdsHdrSize(s[-1]));
+void *hi_sdsAllocPtr(hisds s) {
+    return (void*) (s-hi_sdsHdrSize(s[-1]));
 }
 
-/* Increment the sds length and decrements the left free space at the
+/* Increment the hisds length and decrements the left free space at the
  * end of the string according to 'incr'. Also set the null term
  * in the new end of the string.
  *
  * This function is used in order to fix the string length after the
- * user calls sdsMakeRoomFor(), writes something after the end of
+ * user calls hi_sdsMakeRoomFor(), writes something after the end of
  * the current string, and finally needs to set the new length.
  *
  * Note: it is possible to use a negative increment in order to
@@ -301,48 +301,48 @@ void *sdsAllocPtr(sds s) {
  *
  * Usage example:
  *
- * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the
+ * Using hi_sdsIncrLen() and hi_sdsMakeRoomFor() it is possible to mount the
  * following schema, to cat bytes coming from the kernel to the end of an
- * sds string without copying into an intermediate buffer:
+ * hisds string without copying into an intermediate buffer:
  *
- * oldlen = sdslen(s);
- * s = sdsMakeRoomFor(s, BUFFER_SIZE);
+ * oldlen = hi_hi_sdslen(s);
+ * s = hi_sdsMakeRoomFor(s, BUFFER_SIZE);
  * nread = read(fd, s+oldlen, BUFFER_SIZE);
  * ... check for nread <= 0 and handle it ...
- * sdsIncrLen(s, nread);
+ * hi_sdsIncrLen(s, nread);
  */
-void sdsIncrLen(sds s, int incr) {
+void hi_sdsIncrLen(hisds s, int incr) {
     unsigned char flags = s[-1];
     size_t len;
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5: {
+    switch(flags&HI_SDS_TYPE_MASK) {
+        case HI_SDS_TYPE_5: {
             unsigned char *fp = ((unsigned char*)s)-1;
-            unsigned char oldlen = SDS_TYPE_5_LEN(flags);
+            unsigned char oldlen = HI_SDS_TYPE_5_LEN(flags);
             assert((incr > 0 && oldlen+incr < 32) || (incr < 0 && oldlen >= (unsigned int)(-incr)));
-            *fp = SDS_TYPE_5 | ((oldlen+incr) << SDS_TYPE_BITS);
+            *fp = HI_SDS_TYPE_5 | ((oldlen+incr) << HI_SDS_TYPE_BITS);
             len = oldlen+incr;
             break;
         }
-        case SDS_TYPE_8: {
-            SDS_HDR_VAR(8,s);
+        case HI_SDS_TYPE_8: {
+            HI_SDS_HDR_VAR(8,s);
             assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr)));
             len = (sh->len += incr);
             break;
         }
-        case SDS_TYPE_16: {
-            SDS_HDR_VAR(16,s);
+        case HI_SDS_TYPE_16: {
+            HI_SDS_HDR_VAR(16,s);
             assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr)));
             len = (sh->len += incr);
             break;
         }
-        case SDS_TYPE_32: {
-            SDS_HDR_VAR(32,s);
+        case HI_SDS_TYPE_32: {
+            HI_SDS_HDR_VAR(32,s);
             assert((incr >= 0 && sh->alloc-sh->len >= (unsigned int)incr) || (incr < 0 && sh->len >= (unsigned int)(-incr)));
             len = (sh->len += incr);
             break;
         }
-        case SDS_TYPE_64: {
-            SDS_HDR_VAR(64,s);
+        case HI_SDS_TYPE_64: {
+            HI_SDS_HDR_VAR(64,s);
             assert((incr >= 0 && sh->alloc-sh->len >= (uint64_t)incr) || (incr < 0 && sh->len >= (uint64_t)(-incr)));
             len = (sh->len += incr);
             break;
@@ -352,83 +352,83 @@ void sdsIncrLen(sds s, int incr) {
     s[len] = '\0';
 }
 
-/* Grow the sds to have the specified length. Bytes that were not part of
- * the original length of the sds will be set to zero.
+/* Grow the hisds to have the specified length. Bytes that were not part of
+ * the original length of the hisds will be set to zero.
  *
  * if the specified length is smaller than the current length, no operation
  * is performed. */
-sds sdsgrowzero(sds s, size_t len) {
-    size_t curlen = sdslen(s);
+hisds hi_sdsgrowzero(hisds s, size_t len) {
+    size_t curlen = hi_sdslen(s);
 
     if (len <= curlen) return s;
-    s = sdsMakeRoomFor(s,len-curlen);
+    s = hi_sdsMakeRoomFor(s,len-curlen);
     if (s == NULL) return NULL;
 
     /* Make sure added region doesn't contain garbage */
     memset(s+curlen,0,(len-curlen+1)); /* also set trailing \0 byte */
-    sdssetlen(s, len);
+    hi_sdssetlen(s, len);
     return s;
 }
 
 /* Append the specified binary-safe string pointed by 't' of 'len' bytes to the
- * end of the specified sds string 's'.
+ * end of the specified hisds string 's'.
  *
- * After the call, the passed sds string is no longer valid and all the
+ * After the call, the passed hisds string is no longer valid and all the
  * references must be substituted with the new pointer returned by the call. */
-sds sdscatlen(sds s, const void *t, size_t len) {
-    size_t curlen = sdslen(s);
+hisds hi_sdscatlen(hisds s, const void *t, size_t len) {
+    size_t curlen = hi_sdslen(s);
 
-    s = sdsMakeRoomFor(s,len);
+    s = hi_sdsMakeRoomFor(s,len);
     if (s == NULL) return NULL;
     memcpy(s+curlen, t, len);
-    sdssetlen(s, curlen+len);
+    hi_sdssetlen(s, curlen+len);
     s[curlen+len] = '\0';
     return s;
 }
 
-/* Append the specified null termianted C string to the sds string 's'.
+/* Append the specified null termianted C string to the hisds string 's'.
  *
- * After the call, the passed sds string is no longer valid and all the
+ * After the call, the passed hisds string is no longer valid and all the
  * references must be substituted with the new pointer returned by the call. */
-sds sdscat(sds s, const char *t) {
-    return sdscatlen(s, t, strlen(t));
+hisds hi_sdscat(hisds s, const char *t) {
+    return hi_sdscatlen(s, t, strlen(t));
 }
 
-/* Append the specified sds 't' to the existing sds 's'.
+/* Append the specified hisds 't' to the existing hisds 's'.
  *
- * After the call, the modified sds string is no longer valid and all the
+ * After the call, the modified hisds string is no longer valid and all the
  * references must be substituted with the new pointer returned by the call. */
-sds sdscatsds(sds s, const sds t) {
-    return sdscatlen(s, t, sdslen(t));
+hisds hi_sdscatsds(hisds s, const hisds t) {
+    return hi_sdscatlen(s, t, hi_sdslen(t));
 }
 
-/* Destructively modify the sds string 's' to hold the specified binary
+/* Destructively modify the hisds string 's' to hold the specified binary
  * safe string pointed by 't' of length 'len' bytes. */
-sds sdscpylen(sds s, const char *t, size_t len) {
-    if (sdsalloc(s) < len) {
-        s = sdsMakeRoomFor(s,len-sdslen(s));
+hisds hi_sdscpylen(hisds s, const char *t, size_t len) {
+    if (hi_sdsalloc(s) < len) {
+        s = hi_sdsMakeRoomFor(s,len-hi_sdslen(s));
         if (s == NULL) return NULL;
     }
     memcpy(s, t, len);
     s[len] = '\0';
-    sdssetlen(s, len);
+    hi_sdssetlen(s, len);
     return s;
 }
 
-/* Like sdscpylen() but 't' must be a null-termined string so that the length
+/* Like hi_sdscpylen() but 't' must be a null-termined string so that the length
  * of the string is obtained with strlen(). */
-sds sdscpy(sds s, const char *t) {
-    return sdscpylen(s, t, strlen(t));
+hisds hi_sdscpy(hisds s, const char *t) {
+    return hi_sdscpylen(s, t, strlen(t));
 }
 
-/* Helper for sdscatlonglong() doing the actual number -> string
+/* Helper for hi_sdscatlonglong() doing the actual number -> string
  * conversion. 's' must point to a string with room for at least
- * SDS_LLSTR_SIZE bytes.
+ * HI_SDS_LLSTR_SIZE bytes.
  *
  * The function returns the length of the null-terminated string
  * representation stored at 's'. */
-#define SDS_LLSTR_SIZE 21
-int sdsll2str(char *s, long long value) {
+#define HI_SDS_LLSTR_SIZE 21
+int hi_sdsll2str(char *s, long long value) {
     char *p, aux;
     unsigned long long v;
     size_t l;
@@ -459,8 +459,8 @@ int sdsll2str(char *s, long long value) {
     return l;
 }
 
-/* Identical sdsll2str(), but for unsigned long long type. */
-int sdsull2str(char *s, unsigned long long v) {
+/* Identical hi_sdsll2str(), but for unsigned long long type. */
+int hi_sdsull2str(char *s, unsigned long long v) {
     char *p, aux;
     size_t l;
 
@@ -488,19 +488,19 @@ int sdsull2str(char *s, unsigned long long v) {
     return l;
 }
 
-/* Create an sds string from a long long value. It is much faster than:
+/* Create an hisds string from a long long value. It is much faster than:
  *
- * sdscatprintf(sdsempty(),"%lld\n", value);
+ * hi_sdscatprintf(hi_sdsempty(),"%lld\n", value);
  */
-sds sdsfromlonglong(long long value) {
-    char buf[SDS_LLSTR_SIZE];
-    int len = sdsll2str(buf,value);
+hisds hi_sdsfromlonglong(long long value) {
+    char buf[HI_SDS_LLSTR_SIZE];
+    int len = hi_sdsll2str(buf,value);
 
-    return sdsnewlen(buf,len);
+    return hi_sdsnewlen(buf,len);
 }
 
-/* Like sdscatprintf() but gets va_list instead of being variadic. */
-sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
+/* Like hi_sdscatprintf() but gets va_list instead of being variadic. */
+hisds hi_sdscatvprintf(hisds s, const char *fmt, va_list ap) {
     va_list cpy;
     char staticbuf[1024], *buf = staticbuf, *t;
     size_t buflen = strlen(fmt)*2;
@@ -508,7 +508,7 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
     /* We try to start using a static buffer for speed.
      * If not possible we revert to heap allocation. */
     if (buflen > sizeof(staticbuf)) {
-        buf = s_malloc(buflen);
+        buf = hi_s_malloc(buflen);
         if (buf == NULL) return NULL;
     } else {
         buflen = sizeof(staticbuf);
@@ -522,9 +522,9 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
         vsnprintf(buf, buflen, fmt, cpy);
         va_end(cpy);
         if (buf[buflen-2] != '\0') {
-            if (buf != staticbuf) s_free(buf);
+            if (buf != staticbuf) hi_s_free(buf);
             buflen *= 2;
-            buf = s_malloc(buflen);
+            buf = hi_s_malloc(buflen);
             if (buf == NULL) return NULL;
             continue;
         }
@@ -532,39 +532,39 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
     }
 
     /* Finally concat the obtained string to the SDS string and return it. */
-    t = sdscat(s, buf);
-    if (buf != staticbuf) s_free(buf);
+    t = hi_sdscat(s, buf);
+    if (buf != staticbuf) hi_s_free(buf);
     return t;
 }
 
-/* Append to the sds string 's' a string obtained using printf-alike format
+/* Append to the hisds string 's' a string obtained using printf-alike format
  * specifier.
  *
- * After the call, the modified sds string is no longer valid and all the
+ * After the call, the modified hisds string is no longer valid and all the
  * references must be substituted with the new pointer returned by the call.
  *
  * Example:
  *
- * s = sdsnew("Sum is: ");
- * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b).
+ * s = hi_sdsnew("Sum is: ");
+ * s = hi_sdscatprintf(s,"%d+%d = %d",a,b,a+b).
  *
  * Often you need to create a string from scratch with the printf-alike
- * format. When this is the need, just use sdsempty() as the target string:
+ * format. When this is the need, just use hi_sdsempty() as the target string:
  *
- * s = sdscatprintf(sdsempty(), "... your format ...", args);
+ * s = hi_sdscatprintf(hi_sdsempty(), "... your format ...", args);
  */
-sds sdscatprintf(sds s, const char *fmt, ...) {
+hisds hi_sdscatprintf(hisds s, const char *fmt, ...) {
     va_list ap;
     char *t;
     va_start(ap, fmt);
-    t = sdscatvprintf(s,fmt,ap);
+    t = hi_sdscatvprintf(s,fmt,ap);
     va_end(ap);
     return t;
 }
 
-/* This function is similar to sdscatprintf, but much faster as it does
+/* This function is similar to hi_sdscatprintf, but much faster as it does
  * not rely on sprintf() family functions implemented by the libc that
- * are often very slow. Moreover directly handling the sds string as
+ * are often very slow. Moreover directly handling the hisds string as
  * new data is concatenated provides a performance improvement.
  *
  * However this function only handles an incompatible subset of printf-alike
@@ -578,13 +578,13 @@ sds sdscatprintf(sds s, const char *fmt, ...) {
  * %U - 64 bit unsigned integer (unsigned long long, uint64_t)
  * %% - Verbatim "%" character.
  */
-sds sdscatfmt(sds s, char const *fmt, ...) {
+hisds hi_sdscatfmt(hisds s, char const *fmt, ...) {
     const char *f = fmt;
     int i;
     va_list ap;
 
     va_start(ap,fmt);
-    i = sdslen(s); /* Position of the next byte to write to dest str. */
+    i = hi_sdslen(s); /* Position of the next byte to write to dest str. */
     while(*f) {
         char next, *str;
         size_t l;
@@ -592,8 +592,8 @@ sds sdscatfmt(sds s, char const *fmt, ...) {
         unsigned long long unum;
 
         /* Make sure there is always space for at least 1 char. */
-        if (sdsavail(s)==0) {
-            s = sdsMakeRoomFor(s,1);
+        if (hi_sdsavail(s)==0) {
+            s = hi_sdsMakeRoomFor(s,1);
             if (s == NULL) goto fmt_error;
         }
 
@@ -605,13 +605,13 @@ sds sdscatfmt(sds s, char const *fmt, ...) {
             case 's':
             case 'S':
                 str = va_arg(ap,char*);
-                l = (next == 's') ? strlen(str) : sdslen(str);
-                if (sdsavail(s) < l) {
-                    s = sdsMakeRoomFor(s,l);
+                l = (next == 's') ? strlen(str) : hi_sdslen(str);
+                if (hi_sdsavail(s) < l) {
+                    s = hi_sdsMakeRoomFor(s,l);
                     if (s == NULL) goto fmt_error;
                 }
                 memcpy(s+i,str,l);
-                sdsinclen(s,l);
+                hi_sdsinclen(s,l);
                 i += l;
                 break;
             case 'i':
@@ -621,14 +621,14 @@ sds sdscatfmt(sds s, char const *fmt, ...) {
                 else
                     num = va_arg(ap,long long);
                 {
-                    char buf[SDS_LLSTR_SIZE];
-                    l = sdsll2str(buf,num);
-                    if (sdsavail(s) < l) {
-                        s = sdsMakeRoomFor(s,l);
+                    char buf[HI_SDS_LLSTR_SIZE];
+                    l = hi_sdsll2str(buf,num);
+                    if (hi_sdsavail(s) < l) {
+                        s = hi_sdsMakeRoomFor(s,l);
                         if (s == NULL) goto fmt_error;
                     }
                     memcpy(s+i,buf,l);
-                    sdsinclen(s,l);
+                    hi_sdsinclen(s,l);
                     i += l;
                 }
                 break;
@@ -639,26 +639,26 @@ sds sdscatfmt(sds s, char const *fmt, ...) {
                 else
                     unum = va_arg(ap,unsigned long long);
                 {
-                    char buf[SDS_LLSTR_SIZE];
-                    l = sdsull2str(buf,unum);
-                    if (sdsavail(s) < l) {
-                        s = sdsMakeRoomFor(s,l);
+                    char buf[HI_SDS_LLSTR_SIZE];
+                    l = hi_sdsull2str(buf,unum);
+                    if (hi_sdsavail(s) < l) {
+                        s = hi_sdsMakeRoomFor(s,l);
                         if (s == NULL) goto fmt_error;
                     }
                     memcpy(s+i,buf,l);
-                    sdsinclen(s,l);
+                    hi_sdsinclen(s,l);
                     i += l;
                 }
                 break;
             default: /* Handle %% and generally %<unknown>. */
                 s[i++] = next;
-                sdsinclen(s,1);
+                hi_sdsinclen(s,1);
                 break;
             }
             break;
         default:
             s[i++] = *f;
-            sdsinclen(s,1);
+            hi_sdsinclen(s,1);
             break;
         }
         f++;
@@ -677,29 +677,29 @@ fmt_error:
 /* Remove the part of the string from left and from right composed just of
  * contiguous characters found in 'cset', that is a null terminted C string.
  *
- * After the call, the modified sds string is no longer valid and all the
+ * After the call, the modified hisds string is no longer valid and all the
  * references must be substituted with the new pointer returned by the call.
  *
  * Example:
  *
- * s = sdsnew("AA...AA.a.aa.aHelloWorld     :::");
- * s = sdstrim(s,"Aa. :");
+ * s = hi_sdsnew("AA...AA.a.aa.aHelloWorld     :::");
+ * s = hi_sdstrim(s,"Aa. :");
  * printf("%s\n", s);
  *
  * Output will be just "Hello World".
  */
-sds sdstrim(sds s, const char *cset) {
+hisds hi_sdstrim(hisds s, const char *cset) {
     char *start, *end, *sp, *ep;
     size_t len;
 
     sp = start = s;
-    ep = end = s+sdslen(s)-1;
+    ep = end = s+hi_sdslen(s)-1;
     while(sp <= end && strchr(cset, *sp)) sp++;
     while(ep > sp && strchr(cset, *ep)) ep--;
     len = (sp > ep) ? 0 : ((ep-sp)+1);
     if (s != sp) memmove(s, sp, len);
     s[len] = '\0';
-    sdssetlen(s,len);
+    hi_sdssetlen(s,len);
     return s;
 }
 
@@ -715,16 +715,16 @@ sds sdstrim(sds s, const char *cset) {
  * The string is modified in-place.
  *
  * Return value:
- * -1 (error) if sdslen(s) is larger than maximum positive ssize_t value.
+ * -1 (error) if hi_sdslen(s) is larger than maximum positive ssize_t value.
  *  0 on success.
  *
  * Example:
  *
- * s = sdsnew("Hello World");
- * sdsrange(s,1,-1); => "ello World"
+ * s = hi_sdsnew("Hello World");
+ * hi_sdsrange(s,1,-1); => "ello World"
  */
-int sdsrange(sds s, ssize_t start, ssize_t end) {
-    size_t newlen, len = sdslen(s);
+int hi_sdsrange(hisds s, ssize_t start, ssize_t end) {
+    size_t newlen, len = hi_sdslen(s);
     if (len > SSIZE_MAX) return -1;
 
     if (len == 0) return 0;
@@ -749,25 +749,25 @@ int sdsrange(sds s, ssize_t start, ssize_t end) {
     }
     if (start && newlen) memmove(s, s+start, newlen);
     s[newlen] = 0;
-    sdssetlen(s,newlen);
+    hi_sdssetlen(s,newlen);
     return 0;
 }
 
-/* Apply tolower() to every character of the sds string 's'. */
-void sdstolower(sds s) {
-    int len = sdslen(s), j;
+/* Apply tolower() to every character of the hisds string 's'. */
+void hi_sdstolower(hisds s) {
+    int len = hi_sdslen(s), j;
 
     for (j = 0; j < len; j++) s[j] = tolower(s[j]);
 }
 
-/* Apply toupper() to every character of the sds string 's'. */
-void sdstoupper(sds s) {
-    int len = sdslen(s), j;
+/* Apply toupper() to every character of the hisds string 's'. */
+void hi_sdstoupper(hisds s) {
+    int len = hi_sdslen(s), j;
 
     for (j = 0; j < len; j++) s[j] = toupper(s[j]);
 }
 
-/* Compare two sds strings s1 and s2 with memcmp().
+/* Compare two hisds strings s1 and s2 with memcmp().
  *
  * Return value:
  *
@@ -778,12 +778,12 @@ void sdstoupper(sds s) {
  * If two strings share exactly the same prefix, but one of the two has
  * additional characters, the longer string is considered to be greater than
  * the smaller one. */
-int sdscmp(const sds s1, const sds s2) {
+int hi_sdscmp(const hisds s1, const hisds s2) {
     size_t l1, l2, minlen;
     int cmp;
 
-    l1 = sdslen(s1);
-    l2 = sdslen(s2);
+    l1 = hi_sdslen(s1);
+    l2 = hi_sdslen(s2);
     minlen = (l1 < l2) ? l1 : l2;
     cmp = memcmp(s1,s2,minlen);
     if (cmp == 0) return l1-l2;
@@ -791,7 +791,7 @@ int sdscmp(const sds s1, const sds s2) {
 }
 
 /* Split 's' with separator in 'sep'. An array
- * of sds strings is returned. *count will be set
+ * of hisds strings is returned. *count will be set
  * by reference to the number of tokens returned.
  *
  * On out of memory, zero length string, zero length
@@ -799,20 +799,20 @@ int sdscmp(const sds s1, const sds s2) {
  *
  * Note that 'sep' is able to split a string using
  * a multi-character separator. For example
- * sdssplit("foo_-_bar","_-_"); will return two
+ * hi_sdssplit("foo_-_bar","_-_"); will return two
  * elements "foo" and "bar".
  *
  * This version of the function is binary-safe but
- * requires length arguments. sdssplit() is just the
+ * requires length arguments. hi_sdssplit() is just the
  * same function but for zero-terminated strings.
  */
-sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) {
+hisds *hi_sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) {
     int elements = 0, slots = 5, start = 0, j;
-    sds *tokens;
+    hisds *tokens;
 
     if (seplen < 1 || len < 0) return NULL;
 
-    tokens = s_malloc(sizeof(sds)*slots);
+    tokens = hi_s_malloc(sizeof(hisds)*slots);
     if (tokens == NULL) return NULL;
 
     if (len == 0) {
@@ -822,16 +822,16 @@ sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count
     for (j = 0; j < (len-(seplen-1)); j++) {
         /* make sure there is room for the next element and the final one */
         if (slots < elements+2) {
-            sds *newtokens;
+            hisds *newtokens;
 
             slots *= 2;
-            newtokens = s_realloc(tokens,sizeof(sds)*slots);
+            newtokens = hi_s_realloc(tokens,sizeof(hisds)*slots);
             if (newtokens == NULL) goto cleanup;
             tokens = newtokens;
         }
         /* search the separator */
         if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) {
-            tokens[elements] = sdsnewlen(s+start,j-start);
+            tokens[elements] = hi_sdsnewlen(s+start,j-start);
             if (tokens[elements] == NULL) goto cleanup;
             elements++;
             start = j+seplen;
@@ -839,7 +839,7 @@ sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count
         }
     }
     /* Add the final element. We are sure there is room in the tokens array. */
-    tokens[elements] = sdsnewlen(s+start,len-start);
+    tokens[elements] = hi_sdsnewlen(s+start,len-start);
     if (tokens[elements] == NULL) goto cleanup;
     elements++;
     *count = elements;
@@ -848,55 +848,55 @@ sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count
 cleanup:
     {
         int i;
-        for (i = 0; i < elements; i++) sdsfree(tokens[i]);
-        s_free(tokens);
+        for (i = 0; i < elements; i++) hi_sdsfree(tokens[i]);
+        hi_s_free(tokens);
         *count = 0;
         return NULL;
     }
 }
 
-/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */
-void sdsfreesplitres(sds *tokens, int count) {
+/* Free the result returned by hi_sdssplitlen(), or do nothing if 'tokens' is NULL. */
+void hi_sdsfreesplitres(hisds *tokens, int count) {
     if (!tokens) return;
     while(count--)
-        sdsfree(tokens[count]);
-    s_free(tokens);
+        hi_sdsfree(tokens[count]);
+    hi_s_free(tokens);
 }
 
-/* Append to the sds string "s" an escaped string representation where
+/* Append to the hisds string "s" an escaped string representation where
  * all the non-printable characters (tested with isprint()) are turned into
  * escapes in the form "\n\r\a...." or "\x<hex-number>".
  *
- * After the call, the modified sds string is no longer valid and all the
+ * After the call, the modified hisds string is no longer valid and all the
  * references must be substituted with the new pointer returned by the call. */
-sds sdscatrepr(sds s, const char *p, size_t len) {
-    s = sdscatlen(s,"\"",1);
+hisds hi_sdscatrepr(hisds s, const char *p, size_t len) {
+    s = hi_sdscatlen(s,"\"",1);
     while(len--) {
         switch(*p) {
         case '\\':
         case '"':
-            s = sdscatprintf(s,"\\%c",*p);
+            s = hi_sdscatprintf(s,"\\%c",*p);
             break;
-        case '\n': s = sdscatlen(s,"\\n",2); break;
-        case '\r': s = sdscatlen(s,"\\r",2); break;
-        case '\t': s = sdscatlen(s,"\\t",2); break;
-        case '\a': s = sdscatlen(s,"\\a",2); break;
-        case '\b': s = sdscatlen(s,"\\b",2); break;
+        case '\n': s = hi_sdscatlen(s,"\\n",2); break;
+        case '\r': s = hi_sdscatlen(s,"\\r",2); break;
+        case '\t': s = hi_sdscatlen(s,"\\t",2); break;
+        case '\a': s = hi_sdscatlen(s,"\\a",2); break;
+        case '\b': s = hi_sdscatlen(s,"\\b",2); break;
         default:
             if (isprint(*p))
-                s = sdscatprintf(s,"%c",*p);
+                s = hi_sdscatprintf(s,"%c",*p);
             else
-                s = sdscatprintf(s,"\\x%02x",(unsigned char)*p);
+                s = hi_sdscatprintf(s,"\\x%02x",(unsigned char)*p);
             break;
         }
         p++;
     }
-    return sdscatlen(s,"\"",1);
+    return hi_sdscatlen(s,"\"",1);
 }
 
-/* Helper function for sdssplitargs() that converts a hex digit into an
+/* Helper function for hi_sdssplitargs() that converts a hex digit into an
  * integer from 0 to 15 */
-int hex_digit_to_int(char c) {
+static int hi_hex_digit_to_int(char c) {
     switch(c) {
     case '0': return 0;
     case '1': return 1;
@@ -924,20 +924,20 @@ int hex_digit_to_int(char c) {
  * foo bar "newline are supported\n" and "\xff\x00otherstuff"
  *
  * The number of arguments is stored into *argc, and an array
- * of sds is returned.
+ * of hisds is returned.
  *
- * The caller should free the resulting array of sds strings with
- * sdsfreesplitres().
+ * The caller should free the resulting array of hisds strings with
+ * hi_sdsfreesplitres().
  *
- * Note that sdscatrepr() is able to convert back a string into
- * a quoted string in the same format sdssplitargs() is able to parse.
+ * Note that hi_sdscatrepr() is able to convert back a string into
+ * a quoted string in the same format hi_sdssplitargs() is able to parse.
  *
  * The function returns the allocated tokens on success, even when the
  * input string is empty, or NULL if the input contains unbalanced
  * quotes or closed quotes followed by non space characters
  * as in: "foo"bar or "foo'
  */
-sds *sdssplitargs(const char *line, int *argc) {
+hisds *hi_sdssplitargs(const char *line, int *argc) {
     const char *p = line;
     char *current = NULL;
     char **vector = NULL;
@@ -952,7 +952,7 @@ sds *sdssplitargs(const char *line, int *argc) {
             int insq=0; /* set to 1 if we are in 'single quotes' */
             int done=0;
 
-            if (current == NULL) current = sdsempty();
+            if (current == NULL) current = hi_sdsempty();
             while(!done) {
                 if (inq) {
                     if (*p == '\\' && *(p+1) == 'x' &&
@@ -961,9 +961,9 @@ sds *sdssplitargs(const char *line, int *argc) {
                     {
                         unsigned char byte;
 
-                        byte = (hex_digit_to_int(*(p+2))*16)+
-                                hex_digit_to_int(*(p+3));
-                        current = sdscatlen(current,(char*)&byte,1);
+                        byte = (hi_hex_digit_to_int(*(p+2))*16)+
+                                hi_hex_digit_to_int(*(p+3));
+                        current = hi_sdscatlen(current,(char*)&byte,1);
                         p += 3;
                     } else if (*p == '\\' && *(p+1)) {
                         char c;
@@ -977,7 +977,7 @@ sds *sdssplitargs(const char *line, int *argc) {
                         case 'a': c = '\a'; break;
                         default: c = *p; break;
                         }
-                        current = sdscatlen(current,&c,1);
+                        current = hi_sdscatlen(current,&c,1);
                     } else if (*p == '"') {
                         /* closing quote must be followed by a space or
                          * nothing at all. */
@@ -987,12 +987,12 @@ sds *sdssplitargs(const char *line, int *argc) {
                         /* unterminated quotes */
                         goto err;
                     } else {
-                        current = sdscatlen(current,p,1);
+                        current = hi_sdscatlen(current,p,1);
                     }
                 } else if (insq) {
                     if (*p == '\\' && *(p+1) == '\'') {
                         p++;
-                        current = sdscatlen(current,"'",1);
+                        current = hi_sdscatlen(current,"'",1);
                     } else if (*p == '\'') {
                         /* closing quote must be followed by a space or
                          * nothing at all. */
@@ -1002,7 +1002,7 @@ sds *sdssplitargs(const char *line, int *argc) {
                         /* unterminated quotes */
                         goto err;
                     } else {
-                        current = sdscatlen(current,p,1);
+                        current = hi_sdscatlen(current,p,1);
                     }
                 } else {
                     switch(*p) {
@@ -1020,7 +1020,7 @@ sds *sdssplitargs(const char *line, int *argc) {
                         insq=1;
                         break;
                     default:
-                        current = sdscatlen(current,p,1);
+                        current = hi_sdscatlen(current,p,1);
                         break;
                     }
                 }
@@ -1028,9 +1028,9 @@ sds *sdssplitargs(const char *line, int *argc) {
             }
             /* add the token to the vector */
             {
-                char **new_vector = s_realloc(vector,((*argc)+1)*sizeof(char*));
+                char **new_vector = hi_s_realloc(vector,((*argc)+1)*sizeof(char*));
                 if (new_vector == NULL) {
-                    s_free(vector);
+                    hi_s_free(vector);
                     return NULL;
                 }
 
@@ -1041,16 +1041,16 @@ sds *sdssplitargs(const char *line, int *argc) {
             }
         } else {
             /* Even on empty input string return something not NULL. */
-            if (vector == NULL) vector = s_malloc(sizeof(void*));
+            if (vector == NULL) vector = hi_s_malloc(sizeof(void*));
             return vector;
         }
     }
 
 err:
     while((*argc)--)
-        sdsfree(vector[*argc]);
-    s_free(vector);
-    if (current) sdsfree(current);
+        hi_sdsfree(vector[*argc]);
+    hi_s_free(vector);
+    if (current) hi_sdsfree(current);
     *argc = 0;
     return NULL;
 }
@@ -1059,13 +1059,13 @@ err:
  * characters specified in the 'from' string to the corresponding character
  * in the 'to' array.
  *
- * For instance: sdsmapchars(mystring, "ho", "01", 2)
+ * For instance: hi_sdsmapchars(mystring, "ho", "01", 2)
  * will have the effect of turning the string "hello" into "0ell1".
  *
- * The function returns the sds string pointer, that is always the same
+ * The function returns the hisds string pointer, that is always the same
  * as the input pointer since no resize is needed. */
-sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) {
-    size_t j, i, l = sdslen(s);
+hisds hi_sdsmapchars(hisds s, const char *from, const char *to, size_t setlen) {
+    size_t j, i, l = hi_sdslen(s);
 
     for (j = 0; j < l; j++) {
         for (i = 0; i < setlen; i++) {
@@ -1079,26 +1079,26 @@ sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) {
 }
 
 /* Join an array of C strings using the specified separator (also a C string).
- * Returns the result as an sds string. */
-sds sdsjoin(char **argv, int argc, char *sep) {
-    sds join = sdsempty();
+ * Returns the result as an hisds string. */
+hisds hi_sdsjoin(char **argv, int argc, char *sep) {
+    hisds join = hi_sdsempty();
     int j;
 
     for (j = 0; j < argc; j++) {
-        join = sdscat(join, argv[j]);
-        if (j != argc-1) join = sdscat(join,sep);
+        join = hi_sdscat(join, argv[j]);
+        if (j != argc-1) join = hi_sdscat(join,sep);
     }
     return join;
 }
 
-/* Like sdsjoin, but joins an array of SDS strings. */
-sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) {
-    sds join = sdsempty();
+/* Like hi_sdsjoin, but joins an array of SDS strings. */
+hisds hi_sdsjoinsds(hisds *argv, int argc, const char *sep, size_t seplen) {
+    hisds join = hi_sdsempty();
     int j;
 
     for (j = 0; j < argc; j++) {
-        join = sdscatsds(join, argv[j]);
-        if (j != argc-1) join = sdscatlen(join,sep,seplen);
+        join = hi_sdscatsds(join, argv[j]);
+        if (j != argc-1) join = hi_sdscatlen(join,sep,seplen);
     }
     return join;
 }
@@ -1108,138 +1108,138 @@ sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) {
  * the overhead of function calls. Here we define these wrappers only for
  * the programs SDS is linked to, if they want to touch the SDS internals
  * even if they use a different allocator. */
-void *sds_malloc(size_t size) { return s_malloc(size); }
-void *sds_realloc(void *ptr, size_t size) { return s_realloc(ptr,size); }
-void sds_free(void *ptr) { s_free(ptr); }
+void *hi_sds_malloc(size_t size) { return hi_s_malloc(size); }
+void *hi_sds_realloc(void *ptr, size_t size) { return hi_s_realloc(ptr,size); }
+void hi_sds_free(void *ptr) { hi_s_free(ptr); }
 
-#if defined(SDS_TEST_MAIN)
+#if defined(HI_SDS_TEST_MAIN)
 #include <stdio.h>
 #include "testhelp.h"
 #include "limits.h"
 
 #define UNUSED(x) (void)(x)
-int sdsTest(void) {
+int hi_sdsTest(void) {
     {
-        sds x = sdsnew("foo"), y;
+        hisds x = hi_sdsnew("foo"), y;
 
         test_cond("Create a string and obtain the length",
-            sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0)
+            hi_sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0)
 
-        sdsfree(x);
-        x = sdsnewlen("foo",2);
+        hi_sdsfree(x);
+        x = hi_sdsnewlen("foo",2);
         test_cond("Create a string with specified length",
-            sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0)
+            hi_sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0)
 
-        x = sdscat(x,"bar");
+        x = hi_sdscat(x,"bar");
         test_cond("Strings concatenation",
-            sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0);
+            hi_sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0);
 
-        x = sdscpy(x,"a");
-        test_cond("sdscpy() against an originally longer string",
-            sdslen(x) == 1 && memcmp(x,"a\0",2) == 0)
+        x = hi_sdscpy(x,"a");
+        test_cond("hi_sdscpy() against an originally longer string",
+            hi_sdslen(x) == 1 && memcmp(x,"a\0",2) == 0)
 
-        x = sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk");
-        test_cond("sdscpy() against an originally shorter string",
-            sdslen(x) == 33 &&
+        x = hi_sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk");
+        test_cond("hi_sdscpy() against an originally shorter string",
+            hi_sdslen(x) == 33 &&
             memcmp(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk\0",33) == 0)
 
-        sdsfree(x);
-        x = sdscatprintf(sdsempty(),"%d",123);
-        test_cond("sdscatprintf() seems working in the base case",
-            sdslen(x) == 3 && memcmp(x,"123\0",4) == 0)
+        hi_sdsfree(x);
+        x = hi_sdscatprintf(hi_sdsempty(),"%d",123);
+        test_cond("hi_sdscatprintf() seems working in the base case",
+            hi_sdslen(x) == 3 && memcmp(x,"123\0",4) == 0)
 
-        sdsfree(x);
-        x = sdsnew("--");
-        x = sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX);
-        test_cond("sdscatfmt() seems working in the base case",
-            sdslen(x) == 60 &&
+        hi_sdsfree(x);
+        x = hi_sdsnew("--");
+        x = hi_sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX);
+        test_cond("hi_sdscatfmt() seems working in the base case",
+            hi_sdslen(x) == 60 &&
             memcmp(x,"--Hello Hi! World -9223372036854775808,"
                      "9223372036854775807--",60) == 0)
         printf("[%s]\n",x);
 
-        sdsfree(x);
-        x = sdsnew("--");
-        x = sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX);
-        test_cond("sdscatfmt() seems working with unsigned numbers",
-            sdslen(x) == 35 &&
+        hi_sdsfree(x);
+        x = hi_sdsnew("--");
+        x = hi_sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX);
+        test_cond("hi_sdscatfmt() seems working with unsigned numbers",
+            hi_sdslen(x) == 35 &&
             memcmp(x,"--4294967295,18446744073709551615--",35) == 0)
 
-        sdsfree(x);
-        x = sdsnew(" x ");
-        sdstrim(x," x");
-        test_cond("sdstrim() works when all chars match",
-            sdslen(x) == 0)
+        hi_sdsfree(x);
+        x = hi_sdsnew(" x ");
+        hi_sdstrim(x," x");
+        test_cond("hi_sdstrim() works when all chars match",
+            hi_sdslen(x) == 0)
 
-        sdsfree(x);
-        x = sdsnew(" x ");
-        sdstrim(x," ");
-        test_cond("sdstrim() works when a single char remains",
-            sdslen(x) == 1 && x[0] == 'x')
+        hi_sdsfree(x);
+        x = hi_sdsnew(" x ");
+        hi_sdstrim(x," ");
+        test_cond("hi_sdstrim() works when a single char remains",
+            hi_sdslen(x) == 1 && x[0] == 'x')
 
-        sdsfree(x);
-        x = sdsnew("xxciaoyyy");
-        sdstrim(x,"xy");
-        test_cond("sdstrim() correctly trims characters",
-            sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0)
+        hi_sdsfree(x);
+        x = hi_sdsnew("xxciaoyyy");
+        hi_sdstrim(x,"xy");
+        test_cond("hi_sdstrim() correctly trims characters",
+            hi_sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0)
 
-        y = sdsdup(x);
-        sdsrange(y,1,1);
-        test_cond("sdsrange(...,1,1)",
-            sdslen(y) == 1 && memcmp(y,"i\0",2) == 0)
+        y = hi_sdsdup(x);
+        hi_sdsrange(y,1,1);
+        test_cond("hi_sdsrange(...,1,1)",
+            hi_sdslen(y) == 1 && memcmp(y,"i\0",2) == 0)
 
-        sdsfree(y);
-        y = sdsdup(x);
-        sdsrange(y,1,-1);
-        test_cond("sdsrange(...,1,-1)",
-            sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
+        hi_sdsfree(y);
+        y = hi_sdsdup(x);
+        hi_sdsrange(y,1,-1);
+        test_cond("hi_sdsrange(...,1,-1)",
+            hi_sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
 
-        sdsfree(y);
-        y = sdsdup(x);
-        sdsrange(y,-2,-1);
-        test_cond("sdsrange(...,-2,-1)",
-            sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0)
+        hi_sdsfree(y);
+        y = hi_sdsdup(x);
+        hi_sdsrange(y,-2,-1);
+        test_cond("hi_sdsrange(...,-2,-1)",
+            hi_sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0)
 
-        sdsfree(y);
-        y = sdsdup(x);
-        sdsrange(y,2,1);
-        test_cond("sdsrange(...,2,1)",
-            sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
+        hi_sdsfree(y);
+        y = hi_sdsdup(x);
+        hi_sdsrange(y,2,1);
+        test_cond("hi_sdsrange(...,2,1)",
+            hi_sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
 
-        sdsfree(y);
-        y = sdsdup(x);
-        sdsrange(y,1,100);
-        test_cond("sdsrange(...,1,100)",
-            sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
+        hi_sdsfree(y);
+        y = hi_sdsdup(x);
+        hi_sdsrange(y,1,100);
+        test_cond("hi_sdsrange(...,1,100)",
+            hi_sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
 
-        sdsfree(y);
-        y = sdsdup(x);
-        sdsrange(y,100,100);
-        test_cond("sdsrange(...,100,100)",
-            sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
+        hi_sdsfree(y);
+        y = hi_sdsdup(x);
+        hi_sdsrange(y,100,100);
+        test_cond("hi_sdsrange(...,100,100)",
+            hi_sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
 
-        sdsfree(y);
-        sdsfree(x);
-        x = sdsnew("foo");
-        y = sdsnew("foa");
-        test_cond("sdscmp(foo,foa)", sdscmp(x,y) > 0)
+        hi_sdsfree(y);
+        hi_sdsfree(x);
+        x = hi_sdsnew("foo");
+        y = hi_sdsnew("foa");
+        test_cond("hi_sdscmp(foo,foa)", hi_sdscmp(x,y) > 0)
 
-        sdsfree(y);
-        sdsfree(x);
-        x = sdsnew("bar");
-        y = sdsnew("bar");
-        test_cond("sdscmp(bar,bar)", sdscmp(x,y) == 0)
+        hi_sdsfree(y);
+        hi_sdsfree(x);
+        x = hi_sdsnew("bar");
+        y = hi_sdsnew("bar");
+        test_cond("hi_sdscmp(bar,bar)", hi_sdscmp(x,y) == 0)
 
-        sdsfree(y);
-        sdsfree(x);
-        x = sdsnew("aar");
-        y = sdsnew("bar");
-        test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0)
+        hi_sdsfree(y);
+        hi_sdsfree(x);
+        x = hi_sdsnew("aar");
+        y = hi_sdsnew("bar");
+        test_cond("hi_sdscmp(bar,bar)", hi_sdscmp(x,y) < 0)
 
-        sdsfree(y);
-        sdsfree(x);
-        x = sdsnewlen("\a\n\0foo\r",7);
-        y = sdscatrepr(sdsempty(),x,sdslen(x));
-        test_cond("sdscatrepr(...data...)",
+        hi_sdsfree(y);
+        hi_sdsfree(x);
+        x = hi_sdsnewlen("\a\n\0foo\r",7);
+        y = hi_sdscatrepr(hi_sdsempty(),x,hi_sdslen(x));
+        test_cond("hi_sdscatrepr(...data...)",
             memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0)
 
         {
@@ -1247,43 +1247,43 @@ int sdsTest(void) {
             char *p;
             int step = 10, j, i;
 
-            sdsfree(x);
-            sdsfree(y);
-            x = sdsnew("0");
-            test_cond("sdsnew() free/len buffers", sdslen(x) == 1 && sdsavail(x) == 0);
+            hi_sdsfree(x);
+            hi_sdsfree(y);
+            x = hi_sdsnew("0");
+            test_cond("hi_sdsnew() free/len buffers", hi_sdslen(x) == 1 && hi_sdsavail(x) == 0);
 
             /* Run the test a few times in order to hit the first two
              * SDS header types. */
             for (i = 0; i < 10; i++) {
-                int oldlen = sdslen(x);
-                x = sdsMakeRoomFor(x,step);
-                int type = x[-1]&SDS_TYPE_MASK;
+                int oldlen = hi_sdslen(x);
+                x = hi_sdsMakeRoomFor(x,step);
+                int type = x[-1]&HI_SDS_TYPE_MASK;
 
-                test_cond("sdsMakeRoomFor() len", sdslen(x) == oldlen);
-                if (type != SDS_TYPE_5) {
-                    test_cond("sdsMakeRoomFor() free", sdsavail(x) >= step);
-                    oldfree = sdsavail(x);
+                test_cond("sdsMakeRoomFor() len", hi_sdslen(x) == oldlen);
+                if (type != HI_SDS_TYPE_5) {
+                    test_cond("hi_sdsMakeRoomFor() free", hi_sdsavail(x) >= step);
+                    oldfree = hi_sdsavail(x);
                 }
                 p = x+oldlen;
                 for (j = 0; j < step; j++) {
                     p[j] = 'A'+j;
                 }
-                sdsIncrLen(x,step);
+                hi_sdsIncrLen(x,step);
             }
-            test_cond("sdsMakeRoomFor() content",
+            test_cond("hi_sdsMakeRoomFor() content",
                 memcmp("0ABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJ",x,101) == 0);
-            test_cond("sdsMakeRoomFor() final length",sdslen(x)==101);
+            test_cond("sdsMakeRoomFor() final length",hi_sdslen(x)==101);
 
-            sdsfree(x);
+            hi_sdsfree(x);
         }
     }
-    test_report()
+    test_report();
     return 0;
 }
 #endif
 
-#ifdef SDS_TEST_MAIN
+#ifdef HI_SDS_TEST_MAIN
 int main(void) {
-    return sdsTest();
+    return hi_sdsTest();
 }
 #endif
diff --git a/sds.h b/sds.h
index eda8833b5..573d6dd19 100644
--- a/sds.h
+++ b/sds.h
@@ -30,10 +30,10 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-#ifndef __SDS_H
-#define __SDS_H
+#ifndef HIREDIS_SDS_H
+#define HIREDIS_SDS_H
 
-#define SDS_MAX_PREALLOC (1024*1024)
+#define HI_SDS_MAX_PREALLOC (1024*1024)
 #ifdef _MSC_VER
 #define __attribute__(x)
 typedef long long ssize_t;
@@ -44,235 +44,235 @@ typedef long long ssize_t;
 #include <stdarg.h>
 #include <stdint.h>
 
-typedef char *sds;
+typedef char *hisds;
 
 /* Note: sdshdr5 is never used, we just access the flags byte directly.
  * However is here to document the layout of type 5 SDS strings. */
-struct __attribute__ ((__packed__)) sdshdr5 {
+struct __attribute__ ((__packed__)) hisdshdr5 {
     unsigned char flags; /* 3 lsb of type, and 5 msb of string length */
     char buf[];
 };
-struct __attribute__ ((__packed__)) sdshdr8 {
+struct __attribute__ ((__packed__)) hisdshdr8 {
     uint8_t len; /* used */
     uint8_t alloc; /* excluding the header and null terminator */
     unsigned char flags; /* 3 lsb of type, 5 unused bits */
     char buf[];
 };
-struct __attribute__ ((__packed__)) sdshdr16 {
+struct __attribute__ ((__packed__)) hisdshdr16 {
     uint16_t len; /* used */
     uint16_t alloc; /* excluding the header and null terminator */
     unsigned char flags; /* 3 lsb of type, 5 unused bits */
     char buf[];
 };
-struct __attribute__ ((__packed__)) sdshdr32 {
+struct __attribute__ ((__packed__)) hisdshdr32 {
     uint32_t len; /* used */
     uint32_t alloc; /* excluding the header and null terminator */
     unsigned char flags; /* 3 lsb of type, 5 unused bits */
     char buf[];
 };
-struct __attribute__ ((__packed__)) sdshdr64 {
+struct __attribute__ ((__packed__)) hisdshdr64 {
     uint64_t len; /* used */
     uint64_t alloc; /* excluding the header and null terminator */
     unsigned char flags; /* 3 lsb of type, 5 unused bits */
     char buf[];
 };
 
-#define SDS_TYPE_5  0
-#define SDS_TYPE_8  1
-#define SDS_TYPE_16 2
-#define SDS_TYPE_32 3
-#define SDS_TYPE_64 4
-#define SDS_TYPE_MASK 7
-#define SDS_TYPE_BITS 3
-#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T)));
-#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))))
-#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS)
+#define HI_SDS_TYPE_5  0
+#define HI_SDS_TYPE_8  1
+#define HI_SDS_TYPE_16 2
+#define HI_SDS_TYPE_32 3
+#define HI_SDS_TYPE_64 4
+#define HI_SDS_TYPE_MASK 7
+#define HI_SDS_TYPE_BITS 3
+#define HI_SDS_HDR_VAR(T,s) struct hisdshdr##T *sh = (struct hisdshdr##T *)((s)-(sizeof(struct hisdshdr##T)));
+#define HI_SDS_HDR(T,s) ((struct hisdshdr##T *)((s)-(sizeof(struct hisdshdr##T))))
+#define HI_SDS_TYPE_5_LEN(f) ((f)>>HI_SDS_TYPE_BITS)
 
-static inline size_t sdslen(const sds s) {
+static inline size_t hi_sdslen(const hisds s) {
     unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
-            return SDS_TYPE_5_LEN(flags);
-        case SDS_TYPE_8:
-            return SDS_HDR(8,s)->len;
-        case SDS_TYPE_16:
-            return SDS_HDR(16,s)->len;
-        case SDS_TYPE_32:
-            return SDS_HDR(32,s)->len;
-        case SDS_TYPE_64:
-            return SDS_HDR(64,s)->len;
+    switch(flags & HI_SDS_TYPE_MASK) {
+        case HI_SDS_TYPE_5:
+            return HI_SDS_TYPE_5_LEN(flags);
+        case HI_SDS_TYPE_8:
+            return HI_SDS_HDR(8,s)->len;
+        case HI_SDS_TYPE_16:
+            return HI_SDS_HDR(16,s)->len;
+        case HI_SDS_TYPE_32:
+            return HI_SDS_HDR(32,s)->len;
+        case HI_SDS_TYPE_64:
+            return HI_SDS_HDR(64,s)->len;
     }
     return 0;
 }
 
-static inline size_t sdsavail(const sds s) {
+static inline size_t hi_sdsavail(const hisds s) {
     unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5: {
+    switch(flags&HI_SDS_TYPE_MASK) {
+        case HI_SDS_TYPE_5: {
             return 0;
         }
-        case SDS_TYPE_8: {
-            SDS_HDR_VAR(8,s);
+        case HI_SDS_TYPE_8: {
+            HI_SDS_HDR_VAR(8,s);
             return sh->alloc - sh->len;
         }
-        case SDS_TYPE_16: {
-            SDS_HDR_VAR(16,s);
+        case HI_SDS_TYPE_16: {
+            HI_SDS_HDR_VAR(16,s);
             return sh->alloc - sh->len;
         }
-        case SDS_TYPE_32: {
-            SDS_HDR_VAR(32,s);
+        case HI_SDS_TYPE_32: {
+            HI_SDS_HDR_VAR(32,s);
             return sh->alloc - sh->len;
         }
-        case SDS_TYPE_64: {
-            SDS_HDR_VAR(64,s);
+        case HI_SDS_TYPE_64: {
+            HI_SDS_HDR_VAR(64,s);
             return sh->alloc - sh->len;
         }
     }
     return 0;
 }
 
-static inline void sdssetlen(sds s, size_t newlen) {
+static inline void hi_sdssetlen(hisds s, size_t newlen) {
     unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
+    switch(flags&HI_SDS_TYPE_MASK) {
+        case HI_SDS_TYPE_5:
             {
                 unsigned char *fp = ((unsigned char*)s)-1;
-                *fp = (unsigned char)(SDS_TYPE_5 | (newlen << SDS_TYPE_BITS));
+                *fp = (unsigned char)(HI_SDS_TYPE_5 | (newlen << HI_SDS_TYPE_BITS));
             }
             break;
-        case SDS_TYPE_8:
-            SDS_HDR(8,s)->len = (uint8_t)newlen;
+        case HI_SDS_TYPE_8:
+            HI_SDS_HDR(8,s)->len = (uint8_t)newlen;
             break;
-        case SDS_TYPE_16:
-            SDS_HDR(16,s)->len = (uint16_t)newlen;
+        case HI_SDS_TYPE_16:
+            HI_SDS_HDR(16,s)->len = (uint16_t)newlen;
             break;
-        case SDS_TYPE_32:
-            SDS_HDR(32,s)->len = (uint32_t)newlen;
+        case HI_SDS_TYPE_32:
+            HI_SDS_HDR(32,s)->len = (uint32_t)newlen;
             break;
-        case SDS_TYPE_64:
-            SDS_HDR(64,s)->len = (uint64_t)newlen;
+        case HI_SDS_TYPE_64:
+            HI_SDS_HDR(64,s)->len = (uint64_t)newlen;
             break;
     }
 }
 
-static inline void sdsinclen(sds s, size_t inc) {
+static inline void hi_sdsinclen(hisds s, size_t inc) {
     unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
+    switch(flags&HI_SDS_TYPE_MASK) {
+        case HI_SDS_TYPE_5:
             {
                 unsigned char *fp = ((unsigned char*)s)-1;
-                unsigned char newlen = SDS_TYPE_5_LEN(flags)+(unsigned char)inc;
-                *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
+                unsigned char newlen = HI_SDS_TYPE_5_LEN(flags)+(unsigned char)inc;
+                *fp = HI_SDS_TYPE_5 | (newlen << HI_SDS_TYPE_BITS);
             }
             break;
-        case SDS_TYPE_8:
-            SDS_HDR(8,s)->len += (uint8_t)inc;
+        case HI_SDS_TYPE_8:
+            HI_SDS_HDR(8,s)->len += (uint8_t)inc;
             break;
-        case SDS_TYPE_16:
-            SDS_HDR(16,s)->len += (uint16_t)inc;
+        case HI_SDS_TYPE_16:
+            HI_SDS_HDR(16,s)->len += (uint16_t)inc;
             break;
-        case SDS_TYPE_32:
-            SDS_HDR(32,s)->len += (uint32_t)inc;
+        case HI_SDS_TYPE_32:
+            HI_SDS_HDR(32,s)->len += (uint32_t)inc;
             break;
-        case SDS_TYPE_64:
-            SDS_HDR(64,s)->len += (uint64_t)inc;
+        case HI_SDS_TYPE_64:
+            HI_SDS_HDR(64,s)->len += (uint64_t)inc;
             break;
     }
 }
 
-/* sdsalloc() = sdsavail() + sdslen() */
-static inline size_t sdsalloc(const sds s) {
+/* hi_sdsalloc() = hi_sdsavail() + hi_sdslen() */
+static inline size_t hi_sdsalloc(const hisds s) {
     unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
-            return SDS_TYPE_5_LEN(flags);
-        case SDS_TYPE_8:
-            return SDS_HDR(8,s)->alloc;
-        case SDS_TYPE_16:
-            return SDS_HDR(16,s)->alloc;
-        case SDS_TYPE_32:
-            return SDS_HDR(32,s)->alloc;
-        case SDS_TYPE_64:
-            return SDS_HDR(64,s)->alloc;
+    switch(flags & HI_SDS_TYPE_MASK) {
+        case HI_SDS_TYPE_5:
+            return HI_SDS_TYPE_5_LEN(flags);
+        case HI_SDS_TYPE_8:
+            return HI_SDS_HDR(8,s)->alloc;
+        case HI_SDS_TYPE_16:
+            return HI_SDS_HDR(16,s)->alloc;
+        case HI_SDS_TYPE_32:
+            return HI_SDS_HDR(32,s)->alloc;
+        case HI_SDS_TYPE_64:
+            return HI_SDS_HDR(64,s)->alloc;
     }
     return 0;
 }
 
-static inline void sdssetalloc(sds s, size_t newlen) {
+static inline void hi_sdssetalloc(hisds s, size_t newlen) {
     unsigned char flags = s[-1];
-    switch(flags&SDS_TYPE_MASK) {
-        case SDS_TYPE_5:
+    switch(flags&HI_SDS_TYPE_MASK) {
+        case HI_SDS_TYPE_5:
             /* Nothing to do, this type has no total allocation info. */
             break;
-        case SDS_TYPE_8:
-            SDS_HDR(8,s)->alloc = (uint8_t)newlen;
+        case HI_SDS_TYPE_8:
+            HI_SDS_HDR(8,s)->alloc = (uint8_t)newlen;
             break;
-        case SDS_TYPE_16:
-            SDS_HDR(16,s)->alloc = (uint16_t)newlen;
+        case HI_SDS_TYPE_16:
+            HI_SDS_HDR(16,s)->alloc = (uint16_t)newlen;
             break;
-        case SDS_TYPE_32:
-            SDS_HDR(32,s)->alloc = (uint32_t)newlen;
+        case HI_SDS_TYPE_32:
+            HI_SDS_HDR(32,s)->alloc = (uint32_t)newlen;
             break;
-        case SDS_TYPE_64:
-            SDS_HDR(64,s)->alloc = (uint64_t)newlen;
+        case HI_SDS_TYPE_64:
+            HI_SDS_HDR(64,s)->alloc = (uint64_t)newlen;
             break;
     }
 }
 
-sds sdsnewlen(const void *init, size_t initlen);
-sds sdsnew(const char *init);
-sds sdsempty(void);
-sds sdsdup(const sds s);
-void sdsfree(sds s);
-sds sdsgrowzero(sds s, size_t len);
-sds sdscatlen(sds s, const void *t, size_t len);
-sds sdscat(sds s, const char *t);
-sds sdscatsds(sds s, const sds t);
-sds sdscpylen(sds s, const char *t, size_t len);
-sds sdscpy(sds s, const char *t);
+hisds hi_sdsnewlen(const void *init, size_t initlen);
+hisds hi_sdsnew(const char *init);
+hisds hi_sdsempty(void);
+hisds hi_sdsdup(const hisds s);
+void  hi_sdsfree(hisds s);
+hisds hi_sdsgrowzero(hisds s, size_t len);
+hisds hi_sdscatlen(hisds s, const void *t, size_t len);
+hisds hi_sdscat(hisds s, const char *t);
+hisds hi_sdscatsds(hisds s, const hisds t);
+hisds hi_sdscpylen(hisds s, const char *t, size_t len);
+hisds hi_sdscpy(hisds s, const char *t);
 
-sds sdscatvprintf(sds s, const char *fmt, va_list ap);
+hisds hi_sdscatvprintf(hisds s, const char *fmt, va_list ap);
 #ifdef __GNUC__
-sds sdscatprintf(sds s, const char *fmt, ...)
+hisds hi_sdscatprintf(hisds s, const char *fmt, ...)
     __attribute__((format(printf, 2, 3)));
 #else
-sds sdscatprintf(sds s, const char *fmt, ...);
+hisds hi_sdscatprintf(hisds s, const char *fmt, ...);
 #endif
 
-sds sdscatfmt(sds s, char const *fmt, ...);
-sds sdstrim(sds s, const char *cset);
-int sdsrange(sds s, ssize_t start, ssize_t end);
-void sdsupdatelen(sds s);
-void sdsclear(sds s);
-int sdscmp(const sds s1, const sds s2);
-sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count);
-void sdsfreesplitres(sds *tokens, int count);
-void sdstolower(sds s);
-void sdstoupper(sds s);
-sds sdsfromlonglong(long long value);
-sds sdscatrepr(sds s, const char *p, size_t len);
-sds *sdssplitargs(const char *line, int *argc);
-sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen);
-sds sdsjoin(char **argv, int argc, char *sep);
-sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen);
+hisds hi_sdscatfmt(hisds s, char const *fmt, ...);
+hisds hi_sdstrim(hisds s, const char *cset);
+int hi_sdsrange(hisds s, ssize_t start, ssize_t end);
+void hi_sdsupdatelen(hisds s);
+void hi_sdsclear(hisds s);
+int hi_sdscmp(const hisds s1, const hisds s2);
+hisds *hi_sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count);
+void hi_sdsfreesplitres(hisds *tokens, int count);
+void hi_sdstolower(hisds s);
+void hi_sdstoupper(hisds s);
+hisds hi_sdsfromlonglong(long long value);
+hisds hi_sdscatrepr(hisds s, const char *p, size_t len);
+hisds *hi_sdssplitargs(const char *line, int *argc);
+hisds hi_sdsmapchars(hisds s, const char *from, const char *to, size_t setlen);
+hisds hi_sdsjoin(char **argv, int argc, char *sep);
+hisds hi_sdsjoinsds(hisds *argv, int argc, const char *sep, size_t seplen);
 
 /* Low level functions exposed to the user API */
-sds sdsMakeRoomFor(sds s, size_t addlen);
-void sdsIncrLen(sds s, int incr);
-sds sdsRemoveFreeSpace(sds s);
-size_t sdsAllocSize(sds s);
-void *sdsAllocPtr(sds s);
+hisds hi_sdsMakeRoomFor(hisds s, size_t addlen);
+void hi_sdsIncrLen(hisds s, int incr);
+hisds hi_sdsRemoveFreeSpace(hisds s);
+size_t hi_sdsAllocSize(hisds s);
+void *hi_sdsAllocPtr(hisds s);
 
 /* Export the allocator used by SDS to the program using SDS.
  * Sometimes the program SDS is linked to, may use a different set of
  * allocators, but may want to allocate or free things that SDS will
  * respectively free or allocate. */
-void *sds_malloc(size_t size);
-void *sds_realloc(void *ptr, size_t size);
-void sds_free(void *ptr);
+void *hi_sds_malloc(size_t size);
+void *hi_sds_realloc(void *ptr, size_t size);
+void hi_sds_free(void *ptr);
 
 #ifdef REDIS_TEST
-int sdsTest(int argc, char *argv[]);
+int hi_sdsTest(int argc, char *argv[]);
 #endif
 
-#endif
+#endif /* HIREDIS_SDS_H */
diff --git a/sdsalloc.h b/sdsalloc.h
index 5538dd94c..c9dcc3df8 100644
--- a/sdsalloc.h
+++ b/sdsalloc.h
@@ -39,6 +39,6 @@
 
 #include "alloc.h"
 
-#define s_malloc hi_malloc
-#define s_realloc hi_realloc
-#define s_free hi_free
+#define hi_s_malloc hi_malloc
+#define hi_s_realloc hi_realloc
+#define hi_s_free hi_free
diff --git a/sdscompat.h b/sdscompat.h
new file mode 100644
index 000000000..e5a2574f3
--- /dev/null
+++ b/sdscompat.h
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2020, Michael Grunder <michael dot grunder at gmail dot com>
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * SDS compatibility header.
+ *
+ * This simple file maps sds types and calls to their unique hiredis symbol names.
+ * It's useful when we build Hiredis as a dependency of Redis and want to call
+ * Hiredis' sds symbols rather than the ones built into Redis, as the libraries
+ * have slightly diverged and could cause hard to track down ABI incompatibility
+ * bugs.
+ *
+ */
+
+#ifndef HIREDIS_SDS_COMPAT
+#define HIREDIS_SDS_COMPAT
+
+#define sds hisds
+
+#define sdslen hi_sdslen
+#define sdsavail hi_sdsavail
+#define sdssetlen hi_sdssetlen
+#define sdsinclen hi_sdsinclen
+#define sdsalloc hi_sdsalloc
+#define sdssetalloc hi_sdssetalloc
+
+#define sdsAllocPtr hi_sdsAllocPtr
+#define sdsAllocSize hi_sdsAllocSize
+#define sdscat hi_sdscat
+#define sdscatfmt hi_sdscatfmt
+#define sdscatlen hi_sdscatlen
+#define sdscatprintf hi_sdscatprintf
+#define sdscatrepr hi_sdscatrepr
+#define sdscatsds hi_sdscatsds
+#define sdscatvprintf hi_sdscatvprintf
+#define sdsclear hi_sdsclear
+#define sdscmp hi_sdscmp
+#define sdscpy hi_sdscpy
+#define sdscpylen hi_sdscpylen
+#define sdsdup hi_sdsdup
+#define sdsempty hi_sdsempty
+#define sds_free hi_sds_free
+#define sdsfree hi_sdsfree
+#define sdsfreesplitres hi_sdsfreesplitres
+#define sdsfromlonglong hi_sdsfromlonglong
+#define sdsgrowzero hi_sdsgrowzero
+#define sdsIncrLen hi_sdsIncrLen
+#define sdsjoin hi_sdsjoin
+#define sdsjoinsds hi_sdsjoinsds
+#define sdsll2str hi_sdsll2str
+#define sdsMakeRoomFor hi_sdsMakeRoomFor
+#define sds_malloc hi_sds_malloc
+#define sdsmapchars hi_sdsmapchars
+#define sdsnew hi_sdsnew
+#define sdsnewlen hi_sdsnewlen
+#define sdsrange hi_sdsrange
+#define sds_realloc hi_sds_realloc
+#define sdsRemoveFreeSpace hi_sdsRemoveFreeSpace
+#define sdssplitargs hi_sdssplitargs
+#define sdssplitlen hi_sdssplitlen
+#define sdstolower hi_sdstolower
+#define sdstoupper hi_sdstoupper
+#define sdstrim hi_sdstrim
+#define sdsull2str hi_sdsull2str
+#define sdsupdatelen hi_sdsupdatelen
+
+#endif /* HIREDIS_SDS_COMPAT */
diff --git a/ssl.c b/ssl.c
index 7df58fbde..fe9a2fdce 100644
--- a/ssl.c
+++ b/ssl.c
@@ -437,7 +437,7 @@ static ssize_t redisSSLRead(redisContext *c, char *buf, size_t bufcap) {
 static ssize_t redisSSLWrite(redisContext *c) {
     redisSSL *rssl = c->privctx;
 
-    size_t len = rssl->lastLen ? rssl->lastLen : sdslen(c->obuf);
+    size_t len = rssl->lastLen ? rssl->lastLen : hi_sdslen(c->obuf);
     int rv = SSL_write(rssl->ssl, c->obuf, len);
 
     if (rv > 0) {
diff --git a/test.c b/test.c
index c0eeca70d..012b6ad61 100644
--- a/test.c
+++ b/test.c
@@ -340,21 +340,21 @@ static void test_format_commands(void) {
         len == 4+4+(3+2)+4+(7+2)+4+(3+2));
     hi_free(cmd);
 
-    sds sds_cmd;
+    hisds sds_cmd;
 
     sds_cmd = NULL;
-    test("Format command into sds by passing argc/argv without lengths: ");
+    test("Format command into hisds by passing argc/argv without lengths: ");
     len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,NULL);
     test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 &&
         len == 4+4+(3+2)+4+(3+2)+4+(3+2));
-    sdsfree(sds_cmd);
+    hi_sdsfree(sds_cmd);
 
     sds_cmd = NULL;
-    test("Format command into sds by passing argc/argv with lengths: ");
+    test("Format command into hisds by passing argc/argv with lengths: ");
     len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,lens);
     test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 &&
         len == 4+4+(3+2)+4+(7+2)+4+(3+2));
-    sdsfree(sds_cmd);
+    hi_sdsfree(sds_cmd);
 }
 
 static void test_append_formatted_commands(struct config config) {

From 275068c86f3781fe016dfa4da2d59f1b486568e4 Mon Sep 17 00:00:00 2001
From: michael-grunder <michael.grunder@gmail.com>
Date: Sat, 15 Aug 2020 13:13:23 -0700
Subject: [PATCH 0583/1098] Use Hiredis' sdscompat.h to map sds* calls to
 hi_sds*

---
 src/redis-benchmark.c | 1 +
 src/redis-cli.c       | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index fc24afbff..a86079156 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -42,6 +42,7 @@
 #include <math.h>
 #include <pthread.h>
 
+#include <sdscompat.h> /* Use hiredis' sds compat header that maps sds calls to their hi_ variants */
 #include <sds.h> /* Use hiredis sds. */
 #include "ae.h"
 #include "hiredis.h"
diff --git a/src/redis-cli.c b/src/redis-cli.c
index 4df157d20..aefef2c5a 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -52,6 +52,7 @@
 #include <openssl/err.h>
 #include <hiredis_ssl.h>
 #endif
+#include <sdscompat.h> /* Use hiredis' sds compat header that maps sds calls to their hi_ variants */
 #include <sds.h> /* use sds.h from hiredis, so that only one set of sds functions will be present in the binary */
 #include "dict.h"
 #include "adlist.h"

From fb2a94af3fbb3f3cf8b26b8bd89387669cb111a1 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 17 Aug 2020 17:36:50 +0300
Subject: [PATCH 0584/1098] TLS: relax verification on CONFIG SET. (#7665)

Avoid re-configuring (and validating) SSL/TLS configuration on `CONFIG
SET` when TLS is not actively enabled for incoming connections, cluster
bus or replication.

This fixes failures when tests run without `--tls` on binaries that were
built with TLS support.

An additional benefit is that it's now possible to perform a multi-step
configuration process while TLS is disabled. The new configuration will
be verified and applied only when TLS is effectively enabled.
---
 .github/workflows/daily.yml | 26 ++++++++++++++++++++------
 src/config.c                |  5 ++++-
 2 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 010a32289..5b395351b 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -85,12 +85,19 @@ jobs:
         sudo apt-get install tcl8.5 tcl-tls
         ./utils/gen-test-certs.sh
         ./runtest --accurate --verbose --tls
+        ./runtest --accurate --verbose
     - name: module api test
-      run: ./runtest-moduleapi --verbose --tls
+      run: |
+        ./runtest-moduleapi --verbose --tls
+        ./runtest-moduleapi --verbose
     - name: sentinel tests
-      run: ./runtest-sentinel --tls
+      run: |
+        ./runtest-sentinel --tls
+        ./runtest-sentinel
     - name: cluster tests
-      run: ./runtest-cluster --tls
+      run: |
+        ./runtest-cluster --tls
+        ./runtest-cluster
 
   test-valgrind:
     runs-on: ubuntu-latest
@@ -147,10 +154,17 @@ jobs:
         yum -y install tcl tcltls
         ./utils/gen-test-certs.sh
         ./runtest --accurate --verbose --tls
+        ./runtest --accurate --verbose
     - name: module api test
-      run: ./runtest-moduleapi --verbose --tls
+      run: |
+        ./runtest-moduleapi --verbose --tls
+        ./runtest-moduleapi --verbose
     - name: sentinel tests
-      run: ./runtest-sentinel --tls
+      run: |
+        ./runtest-sentinel --tls
+        ./runtest-sentinel
     - name: cluster tests
-      run: ./runtest-cluster --tls
+      run: |
+        ./runtest-cluster --tls
+        ./runtest-cluster
 
diff --git a/src/config.c b/src/config.c
index 866665a68..5b73d9e28 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2205,7 +2205,10 @@ static int updateTlsCfg(char *val, char *prev, char **err) {
     UNUSED(val);
     UNUSED(prev);
     UNUSED(err);
-    if (tlsConfigure(&server.tls_ctx_config) == C_ERR) {
+
+    /* If TLS is enabled, try to configure OpenSSL. */
+    if ((server.tls_port || server.tls_replication || server.tls_cluster)
+            && tlsConfigure(&server.tls_ctx_config) == C_ERR) {
         *err = "Unable to update TLS configuration. Check server logs.";
         return 0;
     }

From 64c360c5156ca6ee6d1eb52bfeb3fa48f3b25da5 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 17 Aug 2020 17:46:54 +0300
Subject: [PATCH 0585/1098] Module API: fix missing RM_CLIENTINFO_FLAG_SSL.
 (#7666)

The `REDISMODULE_CLIENTINFO_FLAG_SSL` flag was already a part of the `RedisModuleClientInfo` structure but was not implemented.
---
 src/connection.c              |  8 +++++++-
 src/connection.h              |  9 +++++++++
 src/module.c                  |  2 ++
 src/tls.c                     |  7 +++++++
 tests/modules/misc.c          | 38 +++++++++++++++++++++++++++++++++++
 tests/unit/moduleapi/misc.tcl | 19 ++++++++++++++++++
 6 files changed, 82 insertions(+), 1 deletion(-)

diff --git a/src/connection.c b/src/connection.c
index 09fa12f2a..23b44a314 100644
--- a/src/connection.c
+++ b/src/connection.c
@@ -329,6 +329,11 @@ static ssize_t connSocketSyncReadLine(connection *conn, char *ptr, ssize_t size,
     return syncReadLine(conn->fd, ptr, size, timeout);
 }
 
+static int connSocketGetType(connection *conn) {
+    (void) conn;
+
+    return CONN_TYPE_SOCKET;
+}
 
 ConnectionType CT_Socket = {
     .ae_handler = connSocketEventHandler,
@@ -343,7 +348,8 @@ ConnectionType CT_Socket = {
     .blocking_connect = connSocketBlockingConnect,
     .sync_write = connSocketSyncWrite,
     .sync_read = connSocketSyncRead,
-    .sync_readline = connSocketSyncReadLine
+    .sync_readline = connSocketSyncReadLine,
+    .get_type = connSocketGetType
 };
 
 
diff --git a/src/connection.h b/src/connection.h
index 0fd6c5f24..85585a3d0 100644
--- a/src/connection.h
+++ b/src/connection.h
@@ -48,6 +48,9 @@ typedef enum {
 #define CONN_FLAG_CLOSE_SCHEDULED   (1<<0)      /* Closed scheduled by a handler */
 #define CONN_FLAG_WRITE_BARRIER     (1<<1)      /* Write barrier requested */
 
+#define CONN_TYPE_SOCKET            1
+#define CONN_TYPE_TLS               2
+
 typedef void (*ConnectionCallbackFunc)(struct connection *conn);
 
 typedef struct ConnectionType {
@@ -64,6 +67,7 @@ typedef struct ConnectionType {
     ssize_t (*sync_write)(struct connection *conn, char *ptr, ssize_t size, long long timeout);
     ssize_t (*sync_read)(struct connection *conn, char *ptr, ssize_t size, long long timeout);
     ssize_t (*sync_readline)(struct connection *conn, char *ptr, ssize_t size, long long timeout);
+    int (*get_type)(struct connection *conn);
 } ConnectionType;
 
 struct connection {
@@ -194,6 +198,11 @@ static inline ssize_t connSyncReadLine(connection *conn, char *ptr, ssize_t size
     return conn->type->sync_readline(conn, ptr, size, timeout);
 }
 
+/* Return CONN_TYPE_* for the specified connection */
+static inline int connGetType(connection *conn) {
+    return conn->type->get_type(conn);
+}
+
 connection *connCreateSocket();
 connection *connCreateAcceptedSocket(int fd);
 
diff --git a/src/module.c b/src/module.c
index 0714458f7..94a7344e1 100644
--- a/src/module.c
+++ b/src/module.c
@@ -1753,6 +1753,8 @@ int modulePopulateClientInfoStructure(void *ci, client *client, int structver) {
         ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_TRACKING;
     if (client->flags & CLIENT_BLOCKED)
         ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_BLOCKED;
+    if (connGetType(client->conn) == CONN_TYPE_TLS)
+        ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_SSL;
 
     int port;
     connPeerToString(client->conn,ci1->addr,sizeof(ci1->addr),&port);
diff --git a/src/tls.c b/src/tls.c
index 4f0ea4d65..52887cd23 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -823,6 +823,12 @@ exit:
     return nread;
 }
 
+static int connTLSGetType(connection *conn_) {
+    (void) conn_;
+
+    return CONN_TYPE_TLS;
+}
+
 ConnectionType CT_TLS = {
     .ae_handler = tlsEventHandler,
     .accept = connTLSAccept,
@@ -837,6 +843,7 @@ ConnectionType CT_TLS = {
     .sync_write = connTLSSyncWrite,
     .sync_read = connTLSSyncRead,
     .sync_readline = connTLSSyncReadLine,
+    .get_type = connTLSGetType
 };
 
 int tlsHasPendingData() {
diff --git a/tests/modules/misc.c b/tests/modules/misc.c
index 1048d5065..1f9cb1932 100644
--- a/tests/modules/misc.c
+++ b/tests/modules/misc.c
@@ -195,6 +195,42 @@ int test_setlfu(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     return REDISMODULE_OK;
 }
 
+int test_clientinfo(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    (void) argv;
+    (void) argc;
+
+    RedisModuleClientInfo ci = { .version = REDISMODULE_CLIENTINFO_VERSION };
+
+    if (RedisModule_GetClientInfoById(&ci, RedisModule_GetClientId(ctx)) == REDISMODULE_ERR) {
+            RedisModule_ReplyWithError(ctx, "failed to get client info");
+            return REDISMODULE_OK;
+    }
+
+    RedisModule_ReplyWithArray(ctx, 10);
+    char flags[512];
+    snprintf(flags, sizeof(flags) - 1, "%s:%s:%s:%s:%s:%s",
+        ci.flags & REDISMODULE_CLIENTINFO_FLAG_SSL ? "ssl" : "",
+        ci.flags & REDISMODULE_CLIENTINFO_FLAG_PUBSUB ? "pubsub" : "",
+        ci.flags & REDISMODULE_CLIENTINFO_FLAG_BLOCKED ? "blocked" : "",
+        ci.flags & REDISMODULE_CLIENTINFO_FLAG_TRACKING ? "tracking" : "",
+        ci.flags & REDISMODULE_CLIENTINFO_FLAG_UNIXSOCKET ? "unixsocket" : "",
+        ci.flags & REDISMODULE_CLIENTINFO_FLAG_MULTI ? "multi" : "");
+
+    RedisModule_ReplyWithCString(ctx, "flags");
+    RedisModule_ReplyWithCString(ctx, flags);
+    RedisModule_ReplyWithCString(ctx, "id");
+    RedisModule_ReplyWithLongLong(ctx, ci.id);
+    RedisModule_ReplyWithCString(ctx, "addr");
+    RedisModule_ReplyWithCString(ctx, ci.addr);
+    RedisModule_ReplyWithCString(ctx, "port");
+    RedisModule_ReplyWithLongLong(ctx, ci.port);
+    RedisModule_ReplyWithCString(ctx, "db");
+    RedisModule_ReplyWithLongLong(ctx, ci.db);
+
+    return REDISMODULE_OK;
+}
+
 int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
@@ -221,6 +257,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
         return REDISMODULE_ERR;
     if (RedisModule_CreateCommand(ctx,"test.getlfu", test_getlfu,"",0,0,0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
+    if (RedisModule_CreateCommand(ctx,"test.clientinfo", test_clientinfo,"",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
 
     return REDISMODULE_OK;
 }
diff --git a/tests/unit/moduleapi/misc.tcl b/tests/unit/moduleapi/misc.tcl
index 748016f1a..b57a94f6a 100644
--- a/tests/unit/moduleapi/misc.tcl
+++ b/tests/unit/moduleapi/misc.tcl
@@ -67,4 +67,23 @@ start_server {tags {"modules"}} {
         assert { $was_set == 0 }
     }
 
+    test {test module clientinfo api} {
+        # Test basic sanity and SSL flag
+        set info [r test.clientinfo]
+        set ssl_flag [expr $::tls ? {"ssl:"} : {":"}]
+
+        assert { [dict get $info db] == 9 }
+        assert { [dict get $info flags] == "${ssl_flag}::::" }
+
+        # Test MULTI flag
+        r multi
+        r test.clientinfo
+        set info [lindex [r exec] 0]
+        assert { [dict get $info flags] == "${ssl_flag}::::multi" }
+
+        # Test TRACKING flag
+        r client tracking on
+        set info [r test.clientinfo]
+        assert { [dict get $info flags] == "${ssl_flag}::tracking::" }
+    }
 }

From cdd925b2898ac270afdf3d72f065410a96980f80 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 18 Aug 2020 08:28:43 +0300
Subject: [PATCH 0586/1098] Trim trailing spaces in error replies coming from
 rejectCommand (#7668)

65a3307bc9 added rejectCommand which takes an robj reply and passes it
through addReplyErrorSafe to addReplyErrorLength.
The robj contains newline at it's end, but addReplyErrorSafe converts it
to spaces, and passes it to addReplyErrorLength which adds the protocol
newlines.

The result was that most error replies (like OOM) had extra two trailing
spaces in them.
---
 src/networking.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index 8ec85b393..f0ebe4134 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -412,10 +412,14 @@ void addReplyError(client *c, const char *err) {
  * is emitted. */
 void addReplyErrorSafe(client *c, char *s, size_t len) {
     size_t j;
+    /* Trim any newlines at the end (ones will be added by addReplyErrorLength) */
+    while (s[len-1] == '\r' || s[len-1] == '\n')
+        len--;
+    /* Replace any newlines in the rest of the string with spaces. */
     for (j = 0; j < len; j++) {
         if (s[j] == '\r' || s[j] == '\n') s[j] = ' ';
     }
-    addReplyErrorLength(c,s,sdslen(s));
+    addReplyErrorLength(c,s,len);
 }
 
 void addReplyErrorFormat(client *c, const char *fmt, ...) {

From 93d87d6d4cd2aed9a45c4307b4c7b0b19a47b2e9 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Tue, 18 Aug 2020 01:50:03 -0400
Subject: [PATCH 0587/1098] [module] using predefined REDISMODULE_NO_EXPIRE in
 RM_GetExpire (#7669)

It was already defined in the API header and the documentation, but not used by the implementation.
---
 src/module.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/module.c b/src/module.c
index 94a7344e1..2c90c599a 100644
--- a/src/module.c
+++ b/src/module.c
@@ -2159,7 +2159,8 @@ int RM_UnlinkKey(RedisModuleKey *key) {
  * REDISMODULE_NO_EXPIRE is returned. */
 mstime_t RM_GetExpire(RedisModuleKey *key) {
     mstime_t expire = getExpire(key->db,key->key);
-    if (expire == -1 || key->value == NULL) return -1;
+    if (expire == -1 || key->value == NULL) 
+        return REDISMODULE_NO_EXPIRE;
     expire -= mstime();
     return expire >= 0 ? expire : 0;
 }

From 88662c243d3b128141842d1f830b01d6b70e4960 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Tue, 18 Aug 2020 01:59:24 -0400
Subject: [PATCH 0588/1098] edit auth failed message (#7648)

Edit auth failed message include user disabled case in hello command
---
 src/acl.c        | 2 +-
 src/networking.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index e0432ba5c..987427899 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1981,7 +1981,7 @@ void authCommand(client *c) {
     if (ACLAuthenticateUser(c,username,password) == C_OK) {
         addReply(c,shared.ok);
     } else {
-        addReplyError(c,"-WRONGPASS invalid username-password pair");
+        addReplyError(c,"-WRONGPASS invalid username-password pair or user is disabled.");
     }
 
     /* Free the "default" string object we created for the two
diff --git a/src/networking.c b/src/networking.c
index f0ebe4134..a6786492b 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2527,7 +2527,7 @@ void helloCommand(client *c) {
         const char *opt = c->argv[j]->ptr;
         if (!strcasecmp(opt,"AUTH") && moreargs >= 2) {
             if (ACLAuthenticateUser(c, c->argv[j+1], c->argv[j+2]) == C_ERR) {
-                addReplyError(c,"-WRONGPASS invalid username-password pair");
+                addReplyError(c,"-WRONGPASS invalid username-password pair or user is disabled.");
                 return;
             }
             j += 2;

From 0f741a9e2de355211667c0f2ad38f7bead686c28 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 18 Aug 2020 09:53:59 +0300
Subject: [PATCH 0589/1098] OOM Crash log include size of allocation attempt.
 (#7670)

Since users often post just the crash log in github issues, the log
print that's above it is missing.
No reason not to include the size in the panic message itself.
---
 src/server.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index 5c083449b..7c381ce24 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5006,7 +5006,8 @@ void loadDataFromDisk(void) {
 void redisOutOfMemoryHandler(size_t allocation_size) {
     serverLog(LL_WARNING,"Out Of Memory allocating %zu bytes!",
         allocation_size);
-    serverPanic("Redis aborting for OUT OF MEMORY");
+    serverPanic("Redis aborting for OUT OF MEMORY. Allocating %zu bytes!", 
+        allocation_size);
 }
 
 void redisSetProcTitle(char *title) {

From b87c288016c68716cbb97534b7748045ca58c953 Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Tue, 18 Aug 2020 18:07:59 +0200
Subject: [PATCH 0590/1098] PERSIST should signalModifiedKey (Like EXPIRE does)
 (#7671)

---
 src/expire.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/expire.c b/src/expire.c
index f2d135e2b..1c4f71df3 100644
--- a/src/expire.c
+++ b/src/expire.c
@@ -594,6 +594,7 @@ void pttlCommand(client *c) {
 void persistCommand(client *c) {
     if (lookupKeyWrite(c->db,c->argv[1])) {
         if (removeExpire(c->db,c->argv[1])) {
+            signalModifiedKey(c,c->db,c->argv[1]);
             notifyKeyspaceEvent(NOTIFY_GENERIC,"persist",c->argv[1],c->db->id);
             addReply(c,shared.cone);
             server.dirty++;

From 89d544d6f263561160352db14afe8c70cdb7e707 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuan21@baidu.com>
Date: Wed, 19 Aug 2020 15:52:53 +0800
Subject: [PATCH 0591/1098] Add comments on 'slave.repldboff' when use diskless
 replication (#7679)

---
 src/replication.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/replication.c b/src/replication.c
index 1e1be345c..d629a197f 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1186,6 +1186,8 @@ void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData,
                 /* An error and still in connected state, is equivalent to EAGAIN */
                 slave->repldboff = 0;
             } else {
+                /* Note: when use diskless replication, 'repldboff' is the offset
+                 * of 'rdb_pipe_buff' sent rather than the offset of entire RDB. */
                 slave->repldboff = nwritten;
                 server.stat_net_output_bytes += nwritten;
             }

From cbd9af85838a73c9c9654829cf8e511ecc5853de Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Wed, 19 Aug 2020 19:07:43 -0700
Subject: [PATCH 0592/1098] Fixed hset error since it's shared with hmset
 (#7678)

---
 src/t_hash.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_hash.c b/src/t_hash.c
index 5dacd735b..e6f7b6be9 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -529,7 +529,7 @@ void hsetCommand(client *c) {
     robj *o;
 
     if ((c->argc % 2) == 1) {
-        addReplyError(c,"wrong number of arguments for HMSET");
+        addReplyErrorFormat(c,"wrong number of arguments for '%s' command",c->cmd->name);
         return;
     }
 

From 34c3be365a3a8bc74f7a99b3e35cb211afaa58ca Mon Sep 17 00:00:00 2001
From: Raghav Muddur <r@nmvk.com>
Date: Wed, 19 Aug 2020 19:13:32 -0700
Subject: [PATCH 0593/1098] Update clusterMsgDataPublish to clusterMsgModule
 (#7682)

Correcting the variable to clusterMsgModule.
---
 src/cluster.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cluster.c b/src/cluster.c
index 3bf8db9af..463503700 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -1767,7 +1767,7 @@ int clusterProcessPacket(clusterLink *link) {
     } else if (type == CLUSTERMSG_TYPE_MODULE) {
         uint32_t explen = sizeof(clusterMsg)-sizeof(union clusterMsgData);
 
-        explen += sizeof(clusterMsgDataPublish) -
+        explen += sizeof(clusterMsgModule) -
                 3 + ntohl(hdr->data.module.msg.len);
         if (totlen != explen) return 1;
     }

From cbaf3c5bbafd43e009a2d6b38dd0e9fc450a3e12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9D=A8=E5=8D=9A=E4=B8=9C?= <bodong.ybd@alibaba-inc.com>
Date: Thu, 20 Aug 2020 13:59:02 +0800
Subject: [PATCH 0594/1098] Fix flock cluster config may cause failure to
 restart after kill -9 (#7674)

After fork, the child process(redis-aof-rewrite) will get the fd opened
by the parent process(redis), when redis killed by kill -9, it will not
graceful exit(call prepareForShutdown()), so redis-aof-rewrite thread may still
alive, the fd(lock) will still be held by redis-aof-rewrite thread, and
redis restart will fail to get lock, means fail to start.

This issue was causing failures in the cluster tests in github actions.

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/cluster.c       | 11 ++++++++++-
 src/server.c        | 12 +++++++++++-
 src/server.h        |  1 +
 tests/instances.tcl | 14 +++++++++-----
 4 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 463503700..ba0ada04a 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -418,7 +418,15 @@ int clusterLockConfig(char *filename) {
         return C_ERR;
     }
     /* Lock acquired: leak the 'fd' by not closing it, so that we'll retain the
-     * lock to the file as long as the process exists. */
+     * lock to the file as long as the process exists.
+     *
+     * After fork, the child process will get the fd opened by the parent process,
+     * we need save `fd` to `cluster_config_file_lock_fd`, so that in redisFork(),
+     * it will be closed in the child process.
+     * If it is not closed, when the main process is killed -9, but the child process
+     * (redis-aof-rewrite) is still alive, the fd(lock) will still be held by the
+     * child process, and the main process will fail to get lock, means fail to start. */
+    server.cluster_config_file_lock_fd = fd;
 #endif /* __sun */
 
     return C_OK;
@@ -468,6 +476,7 @@ void clusterInit(void) {
 
     /* Lock the cluster config file to make sure every node uses
      * its own nodes.conf. */
+    server.cluster_config_file_lock_fd = -1;
     if (clusterLockConfig(server.cluster_configfile) == C_ERR)
         exit(1);
 
diff --git a/src/server.c b/src/server.c
index 7c381ce24..454c57862 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4917,14 +4917,24 @@ void setupChildSignalHandlers(void) {
     return;
 }
 
+/* After fork, the child process will inherit the resources
+ * of the parent process, e.g. fd(socket or flock) etc.
+ * should close the resources not used by the child process, so that if the
+ * parent restarts it can bind/lock despite the child possibly still running. */
+void closeClildUnusedResourceAfterFork() {
+    closeListeningSockets(0);
+    if (server.cluster_enabled && server.cluster_config_file_lock_fd != -1)
+        close(server.cluster_config_file_lock_fd);  /* don't care if this fails */
+}
+
 int redisFork() {
     int childpid;
     long long start = ustime();
     if ((childpid = fork()) == 0) {
         /* Child */
         setOOMScoreAdj(CONFIG_OOM_BGCHILD);
-        closeListeningSockets(0);
         setupChildSignalHandlers();
+        closeClildUnusedResourceAfterFork();
     } else {
         /* Parent */
         server.stat_fork_time = ustime()-start;
diff --git a/src/server.h b/src/server.h
index 4940b0c5b..9badea9e8 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1419,6 +1419,7 @@ struct redisServer {
                                       REDISMODULE_CLUSTER_FLAG_*. */
     int cluster_allow_reads_when_down; /* Are reads allowed when the cluster
                                         is down? */
+    int cluster_config_file_lock_fd;   /* cluster config fd, will be flock */
     /* Scripting */
     lua_State *lua; /* The Lua interpreter. We use just one for all clients */
     client *lua_client;   /* The "fake client" to query Redis from Lua */
diff --git a/tests/instances.tcl b/tests/instances.tcl
index e2aa4ab13..a43a4cc87 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -116,7 +116,9 @@ proc spawn_instance {type base_port count {conf {}}} {
 
         # Check availability finally
         if {[server_is_up 127.0.0.1 $port 100] == 0} {
-            abort_sentinel_test "Problems starting $type #$j: ping timeout"
+            set logfile [file join $dirname log.txt]
+            puts [exec tail $logfile]
+            abort_sentinel_test "Problems starting $type #$j: ping timeout, maybe server start failed, check $logfile"
         }
 
         # Push the instance into the right list
@@ -493,12 +495,12 @@ proc kill_instance {type id} {
 
     # Wait for the port it was using to be available again, so that's not
     # an issue to start a new server ASAP with the same port.
-    set retry 10
+    set retry 100
     while {[incr retry -1]} {
-        set port_is_free [catch {set s [socket 127.0.01 $port]}]
+        set port_is_free [catch {set s [socket 127.0.0.1 $port]}]
         if {$port_is_free} break
         catch {close $s}
-        after 1000
+        after 100
     }
     if {$retry == 0} {
         error "Port $port does not return available after killing instance."
@@ -525,7 +527,9 @@ proc restart_instance {type id} {
 
     # Check that the instance is running
     if {[server_is_up 127.0.0.1 $port 100] == 0} {
-        abort_sentinel_test "Problems starting $type #$id: ping timeout"
+        set logfile [file join $dirname log.txt]
+        puts [exec tail $logfile]
+        abort_sentinel_test "Problems starting $type #$id: ping timeout, maybe server start failed, check $logfile"
     }
 
     # Connect with it with a fresh link

From 65c24bd3d436a08a680fa80bf5b3f4f9cf8ef395 Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Thu, 20 Aug 2020 18:55:14 +0200
Subject: [PATCH 0595/1098] Modules: Invalidate saved_oparray after use (#7688)

We wanna avoid a chance of someone using the pointer in it after it'll be freed / realloced.
---
 src/module.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/module.c b/src/module.c
index 2c90c599a..d29b2e3f1 100644
--- a/src/module.c
+++ b/src/module.c
@@ -609,6 +609,8 @@ void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) {
         redisOpArrayFree(&server.also_propagate);
         /* Restore the previous oparray in case of nexted use of the API. */
         server.also_propagate = ctx->saved_oparray;
+        /* We're done with saved_oparray, let's invalidate it. */
+        redisOpArrayInit(&ctx->saved_oparray);
     }
 }
 

From a3d4d7bf68bf825584b34785ed2b117dd24c1754 Mon Sep 17 00:00:00 2001
From: huangzhw <huang_zhw@126.com>
Date: Fri, 21 Aug 2020 04:31:06 +0800
Subject: [PATCH 0596/1098] RedisModuleEvent_LoadingProgress always at 100%
 progress (#7685)

It was also using the wrong struct, but luckily RedisModuleFlushInfo and RedisModuleLoadingProgress
are identical.
---
 src/module.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/module.c b/src/module.c
index d29b2e3f1..ce1f1edc9 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7264,8 +7264,8 @@ void processModuleLoadingProgressEvent(int is_aof) {
         /* Fire the loading progress modules end event. */
         int progress = -1;
         if (server.loading_total_bytes)
-            progress = (server.loading_total_bytes<<10) / server.loading_total_bytes;
-        RedisModuleFlushInfoV1 fi = {REDISMODULE_LOADING_PROGRESS_VERSION,
+            progress = (server.loading_loaded_bytes<<10) / server.loading_total_bytes;
+        RedisModuleLoadingProgressV1 fi = {REDISMODULE_LOADING_PROGRESS_VERSION,
                                      server.hz,
                                      progress};
         moduleFireServerEvent(REDISMODULE_EVENT_LOADING_PROGRESS,

From 89f2bfbb58b907129de38917a7ae12c3b948daa6 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Thu, 20 Aug 2020 17:14:09 -0400
Subject: [PATCH 0597/1098] use dictSlots for getting total slots number in
 dict (#7691)

---
 src/dict.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/dict.c b/src/dict.c
index a26ef18d3..39af94707 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -621,9 +621,7 @@ dictEntry *dictGetRandomKey(dict *d)
         do {
             /* We are sure there are no elements in indexes from 0
              * to rehashidx-1 */
-            h = d->rehashidx + (random() % (d->ht[0].size +
-                                            d->ht[1].size -
-                                            d->rehashidx));
+            h = d->rehashidx + (random() % (dictSlots(d) - d->rehashidx));
             he = (h >= d->ht[0].size) ? d->ht[1].table[h - d->ht[0].size] :
                                       d->ht[0].table[h];
         } while(he == NULL);

From e61adc0d897074d8c2ca8e0f7bf08fa2985d9b01 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Fri, 21 Aug 2020 16:37:49 -0400
Subject: [PATCH 0598/1098] fix make warnings (#7692)

---
 src/server.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/server.c b/src/server.c
index 454c57862..5e2f6a8b0 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2551,9 +2551,6 @@ static void readOOMScoreAdj(void) {
  * depending on current role.
  */
 int setOOMScoreAdj(int process_class) {
-    int fd;
-    int val;
-    char buf[64];
 
     if (!server.oom_score_adj) return C_OK;
     if (process_class == -1)
@@ -2562,6 +2559,10 @@ int setOOMScoreAdj(int process_class) {
     serverAssert(process_class >= 0 && process_class < CONFIG_OOM_COUNT);
 
 #ifdef HAVE_PROC_OOM_SCORE_ADJ
+    int fd;
+    int val;
+    char buf[64];
+
     val = server.oom_score_adj_base + server.oom_score_adj_values[process_class];
     if (val > 1000) val = 1000;
     if (val < -1000) val = -1000;

From f80f3f492a0ca56e163899eeca7ad40d67d903be Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 23 Aug 2020 10:17:43 +0300
Subject: [PATCH 0599/1098] Tests: fix redis-cli with remote hosts. (#7693)

---
 tests/integration/redis-cli.tcl | 4 ++--
 tests/support/cli.tcl           | 4 ++--
 tests/unit/wait.tcl             | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/integration/redis-cli.tcl b/tests/integration/redis-cli.tcl
index 44ff430e2..2d4145ff0 100644
--- a/tests/integration/redis-cli.tcl
+++ b/tests/integration/redis-cli.tcl
@@ -3,7 +3,7 @@ source tests/support/cli.tcl
 start_server {tags {"cli"}} {
     proc open_cli {{opts "-n 9"} {infile ""}} {
         set ::env(TERM) dumb
-        set cmdline [rediscli [srv port] $opts]
+        set cmdline [rediscli [srv host] [srv port] $opts]
         if {$infile ne ""} {
             set cmdline "$cmdline < $infile"
             set mode "r"
@@ -65,7 +65,7 @@ start_server {tags {"cli"}} {
     }
 
     proc _run_cli {opts args} {
-        set cmd [rediscli [srv port] [list -n 9 {*}$args]]
+        set cmd [rediscli [srv host] [srv port] [list -n 9 {*}$args]]
         foreach {key value} $opts {
             if {$key eq "pipe"} {
                 set cmd "sh -c \"$value | $cmd\""
diff --git a/tests/support/cli.tcl b/tests/support/cli.tcl
index 37c902a50..d55487931 100644
--- a/tests/support/cli.tcl
+++ b/tests/support/cli.tcl
@@ -11,8 +11,8 @@ proc rediscli_tls_config {testsdir} {
     }
 }
 
-proc rediscli {port {opts {}}} {
-    set cmd [list src/redis-cli -p $port]
+proc rediscli {host port {opts {}}} {
+    set cmd [list src/redis-cli -h $host -p $port]
     lappend cmd {*}[rediscli_tls_config "tests"]
     lappend cmd {*}$opts
     return $cmd
diff --git a/tests/unit/wait.tcl b/tests/unit/wait.tcl
index c9cfa6ed4..b1500cff8 100644
--- a/tests/unit/wait.tcl
+++ b/tests/unit/wait.tcl
@@ -33,7 +33,7 @@ start_server {} {
     }
 
     test {WAIT should not acknowledge 1 additional copy if slave is blocked} {
-        set cmd [rediscli $slave_port "-h $slave_host debug sleep 5"]
+        set cmd [rediscli $slave_host $slave_port "debug sleep 5"]
         exec {*}$cmd > /dev/null 2> /dev/null &
         after 1000 ;# Give redis-cli the time to execute the command.
         $master set foo 0

From 9204a9b2c2f6eb59767ab0bddcde62c75e8c20b0 Mon Sep 17 00:00:00 2001
From: Valentino Geron <valentino@redislabs.com>
Date: Thu, 20 Aug 2020 18:48:09 +0300
Subject: [PATCH 0600/1098] Fix LPOS command when RANK is greater than matches

When calling to LPOS command when RANK is higher than matches,
the return value is non valid response. For example:
```
LPUSH l a
:1
LPOS l b RANK 5 COUNT 10
*-4
```
It may break client-side parser.

Now, we count how many replies were replied in the array.
```
LPUSH l a
:1
LPOS l b RANK 5 COUNT 10
*0
```
---
 src/t_list.c             | 5 +++--
 tests/unit/type/list.tcl | 6 ++++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/t_list.c b/src/t_list.c
index 7737da771..ecf09cd94 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -571,13 +571,14 @@ void lposCommand(client *c) {
     li = listTypeInitIterator(o,direction == LIST_HEAD ? -1 : 0,direction);
     listTypeEntry entry;
     long llen = listTypeLength(o);
-    long index = 0, matches = 0, matchindex = -1;
+    long index = 0, matches = 0, matchindex = -1, arraylen = 0;
     while (listTypeNext(li,&entry) && (maxlen == 0 || index < maxlen)) {
         if (listTypeEqual(&entry,ele)) {
             matches++;
             matchindex = (direction == LIST_TAIL) ? index : llen - index - 1;
             if (matches >= rank) {
                 if (arraylenptr) {
+                    arraylen++;
                     addReplyLongLong(c,matchindex);
                     if (count && matches-rank+1 >= count) break;
                 } else {
@@ -593,7 +594,7 @@ void lposCommand(client *c) {
     /* Reply to the client. Note that arraylenptr is not NULL only if
      * the COUNT option was selected. */
     if (arraylenptr != NULL) {
-        setDeferredArrayLen(c,arraylenptr,matches-rank+1);
+        setDeferredArrayLen(c,arraylenptr,arraylen);
     } else {
         if (matchindex != -1)
             addReplyLongLong(c,matchindex);
diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl
index 0e39d7d95..5ea2b9cd1 100644
--- a/tests/unit/type/list.tcl
+++ b/tests/unit/type/list.tcl
@@ -50,6 +50,12 @@ start_server {
         assert {[r LPOS mylist c COUNT 0 MAXLEN 7 RANK 2] == {6}}
     }
 
+    test {LPOS when RANK is greater than matches} {
+        r DEL mylist
+        r LPUSH l a
+        assert {[r LPOS mylist b COUNT 10 RANK 5] eq {}}
+    }
+
     test {LPUSH, RPUSH, LLENGTH, LINDEX, LPOP - ziplist} {
         # first lpush then rpush
         assert_equal 1 [r lpush myziplist1 aa]

From 8b428cf0f7ce5489fbdf4640a1dd36357f3dc5f0 Mon Sep 17 00:00:00 2001
From: Valentino Geron <valentino@redislabs.com>
Date: Sun, 23 Aug 2020 12:19:41 +0300
Subject: [PATCH 0601/1098] Assert that setDeferredAggregateLen isn't called
 with negative value

In case the redis is about to return broken reply we want to crash
with assert so that we are notified about the bug. see #7687.
---
 src/networking.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/networking.c b/src/networking.c
index a6786492b..24424ab90 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -492,6 +492,7 @@ void *addReplyDeferredLen(client *c) {
 
 /* Populate the length object and try gluing it to the next chunk. */
 void setDeferredAggregateLen(client *c, void *node, long length, char prefix) {
+    serverAssert(length >= 0);
     listNode *ln = (listNode*)node;
     clientReplyBlock *next;
     char lenstr[128];

From 5449a2a8b54db6a57b564e453c9f7075ebb81f79 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 24 Aug 2020 12:54:56 +0300
Subject: [PATCH 0602/1098] Add language servers stuff, test/tls to gitignore.
 (#7698)

---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index e445fd201..721f1d74e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,9 +27,13 @@ src/nodes.conf
 deps/lua/src/lua
 deps/lua/src/luac
 deps/lua/src/liblua.a
+tests/tls/*
 .make-*
 .prerequisites
 *.dSYM
 Makefile.dep
 .vscode/*
 .idea/*
+.ccls
+.ccls-cache/*
+compile_commands.json

From 6b4ae919e8f3ebe883902e10305d29a083b3c679 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuan21@baidu.com>
Date: Mon, 24 Aug 2020 18:54:33 +0800
Subject: [PATCH 0603/1098] Fix data race in bugReportStart (#7700)

The previous fix using _Atomic was insufficient, since we check and set it in
different places.

The implications of this bug are just that a portion of the bug report will be shown
twice, in the race case of two concurrent crashes.
---
 src/debug.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/debug.c b/src/debug.c
index 5cfda59c9..b05dc344e 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -55,7 +55,8 @@ typedef ucontext_t sigcontext_t;
 #endif
 
 /* Globals */
-static _Atomic int bug_report_start = 0; /* True if bug report header was already logged. */
+static int bug_report_start = 0; /* True if bug report header was already logged. */
+static pthread_mutex_t bug_report_start_mutex = PTHREAD_MUTEX_INITIALIZER;
 
 /* Forward declarations */
 void bugReportStart(void);
@@ -919,11 +920,13 @@ void _serverPanic(const char *file, int line, const char *msg, ...) {
 }
 
 void bugReportStart(void) {
+    pthread_mutex_lock(&bug_report_start_mutex);
     if (bug_report_start == 0) {
         serverLogRaw(LL_WARNING|LL_RAW,
         "\n\n=== REDIS BUG REPORT START: Cut & paste starting from here ===\n");
         bug_report_start = 1;
     }
+    pthread_mutex_unlock(&bug_report_start_mutex);
 }
 
 #ifdef HAVE_BACKTRACE

From 43af28f5b487370bd3d65d00be93c4a23ee42fa7 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuan21@baidu.com>
Date: Tue, 25 Aug 2020 03:59:56 +0800
Subject: [PATCH 0604/1098] Fix wrong format specifiers of 'sdscatfmt' for the
 INFO command (#7706)

unlike printf, sdscatfmt doesn't take %d
---
 src/server.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index 5e2f6a8b0..70307ff0b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4141,7 +4141,7 @@ sds genRedisInfoString(const char *section) {
             "lru_clock:%u\r\n"
             "executable:%s\r\n"
             "config_file:%s\r\n"
-            "io_threads_active:%d\r\n",
+            "io_threads_active:%i\r\n",
             REDIS_VERSION,
             redisGitSHA1(),
             strtol(redisGitDirty(),NULL,10) > 0,

From 5b0a06af48997794af60dabb58ce4336ef56f73d Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Tue, 25 Aug 2020 15:58:50 +0300
Subject: [PATCH 0605/1098] Expands lazyfree's effort estimate to include
 Streams (#5794)

Otherwise, it is treated as a single allocation and freed synchronously. The following logic is used for estimating the effort in constant-ish time complexity:

1. Check the number of nodes.
1. Add an allocation for each consumer group registered inside the stream.
1. Check the number of PELs in the first CG, and then add this count times the number of CGs.
1. Check the number of consumers in the first CG, and then add this count times the number of CGs.
---
 src/lazyfree.c | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/src/lazyfree.c b/src/lazyfree.c
index f01504e70..cbcc1c240 100644
--- a/src/lazyfree.c
+++ b/src/lazyfree.c
@@ -41,6 +41,30 @@ size_t lazyfreeGetFreeEffort(robj *obj) {
     } else if (obj->type == OBJ_HASH && obj->encoding == OBJ_ENCODING_HT) {
         dict *ht = obj->ptr;
         return dictSize(ht);
+    } else if (obj->type == OBJ_STREAM) {
+        size_t effort = 0;
+        stream *s = obj->ptr;
+
+        /* Make a best effort estimate to maintain constant runtime. Every macro
+         * node in the Stream is one allocation. */
+        effort += s->rax->numnodes;
+
+        /* Every consumer group is an allocation and so are the entries in its
+         * PEL. We use size of the first group's PEL as an estimate for all
+         * others. */
+        if (s->cgroups) {
+            raxIterator ri;
+            streamCG *cg;
+            raxStart(&ri,s->cgroups);
+            raxSeek(&ri,"^",NULL,0);
+            /* There must be at least one group so the following should always
+             * work. */
+            serverAssert(raxNext(&ri));
+            cg = ri.data;
+            effort += raxSize(s->cgroups)*(1+raxSize(cg->pel));
+            raxStop(&ri);
+        }
+        return effort;
     } else {
         return 1; /* Everything else is a single allocation. */
     }

From 21784def707d3b36e17619dc620917eeea805bc8 Mon Sep 17 00:00:00 2001
From: filipe oliveira <filipecosta.90@gmail.com>
Date: Tue, 25 Aug 2020 19:21:29 +0100
Subject: [PATCH 0606/1098] Extended redis-benchmark instant metrics and
 overall latency report (#7600)

A first step to enable a consistent full percentile analysis on query latency so that we can fully understand the performance and stability characteristics of the redis-server system we are measuring. It also improves the instantaneous reported metrics, and the csv output format.
---
 deps/Makefile                      |    7 +
 deps/hdr_histogram/COPYING.txt     |  121 +++
 deps/hdr_histogram/LICENSE.txt     |   41 +
 deps/hdr_histogram/Makefile        |   20 +
 deps/hdr_histogram/README.md       |   10 +
 deps/hdr_histogram/hdr_atomic.h    |  146 ++++
 deps/hdr_histogram/hdr_histogram.c | 1155 ++++++++++++++++++++++++++++
 deps/hdr_histogram/hdr_histogram.h |  509 ++++++++++++
 src/Makefile                       |    6 +-
 src/redis-benchmark.c              |  166 ++--
 10 files changed, 2125 insertions(+), 56 deletions(-)
 create mode 100644 deps/hdr_histogram/COPYING.txt
 create mode 100644 deps/hdr_histogram/LICENSE.txt
 create mode 100644 deps/hdr_histogram/Makefile
 create mode 100644 deps/hdr_histogram/README.md
 create mode 100644 deps/hdr_histogram/hdr_atomic.h
 create mode 100644 deps/hdr_histogram/hdr_histogram.c
 create mode 100644 deps/hdr_histogram/hdr_histogram.h

diff --git a/deps/Makefile b/deps/Makefile
index 700867f3b..0815d15de 100644
--- a/deps/Makefile
+++ b/deps/Makefile
@@ -37,6 +37,7 @@ distclean:
 	-(cd linenoise && $(MAKE) clean) > /dev/null || true
 	-(cd lua && $(MAKE) clean) > /dev/null || true
 	-(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true
+	-(cd hdr_histogram && $(MAKE) clean) > /dev/null || true
 	-(rm -f .make-*)
 
 .PHONY: distclean
@@ -57,6 +58,12 @@ linenoise: .make-prerequisites
 
 .PHONY: linenoise
 
+hdr_histogram: .make-prerequisites
+	@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
+	cd hdr_histogram && $(MAKE)
+
+.PHONY: hdr_histogram
+
 ifeq ($(uname_S),SunOS)
 	# Make isinf() available
 	LUA_CFLAGS= -D__C99FEATURES__=1
diff --git a/deps/hdr_histogram/COPYING.txt b/deps/hdr_histogram/COPYING.txt
new file mode 100644
index 000000000..0e259d42c
--- /dev/null
+++ b/deps/hdr_histogram/COPYING.txt
@@ -0,0 +1,121 @@
+Creative Commons Legal Code
+
+CC0 1.0 Universal
+
+    CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+    LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+    ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+    INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+    REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+    PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+    THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+    HEREUNDER.
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator
+and subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for
+the purpose of contributing to a commons of creative, cultural and
+scientific works ("Commons") that the public can reliably and without fear
+of later claims of infringement build upon, modify, incorporate in other
+works, reuse and redistribute as freely as possible in any form whatsoever
+and for any purposes, including without limitation commercial purposes.
+These owners may contribute to the Commons to promote the ideal of a free
+culture and the further production of creative, cultural and scientific
+works, or to gain reputation or greater distribution for their Work in
+part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any
+expectation of additional consideration or compensation, the person
+associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+is an owner of Copyright and Related Rights in the Work, voluntarily
+elects to apply CC0 to the Work and publicly distribute the Work under its
+terms, with knowledge of his or her Copyright and Related Rights in the
+Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not
+limited to, the following:
+
+  i. the right to reproduce, adapt, distribute, perform, display,
+     communicate, and translate a Work;
+ ii. moral rights retained by the original author(s) and/or performer(s);
+iii. publicity and privacy rights pertaining to a person's image or
+     likeness depicted in a Work;
+ iv. rights protecting against unfair competition in regards to a Work,
+     subject to the limitations in paragraph 4(a), below;
+  v. rights protecting the extraction, dissemination, use and reuse of data
+     in a Work;
+ vi. database rights (such as those arising under Directive 96/9/EC of the
+     European Parliament and of the Council of 11 March 1996 on the legal
+     protection of databases, and under any national implementation
+     thereof, including any amended or successor version of such
+     directive); and
+vii. other similar, equivalent or corresponding rights throughout the
+     world based on applicable law or treaty, and any national
+     implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention
+of, applicable law, Affirmer hereby overtly, fully, permanently,
+irrevocably and unconditionally waives, abandons, and surrenders all of
+Affirmer's Copyright and Related Rights and associated claims and causes
+of action, whether now known or unknown (including existing as well as
+future claims and causes of action), in the Work (i) in all territories
+worldwide, (ii) for the maximum duration provided by applicable law or
+treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional
+purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+member of the public at large and to the detriment of Affirmer's heirs and
+successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public
+as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason
+be judged legally invalid or ineffective under applicable law, then the
+Waiver shall be preserved to the maximum extent permitted taking into
+account Affirmer's express Statement of Purpose. In addition, to the
+extent the Waiver is so judged Affirmer hereby grants to each affected
+person a royalty-free, non transferable, non sublicensable, non exclusive,
+irrevocable and unconditional license to exercise Affirmer's Copyright and
+Related Rights in the Work (i) in all territories worldwide, (ii) for the
+maximum duration provided by applicable law or treaty (including future
+time extensions), (iii) in any current or future medium and for any number
+of copies, and (iv) for any purpose whatsoever, including without
+limitation commercial, advertising or promotional purposes (the
+"License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any
+reason be judged legally invalid or ineffective under applicable law, such
+partial invalidity or ineffectiveness shall not invalidate the remainder
+of the License, and in such case Affirmer hereby affirms that he or she
+will not (i) exercise any of his or her remaining Copyright and Related
+Rights in the Work or (ii) assert any associated claims and causes of
+action with respect to the Work, in either case contrary to Affirmer's
+express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+    surrendered, licensed or otherwise affected by this document.
+ b. Affirmer offers the Work as-is and makes no representations or
+    warranties of any kind concerning the Work, express, implied,
+    statutory or otherwise, including without limitation warranties of
+    title, merchantability, fitness for a particular purpose, non
+    infringement, or the absence of latent or other defects, accuracy, or
+    the present or absence of errors, whether or not discoverable, all to
+    the greatest extent permissible under applicable law.
+ c. Affirmer disclaims responsibility for clearing rights of other persons
+    that may apply to the Work or any use thereof, including without
+    limitation any person's Copyright and Related Rights in the Work.
+    Further, Affirmer disclaims responsibility for obtaining any necessary
+    consents, permissions or other rights required for any use of the
+    Work.
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+    party to this document and has no duty or obligation with respect to
+    this CC0 or use of the Work.
diff --git a/deps/hdr_histogram/LICENSE.txt b/deps/hdr_histogram/LICENSE.txt
new file mode 100644
index 000000000..9b4e66ed7
--- /dev/null
+++ b/deps/hdr_histogram/LICENSE.txt
@@ -0,0 +1,41 @@
+The code in this repository code was Written by Gil Tene, Michael Barker,
+and Matt Warren, and released to the public domain, as explained at
+http://creativecommons.org/publicdomain/zero/1.0/
+
+For users of this code who wish to consume it under the "BSD" license
+rather than under the public domain or CC0 contribution text mentioned
+above, the code found under this directory is *also* provided under the
+following license (commonly referred to as the BSD 2-Clause License). This
+license does not detract from the above stated release of the code into
+the public domain, and simply represents an additional license granted by
+the Author.
+
+-----------------------------------------------------------------------------
+** Beginning of "BSD 2-Clause License" text. **
+
+ Copyright (c) 2012, 2013, 2014 Gil Tene
+ Copyright (c) 2014 Michael Barker
+ Copyright (c) 2014 Matt Warren
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/deps/hdr_histogram/Makefile b/deps/hdr_histogram/Makefile
new file mode 100644
index 000000000..83d3760ae
--- /dev/null
+++ b/deps/hdr_histogram/Makefile
@@ -0,0 +1,20 @@
+STD=
+WARN= -Wall
+OPT= -Os
+
+R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS)
+R_LDFLAGS= $(LDFLAGS)
+DEBUG= -g
+
+R_CC=$(CC) $(R_CFLAGS)
+R_LD=$(CC) $(R_LDFLAGS)
+
+hdr_histogram.o: hdr_histogram.h hdr_histogram.c 
+
+.c.o:
+	$(R_CC) -c  $< 
+
+clean:
+	rm -f *.o
+
+
diff --git a/deps/hdr_histogram/README.md b/deps/hdr_histogram/README.md
new file mode 100644
index 000000000..5f62c234c
--- /dev/null
+++ b/deps/hdr_histogram/README.md
@@ -0,0 +1,10 @@
+HdrHistogram_c v0.11.0
+
+----------------------------------------------
+
+This port contains a subset of the 'C' version of High Dynamic Range (HDR) Histogram available at [github.com/HdrHistogram/HdrHistogram_c](https://github.com/HdrHistogram/HdrHistogram_c).
+
+
+The code present on `hdr_histogram.c`, `hdr_histogram.h`, and `hdr_atomic.c` was Written by Gil Tene, Michael Barker,
+and Matt Warren, and released to the public domain, as explained at
+http://creativecommons.org/publicdomain/zero/1.0/.
\ No newline at end of file
diff --git a/deps/hdr_histogram/hdr_atomic.h b/deps/hdr_histogram/hdr_atomic.h
new file mode 100644
index 000000000..ae1056a83
--- /dev/null
+++ b/deps/hdr_histogram/hdr_atomic.h
@@ -0,0 +1,146 @@
+/**
+ * hdr_atomic.h
+ * Written by Philip Orwig and released to the public domain,
+ * as explained at http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+#ifndef HDR_ATOMIC_H__
+#define HDR_ATOMIC_H__
+
+
+#if defined(_MSC_VER)
+
+#include <stdint.h>
+#include <intrin.h>
+#include <stdbool.h>
+
+static void __inline * hdr_atomic_load_pointer(void** pointer)
+{
+	_ReadBarrier();
+	return *pointer;
+}
+
+static void hdr_atomic_store_pointer(void** pointer, void* value)
+{
+	_WriteBarrier();
+	*pointer = value;
+}
+
+static int64_t __inline hdr_atomic_load_64(int64_t* field)
+{ 
+	_ReadBarrier();
+	return *field;
+}
+
+static void __inline hdr_atomic_store_64(int64_t* field, int64_t value)
+{
+	_WriteBarrier();
+	*field = value;
+}
+
+static int64_t __inline hdr_atomic_exchange_64(volatile int64_t* field, int64_t value)
+{
+#if defined(_WIN64)
+    return _InterlockedExchange64(field, value);
+#else
+    int64_t comparand;
+    int64_t initial_value = *field;
+    do
+    {
+        comparand = initial_value;
+        initial_value = _InterlockedCompareExchange64(field, value, comparand);
+    }
+    while (comparand != initial_value);
+
+    return initial_value;
+#endif
+}
+
+static int64_t __inline hdr_atomic_add_fetch_64(volatile int64_t* field, int64_t value)
+{
+#if defined(_WIN64)
+	return _InterlockedExchangeAdd64(field, value) + value;
+#else
+    int64_t comparand;
+    int64_t initial_value = *field;
+    do
+    {
+        comparand = initial_value;
+        initial_value = _InterlockedCompareExchange64(field, comparand + value, comparand);
+    }
+    while (comparand != initial_value);
+
+    return initial_value + value;
+#endif
+}
+
+static bool __inline hdr_atomic_compare_exchange_64(volatile int64_t* field, int64_t* expected, int64_t desired)
+{
+    return *expected == _InterlockedCompareExchange64(field, desired, *expected);
+}
+
+#elif defined(__ATOMIC_SEQ_CST)
+
+#define hdr_atomic_load_pointer(x) __atomic_load_n(x, __ATOMIC_SEQ_CST)
+#define hdr_atomic_store_pointer(f,v) __atomic_store_n(f,v, __ATOMIC_SEQ_CST)
+#define hdr_atomic_load_64(x) __atomic_load_n(x, __ATOMIC_SEQ_CST)
+#define hdr_atomic_store_64(f,v) __atomic_store_n(f,v, __ATOMIC_SEQ_CST)
+#define hdr_atomic_exchange_64(f,i) __atomic_exchange_n(f,i, __ATOMIC_SEQ_CST)
+#define hdr_atomic_add_fetch_64(field, value) __atomic_add_fetch(field, value, __ATOMIC_SEQ_CST)
+#define hdr_atomic_compare_exchange_64(field, expected, desired) __atomic_compare_exchange_n(field, expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
+
+#elif defined(__x86_64__)
+
+#include <stdint.h>
+#include <stdbool.h>
+
+static inline void* hdr_atomic_load_pointer(void** pointer)
+{
+   void* p =  *pointer;
+	asm volatile ("" ::: "memory");
+	return p;
+}
+
+static inline void hdr_atomic_store_pointer(void** pointer, void* value)
+{
+    asm volatile ("lock; xchgq %0, %1" : "+q" (value), "+m" (*pointer));
+}
+
+static inline int64_t hdr_atomic_load_64(int64_t* field)
+{
+    int64_t i = *field;
+	asm volatile ("" ::: "memory");
+	return i;
+}
+
+static inline void hdr_atomic_store_64(int64_t* field, int64_t value)
+{
+    asm volatile ("lock; xchgq %0, %1" : "+q" (value), "+m" (*field));
+}
+
+static inline int64_t hdr_atomic_exchange_64(volatile int64_t* field, int64_t value)
+{
+    int64_t result = 0;
+    asm volatile ("lock; xchgq %1, %2" : "=r" (result), "+q" (value), "+m" (*field));
+    return result;
+}
+
+static inline int64_t hdr_atomic_add_fetch_64(volatile int64_t* field, int64_t value)
+{
+    return __sync_add_and_fetch(field, value);
+}
+
+static inline bool hdr_atomic_compare_exchange_64(volatile int64_t* field, int64_t* expected, int64_t desired)
+{
+    int64_t original;
+    asm volatile( "lock; cmpxchgq %2, %1" : "=a"(original), "+m"(*field) : "q"(desired), "0"(*expected));
+    return original == *expected;
+}
+
+#else
+
+#error "Unable to determine atomic operations for your platform"
+
+#endif
+
+#endif /* HDR_ATOMIC_H__ */
diff --git a/deps/hdr_histogram/hdr_histogram.c b/deps/hdr_histogram/hdr_histogram.c
new file mode 100644
index 000000000..5f5286f2f
--- /dev/null
+++ b/deps/hdr_histogram/hdr_histogram.c
@@ -0,0 +1,1155 @@
+/**
+ * hdr_histogram.c
+ * Written by Michael Barker and released to the public domain,
+ * as explained at http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+#include <stdlib.h>
+#include <stdbool.h>
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <inttypes.h>
+
+#include "hdr_histogram.h"
+#include "hdr_atomic.h"
+
+/*  ######   #######  ##     ## ##    ## ########  ######  */
+/* ##    ## ##     ## ##     ## ###   ##    ##    ##    ## */
+/* ##       ##     ## ##     ## ####  ##    ##    ##       */
+/* ##       ##     ## ##     ## ## ## ##    ##     ######  */
+/* ##       ##     ## ##     ## ##  ####    ##          ## */
+/* ##    ## ##     ## ##     ## ##   ###    ##    ##    ## */
+/*  ######   #######   #######  ##    ##    ##     ######  */
+
+static int32_t normalize_index(const struct hdr_histogram* h, int32_t index)
+{
+    int32_t normalized_index;
+    int32_t adjustment = 0;
+    if (h->normalizing_index_offset == 0)
+    {
+        return index;
+    }
+
+    normalized_index = index - h->normalizing_index_offset;
+
+    if (normalized_index < 0)
+    {
+        adjustment = h->counts_len;
+    }
+    else if (normalized_index >= h->counts_len)
+    {
+        adjustment = -h->counts_len;
+    }
+
+    return normalized_index + adjustment;
+}
+
+static int64_t counts_get_direct(const struct hdr_histogram* h, int32_t index)
+{
+    return h->counts[index];
+}
+
+static int64_t counts_get_normalised(const struct hdr_histogram* h, int32_t index)
+{
+    return counts_get_direct(h, normalize_index(h, index));
+}
+
+static void counts_inc_normalised(
+    struct hdr_histogram* h, int32_t index, int64_t value)
+{
+    int32_t normalised_index = normalize_index(h, index);
+    h->counts[normalised_index] += value;
+    h->total_count += value;
+}
+
+static void counts_inc_normalised_atomic(
+    struct hdr_histogram* h, int32_t index, int64_t value)
+{
+    int32_t normalised_index = normalize_index(h, index);
+
+    hdr_atomic_add_fetch_64(&h->counts[normalised_index], value);
+    hdr_atomic_add_fetch_64(&h->total_count, value);
+}
+
+static void update_min_max(struct hdr_histogram* h, int64_t value)
+{
+    h->min_value = (value < h->min_value && value != 0) ? value : h->min_value;
+    h->max_value = (value > h->max_value) ? value : h->max_value;
+}
+
+static void update_min_max_atomic(struct hdr_histogram* h, int64_t value)
+{
+    int64_t current_min_value;
+    int64_t current_max_value;
+    do
+    {
+        current_min_value = hdr_atomic_load_64(&h->min_value);
+
+        if (0 == value || current_min_value <= value)
+        {
+            break;
+        }
+    }
+    while (!hdr_atomic_compare_exchange_64(&h->min_value, &current_min_value, value));
+
+    do
+    {
+        current_max_value = hdr_atomic_load_64(&h->max_value);
+
+        if (value <= current_max_value)
+        {
+            break;
+        }
+    }
+    while (!hdr_atomic_compare_exchange_64(&h->max_value, &current_max_value, value));
+}
+
+
+/* ##     ## ######## #### ##       #### ######## ##    ## */
+/* ##     ##    ##     ##  ##        ##     ##     ##  ##  */
+/* ##     ##    ##     ##  ##        ##     ##      ####   */
+/* ##     ##    ##     ##  ##        ##     ##       ##    */
+/* ##     ##    ##     ##  ##        ##     ##       ##    */
+/* ##     ##    ##     ##  ##        ##     ##       ##    */
+/*  #######     ##    #### ######## ####    ##       ##    */
+
+static int64_t power(int64_t base, int64_t exp)
+{
+    int64_t result = 1;
+    while(exp)
+    {
+        result *= base; exp--;
+    }
+    return result;
+}
+
+#if defined(_MSC_VER)
+#   if defined(_WIN64)
+#       pragma intrinsic(_BitScanReverse64)
+#   else
+#       pragma intrinsic(_BitScanReverse)
+#   endif
+#endif
+
+static int32_t count_leading_zeros_64(int64_t value)
+{
+#if defined(_MSC_VER)
+    uint32_t leading_zero = 0;
+#if defined(_WIN64)
+    _BitScanReverse64(&leading_zero, value);
+#else
+    uint32_t high = value >> 32;
+    if  (_BitScanReverse(&leading_zero, high))
+    {
+        leading_zero += 32;
+    }
+    else
+    {
+        uint32_t low = value & 0x00000000FFFFFFFF;
+        _BitScanReverse(&leading_zero, low);
+    }
+#endif
+    return 63 - leading_zero; /* smallest power of 2 containing value */
+#else
+    return __builtin_clzll(value); /* smallest power of 2 containing value */
+#endif
+}
+
+static int32_t get_bucket_index(const struct hdr_histogram* h, int64_t value)
+{
+    int32_t pow2ceiling = 64 - count_leading_zeros_64(value | h->sub_bucket_mask); /* smallest power of 2 containing value */
+    return pow2ceiling - h->unit_magnitude - (h->sub_bucket_half_count_magnitude + 1);
+}
+
+static int32_t get_sub_bucket_index(int64_t value, int32_t bucket_index, int32_t unit_magnitude)
+{
+    return (int32_t)(value >> (bucket_index + unit_magnitude));
+}
+
+static int32_t counts_index(const struct hdr_histogram* h, int32_t bucket_index, int32_t sub_bucket_index)
+{
+    /* Calculate the index for the first entry in the bucket: */
+    /* (The following is the equivalent of ((bucket_index + 1) * subBucketHalfCount) ): */
+    int32_t bucket_base_index = (bucket_index + 1) << h->sub_bucket_half_count_magnitude;
+    /* Calculate the offset in the bucket: */
+    int32_t offset_in_bucket = sub_bucket_index - h->sub_bucket_half_count;
+    /* The following is the equivalent of ((sub_bucket_index  - subBucketHalfCount) + bucketBaseIndex; */
+    return bucket_base_index + offset_in_bucket;
+}
+
+static int64_t value_from_index(int32_t bucket_index, int32_t sub_bucket_index, int32_t unit_magnitude)
+{
+    return ((int64_t) sub_bucket_index) << (bucket_index + unit_magnitude);
+}
+
+int32_t counts_index_for(const struct hdr_histogram* h, int64_t value)
+{
+    int32_t bucket_index     = get_bucket_index(h, value);
+    int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, h->unit_magnitude);
+
+    return counts_index(h, bucket_index, sub_bucket_index);
+}
+
+int64_t hdr_value_at_index(const struct hdr_histogram *h, int32_t index)
+{
+    int32_t bucket_index = (index >> h->sub_bucket_half_count_magnitude) - 1;
+    int32_t sub_bucket_index = (index & (h->sub_bucket_half_count - 1)) + h->sub_bucket_half_count;
+
+    if (bucket_index < 0)
+    {
+        sub_bucket_index -= h->sub_bucket_half_count;
+        bucket_index = 0;
+    }
+
+    return value_from_index(bucket_index, sub_bucket_index, h->unit_magnitude);
+}
+
+int64_t hdr_size_of_equivalent_value_range(const struct hdr_histogram* h, int64_t value)
+{
+    int32_t bucket_index     = get_bucket_index(h, value);
+    int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, h->unit_magnitude);
+    int32_t adjusted_bucket  = (sub_bucket_index >= h->sub_bucket_count) ? (bucket_index + 1) : bucket_index;
+    return INT64_C(1) << (h->unit_magnitude + adjusted_bucket);
+}
+
+static int64_t lowest_equivalent_value(const struct hdr_histogram* h, int64_t value)
+{
+    int32_t bucket_index     = get_bucket_index(h, value);
+    int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, h->unit_magnitude);
+    return value_from_index(bucket_index, sub_bucket_index, h->unit_magnitude);
+}
+
+int64_t hdr_next_non_equivalent_value(const struct hdr_histogram *h, int64_t value)
+{
+    return lowest_equivalent_value(h, value) + hdr_size_of_equivalent_value_range(h, value);
+}
+
+static int64_t highest_equivalent_value(const struct hdr_histogram* h, int64_t value)
+{
+    return hdr_next_non_equivalent_value(h, value) - 1;
+}
+
+int64_t hdr_median_equivalent_value(const struct hdr_histogram *h, int64_t value)
+{
+    return lowest_equivalent_value(h, value) + (hdr_size_of_equivalent_value_range(h, value) >> 1);
+}
+
+static int64_t non_zero_min(const struct hdr_histogram* h)
+{
+    if (INT64_MAX == h->min_value)
+    {
+        return INT64_MAX;
+    }
+
+    return lowest_equivalent_value(h, h->min_value);
+}
+
+void hdr_reset_internal_counters(struct hdr_histogram* h)
+{
+    int min_non_zero_index = -1;
+    int max_index = -1;
+    int64_t observed_total_count = 0;
+    int i;
+
+    for (i = 0; i < h->counts_len; i++)
+    {
+        int64_t count_at_index;
+
+        if ((count_at_index = counts_get_direct(h, i)) > 0)
+        {
+            observed_total_count += count_at_index;
+            max_index = i;
+            if (min_non_zero_index == -1 && i != 0)
+            {
+                min_non_zero_index = i;
+            }
+        }
+    }
+
+    if (max_index == -1)
+    {
+        h->max_value = 0;
+    }
+    else
+    {
+        int64_t max_value = hdr_value_at_index(h, max_index);
+        h->max_value = highest_equivalent_value(h, max_value);
+    }
+
+    if (min_non_zero_index == -1)
+    {
+        h->min_value = INT64_MAX;
+    }
+    else
+    {
+        h->min_value = hdr_value_at_index(h, min_non_zero_index);
+    }
+
+    h->total_count = observed_total_count;
+}
+
+static int32_t buckets_needed_to_cover_value(int64_t value, int32_t sub_bucket_count, int32_t unit_magnitude)
+{
+    int64_t smallest_untrackable_value = ((int64_t) sub_bucket_count) << unit_magnitude;
+    int32_t buckets_needed = 1;
+    while (smallest_untrackable_value <= value)
+    {
+        if (smallest_untrackable_value > INT64_MAX / 2)
+        {
+            return buckets_needed + 1;
+        }
+        smallest_untrackable_value <<= 1;
+        buckets_needed++;
+    }
+
+    return buckets_needed;
+}
+
+/* ##     ## ######## ##     ##  #######  ########  ##    ## */
+/* ###   ### ##       ###   ### ##     ## ##     ##  ##  ##  */
+/* #### #### ##       #### #### ##     ## ##     ##   ####   */
+/* ## ### ## ######   ## ### ## ##     ## ########     ##    */
+/* ##     ## ##       ##     ## ##     ## ##   ##      ##    */
+/* ##     ## ##       ##     ## ##     ## ##    ##     ##    */
+/* ##     ## ######## ##     ##  #######  ##     ##    ##    */
+
+int hdr_calculate_bucket_config(
+        int64_t lowest_trackable_value,
+        int64_t highest_trackable_value,
+        int significant_figures,
+        struct hdr_histogram_bucket_config* cfg)
+{
+    int32_t sub_bucket_count_magnitude;
+    int64_t largest_value_with_single_unit_resolution;
+
+    if (lowest_trackable_value < 1 ||
+            significant_figures < 1 || 5 < significant_figures ||
+            lowest_trackable_value * 2 > highest_trackable_value)
+    {
+        return EINVAL;
+    }
+
+    cfg->lowest_trackable_value = lowest_trackable_value;
+    cfg->significant_figures = significant_figures;
+    cfg->highest_trackable_value = highest_trackable_value;
+
+    largest_value_with_single_unit_resolution = 2 * power(10, significant_figures);
+    sub_bucket_count_magnitude = (int32_t) ceil(log((double)largest_value_with_single_unit_resolution) / log(2));
+    cfg->sub_bucket_half_count_magnitude = ((sub_bucket_count_magnitude > 1) ? sub_bucket_count_magnitude : 1) - 1;
+
+    cfg->unit_magnitude = (int32_t) floor(log((double)lowest_trackable_value) / log(2));
+
+    cfg->sub_bucket_count      = (int32_t) pow(2, (cfg->sub_bucket_half_count_magnitude + 1));
+    cfg->sub_bucket_half_count = cfg->sub_bucket_count / 2;
+    cfg->sub_bucket_mask       = ((int64_t) cfg->sub_bucket_count - 1) << cfg->unit_magnitude;
+
+    if (cfg->unit_magnitude + cfg->sub_bucket_half_count_magnitude > 61)
+    {
+        return EINVAL;
+    }
+
+    cfg->bucket_count = buckets_needed_to_cover_value(highest_trackable_value, cfg->sub_bucket_count, (int32_t)cfg->unit_magnitude);
+    cfg->counts_len = (cfg->bucket_count + 1) * (cfg->sub_bucket_count / 2);
+
+    return 0;
+}
+
+void hdr_init_preallocated(struct hdr_histogram* h, struct hdr_histogram_bucket_config* cfg)
+{
+    h->lowest_trackable_value          = cfg->lowest_trackable_value;
+    h->highest_trackable_value         = cfg->highest_trackable_value;
+    h->unit_magnitude                  = (int32_t)cfg->unit_magnitude;
+    h->significant_figures             = (int32_t)cfg->significant_figures;
+    h->sub_bucket_half_count_magnitude = cfg->sub_bucket_half_count_magnitude;
+    h->sub_bucket_half_count           = cfg->sub_bucket_half_count;
+    h->sub_bucket_mask                 = cfg->sub_bucket_mask;
+    h->sub_bucket_count                = cfg->sub_bucket_count;
+    h->min_value                       = INT64_MAX;
+    h->max_value                       = 0;
+    h->normalizing_index_offset        = 0;
+    h->conversion_ratio                = 1.0;
+    h->bucket_count                    = cfg->bucket_count;
+    h->counts_len                      = cfg->counts_len;
+    h->total_count                     = 0;
+}
+
+int hdr_init(
+        int64_t lowest_trackable_value,
+        int64_t highest_trackable_value,
+        int significant_figures,
+        struct hdr_histogram** result)
+{
+    int64_t* counts;
+    struct hdr_histogram_bucket_config cfg;
+    struct hdr_histogram* histogram;
+
+    int r = hdr_calculate_bucket_config(lowest_trackable_value, highest_trackable_value, significant_figures, &cfg);
+    if (r)
+    {
+        return r;
+    }
+
+    counts = (int64_t*) calloc((size_t) cfg.counts_len, sizeof(int64_t));
+    if (!counts)
+    {
+        return ENOMEM;
+    }
+
+    histogram = (struct hdr_histogram*) calloc(1, sizeof(struct hdr_histogram));
+    if (!histogram)
+    {
+        free(counts);
+        return ENOMEM;
+    }
+
+    histogram->counts = counts;
+
+    hdr_init_preallocated(histogram, &cfg);
+    *result = histogram;
+
+    return 0;
+}
+
+void hdr_close(struct hdr_histogram* h)
+{
+    if (h) {
+	free(h->counts);
+	free(h);
+    }
+}
+
+int hdr_alloc(int64_t highest_trackable_value, int significant_figures, struct hdr_histogram** result)
+{
+    return hdr_init(1, highest_trackable_value, significant_figures, result);
+}
+
+/* reset a histogram to zero. */
+void hdr_reset(struct hdr_histogram *h)
+{
+     h->total_count=0;
+     h->min_value = INT64_MAX;
+     h->max_value = 0;
+     memset(h->counts, 0, (sizeof(int64_t) * h->counts_len));
+}
+
+size_t hdr_get_memory_size(struct hdr_histogram *h)
+{
+    return sizeof(struct hdr_histogram) + h->counts_len * sizeof(int64_t);
+}
+
+/* ##     ## ########  ########     ###    ######## ########  ######  */
+/* ##     ## ##     ## ##     ##   ## ##      ##    ##       ##    ## */
+/* ##     ## ##     ## ##     ##  ##   ##     ##    ##       ##       */
+/* ##     ## ########  ##     ## ##     ##    ##    ######    ######  */
+/* ##     ## ##        ##     ## #########    ##    ##             ## */
+/* ##     ## ##        ##     ## ##     ##    ##    ##       ##    ## */
+/*  #######  ##        ########  ##     ##    ##    ########  ######  */
+
+
+bool hdr_record_value(struct hdr_histogram* h, int64_t value)
+{
+    return hdr_record_values(h, value, 1);
+}
+
+bool hdr_record_value_atomic(struct hdr_histogram* h, int64_t value)
+{
+    return hdr_record_values_atomic(h, value, 1);
+}
+
+bool hdr_record_values(struct hdr_histogram* h, int64_t value, int64_t count)
+{
+    int32_t counts_index;
+
+    if (value < 0)
+    {
+        return false;
+    }
+
+    counts_index = counts_index_for(h, value);
+
+    if (counts_index < 0 || h->counts_len <= counts_index)
+    {
+        return false;
+    }
+
+    counts_inc_normalised(h, counts_index, count);
+    update_min_max(h, value);
+
+    return true;
+}
+
+bool hdr_record_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count)
+{
+    int32_t counts_index;
+
+    if (value < 0)
+    {
+        return false;
+    }
+
+    counts_index = counts_index_for(h, value);
+
+    if (counts_index < 0 || h->counts_len <= counts_index)
+    {
+        return false;
+    }
+
+    counts_inc_normalised_atomic(h, counts_index, count);
+    update_min_max_atomic(h, value);
+
+    return true;
+}
+
+bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t expected_interval)
+{
+    return hdr_record_corrected_values(h, value, 1, expected_interval);
+}
+
+bool hdr_record_corrected_value_atomic(struct hdr_histogram* h, int64_t value, int64_t expected_interval)
+{
+    return hdr_record_corrected_values_atomic(h, value, 1, expected_interval);
+}
+
+bool hdr_record_corrected_values(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval)
+{
+    int64_t missing_value;
+
+    if (!hdr_record_values(h, value, count))
+    {
+        return false;
+    }
+
+    if (expected_interval <= 0 || value <= expected_interval)
+    {
+        return true;
+    }
+
+    missing_value = value - expected_interval;
+    for (; missing_value >= expected_interval; missing_value -= expected_interval)
+    {
+        if (!hdr_record_values(h, missing_value, count))
+        {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+bool hdr_record_corrected_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval)
+{
+    int64_t missing_value;
+
+    if (!hdr_record_values_atomic(h, value, count))
+    {
+        return false;
+    }
+
+    if (expected_interval <= 0 || value <= expected_interval)
+    {
+        return true;
+    }
+
+    missing_value = value - expected_interval;
+    for (; missing_value >= expected_interval; missing_value -= expected_interval)
+    {
+        if (!hdr_record_values_atomic(h, missing_value, count))
+        {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+int64_t hdr_add(struct hdr_histogram* h, const struct hdr_histogram* from)
+{
+    struct hdr_iter iter;
+    int64_t dropped = 0;
+    hdr_iter_recorded_init(&iter, from);
+
+    while (hdr_iter_next(&iter))
+    {
+        int64_t value = iter.value;
+        int64_t count = iter.count;
+
+        if (!hdr_record_values(h, value, count))
+        {
+            dropped += count;
+        }
+    }
+
+    return dropped;
+}
+
+int64_t hdr_add_while_correcting_for_coordinated_omission(
+        struct hdr_histogram* h, struct hdr_histogram* from, int64_t expected_interval)
+{
+    struct hdr_iter iter;
+    int64_t dropped = 0;
+    hdr_iter_recorded_init(&iter, from);
+
+    while (hdr_iter_next(&iter))
+    {
+        int64_t value = iter.value;
+        int64_t count = iter.count;
+
+        if (!hdr_record_corrected_values(h, value, count, expected_interval))
+        {
+            dropped += count;
+        }
+    }
+
+    return dropped;
+}
+
+
+
+/* ##     ##    ###    ##       ##     ## ########  ######  */
+/* ##     ##   ## ##   ##       ##     ## ##       ##    ## */
+/* ##     ##  ##   ##  ##       ##     ## ##       ##       */
+/* ##     ## ##     ## ##       ##     ## ######    ######  */
+/*  ##   ##  ######### ##       ##     ## ##             ## */
+/*   ## ##   ##     ## ##       ##     ## ##       ##    ## */
+/*    ###    ##     ## ########  #######  ########  ######  */
+
+
+int64_t hdr_max(const struct hdr_histogram* h)
+{
+    if (0 == h->max_value)
+    {
+        return 0;
+    }
+
+    return highest_equivalent_value(h, h->max_value);
+}
+
+int64_t hdr_min(const struct hdr_histogram* h)
+{
+    if (0 < hdr_count_at_index(h, 0))
+    {
+        return 0;
+    }
+
+    return non_zero_min(h);
+}
+
+int64_t hdr_value_at_percentile(const struct hdr_histogram* h, double percentile)
+{
+    struct hdr_iter iter;
+    int64_t total = 0;
+    double requested_percentile = percentile < 100.0 ? percentile : 100.0;
+    int64_t count_at_percentile =
+        (int64_t) (((requested_percentile / 100) * h->total_count) + 0.5);
+    count_at_percentile = count_at_percentile > 1 ? count_at_percentile : 1;
+
+    hdr_iter_init(&iter, h);
+
+    while (hdr_iter_next(&iter))
+    {
+        total += iter.count;
+
+        if (total >= count_at_percentile)
+        {
+            int64_t value_from_index = iter.value;
+            return highest_equivalent_value(h, value_from_index);
+        }
+    }
+
+    return 0;
+}
+
+double hdr_mean(const struct hdr_histogram* h)
+{
+    struct hdr_iter iter;
+    int64_t total = 0;
+
+    hdr_iter_init(&iter, h);
+
+    while (hdr_iter_next(&iter))
+    {
+        if (0 != iter.count)
+        {
+            total += iter.count * hdr_median_equivalent_value(h, iter.value);
+        }
+    }
+
+    return (total * 1.0) / h->total_count;
+}
+
+double hdr_stddev(const struct hdr_histogram* h)
+{
+    double mean = hdr_mean(h);
+    double geometric_dev_total = 0.0;
+
+    struct hdr_iter iter;
+    hdr_iter_init(&iter, h);
+
+    while (hdr_iter_next(&iter))
+    {
+        if (0 != iter.count)
+        {
+            double dev = (hdr_median_equivalent_value(h, iter.value) * 1.0) - mean;
+            geometric_dev_total += (dev * dev) * iter.count;
+        }
+    }
+
+    return sqrt(geometric_dev_total / h->total_count);
+}
+
+bool hdr_values_are_equivalent(const struct hdr_histogram* h, int64_t a, int64_t b)
+{
+    return lowest_equivalent_value(h, a) == lowest_equivalent_value(h, b);
+}
+
+int64_t hdr_lowest_equivalent_value(const struct hdr_histogram* h, int64_t value)
+{
+    return lowest_equivalent_value(h, value);
+}
+
+int64_t hdr_count_at_value(const struct hdr_histogram* h, int64_t value)
+{
+    return counts_get_normalised(h, counts_index_for(h, value));
+}
+
+int64_t hdr_count_at_index(const struct hdr_histogram* h, int32_t index)
+{
+    return counts_get_normalised(h, index);
+}
+
+
+/* #### ######## ######## ########     ###    ########  #######  ########   ######  */
+/*  ##     ##    ##       ##     ##   ## ##      ##    ##     ## ##     ## ##    ## */
+/*  ##     ##    ##       ##     ##  ##   ##     ##    ##     ## ##     ## ##       */
+/*  ##     ##    ######   ########  ##     ##    ##    ##     ## ########   ######  */
+/*  ##     ##    ##       ##   ##   #########    ##    ##     ## ##   ##         ## */
+/*  ##     ##    ##       ##    ##  ##     ##    ##    ##     ## ##    ##  ##    ## */
+/* ####    ##    ######## ##     ## ##     ##    ##     #######  ##     ##  ######  */
+
+
+static bool has_buckets(struct hdr_iter* iter)
+{
+    return iter->counts_index < iter->h->counts_len;
+}
+
+static bool has_next(struct hdr_iter* iter)
+{
+    return iter->cumulative_count < iter->total_count;
+}
+
+static bool move_next(struct hdr_iter* iter)
+{
+    iter->counts_index++;
+
+    if (!has_buckets(iter))
+    {
+        return false;
+    }
+
+    iter->count = counts_get_normalised(iter->h, iter->counts_index);
+    iter->cumulative_count += iter->count;
+
+    iter->value = hdr_value_at_index(iter->h, iter->counts_index);
+    iter->highest_equivalent_value = highest_equivalent_value(iter->h, iter->value);
+    iter->lowest_equivalent_value = lowest_equivalent_value(iter->h, iter->value);
+    iter->median_equivalent_value = hdr_median_equivalent_value(iter->h, iter->value);
+
+    return true;
+}
+
+static int64_t peek_next_value_from_index(struct hdr_iter* iter)
+{
+    return hdr_value_at_index(iter->h, iter->counts_index + 1);
+}
+
+static bool next_value_greater_than_reporting_level_upper_bound(
+    struct hdr_iter *iter, int64_t reporting_level_upper_bound)
+{
+    if (iter->counts_index >= iter->h->counts_len)
+    {
+        return false;
+    }
+
+    return peek_next_value_from_index(iter) > reporting_level_upper_bound;
+}
+
+static bool basic_iter_next(struct hdr_iter *iter)
+{
+    if (!has_next(iter) || iter->counts_index >= iter->h->counts_len)
+    {
+        return false;
+    }
+
+    move_next(iter);
+
+    return true;
+}
+
+static void update_iterated_values(struct hdr_iter* iter, int64_t new_value_iterated_to)
+{
+    iter->value_iterated_from = iter->value_iterated_to;
+    iter->value_iterated_to = new_value_iterated_to;
+}
+
+static bool all_values_iter_next(struct hdr_iter* iter)
+{
+    bool result = move_next(iter);
+
+    if (result)
+    {
+        update_iterated_values(iter, iter->value);
+    }
+
+    return result;
+}
+
+void hdr_iter_init(struct hdr_iter* iter, const struct hdr_histogram* h)
+{
+    iter->h = h;
+
+    iter->counts_index = -1;
+    iter->total_count = h->total_count;
+    iter->count = 0;
+    iter->cumulative_count = 0;
+    iter->value = 0;
+    iter->highest_equivalent_value = 0;
+    iter->value_iterated_from = 0;
+    iter->value_iterated_to = 0;
+
+    iter->_next_fp = all_values_iter_next;
+}
+
+bool hdr_iter_next(struct hdr_iter* iter)
+{
+    return iter->_next_fp(iter);
+}
+
+/* ########  ######## ########   ######  ######## ##    ## ######## #### ##       ########  ######  */
+/* ##     ## ##       ##     ## ##    ## ##       ###   ##    ##     ##  ##       ##       ##    ## */
+/* ##     ## ##       ##     ## ##       ##       ####  ##    ##     ##  ##       ##       ##       */
+/* ########  ######   ########  ##       ######   ## ## ##    ##     ##  ##       ######    ######  */
+/* ##        ##       ##   ##   ##       ##       ##  ####    ##     ##  ##       ##             ## */
+/* ##        ##       ##    ##  ##    ## ##       ##   ###    ##     ##  ##       ##       ##    ## */
+/* ##        ######## ##     ##  ######  ######## ##    ##    ##    #### ######## ########  ######  */
+
+static bool percentile_iter_next(struct hdr_iter* iter)
+{
+    int64_t temp, half_distance, percentile_reporting_ticks;
+
+    struct hdr_iter_percentiles* percentiles = &iter->specifics.percentiles;
+
+    if (!has_next(iter))
+    {
+        if (percentiles->seen_last_value)
+        {
+            return false;
+        }
+
+        percentiles->seen_last_value = true;
+        percentiles->percentile = 100.0;
+
+        return true;
+    }
+
+    if (iter->counts_index == -1 && !basic_iter_next(iter))
+    {
+        return false;
+    }
+
+    do
+    {
+        double current_percentile = (100.0 * (double) iter->cumulative_count) / iter->h->total_count;
+        if (iter->count != 0 &&
+                percentiles->percentile_to_iterate_to <= current_percentile)
+        {
+            update_iterated_values(iter, highest_equivalent_value(iter->h, iter->value));
+
+            percentiles->percentile = percentiles->percentile_to_iterate_to;
+            temp = (int64_t)(log(100 / (100.0 - (percentiles->percentile_to_iterate_to))) / log(2)) + 1;
+            half_distance = (int64_t) pow(2, (double) temp);
+            percentile_reporting_ticks = percentiles->ticks_per_half_distance * half_distance;
+            percentiles->percentile_to_iterate_to += 100.0 / percentile_reporting_ticks;
+
+            return true;
+        }
+    }
+    while (basic_iter_next(iter));
+
+    return true;
+}
+
+void hdr_iter_percentile_init(struct hdr_iter* iter, const struct hdr_histogram* h, int32_t ticks_per_half_distance)
+{
+    iter->h = h;
+
+    hdr_iter_init(iter, h);
+
+    iter->specifics.percentiles.seen_last_value          = false;
+    iter->specifics.percentiles.ticks_per_half_distance  = ticks_per_half_distance;
+    iter->specifics.percentiles.percentile_to_iterate_to = 0.0;
+    iter->specifics.percentiles.percentile               = 0.0;
+
+    iter->_next_fp = percentile_iter_next;
+}
+
+static void format_line_string(char* str, size_t len, int significant_figures, format_type format)
+{
+#if defined(_MSC_VER)
+#define snprintf _snprintf
+#pragma warning(push)
+#pragma warning(disable: 4996)
+#endif
+    const char* format_str = "%s%d%s";
+
+    switch (format)
+    {
+        case CSV:
+            snprintf(str, len, format_str, "%.", significant_figures, "f,%f,%d,%.2f\n");
+            break;
+        case CLASSIC:
+            snprintf(str, len, format_str, "%12.", significant_figures, "f %12f %12d %12.2f\n");
+            break;
+        default:
+            snprintf(str, len, format_str, "%12.", significant_figures, "f %12f %12d %12.2f\n");
+    }
+#if defined(_MSC_VER)
+#undef snprintf
+#pragma warning(pop)
+#endif
+}
+
+
+/* ########  ########  ######   #######  ########  ########  ######## ########   */
+/* ##     ## ##       ##    ## ##     ## ##     ## ##     ## ##       ##     ##  */
+/* ##     ## ##       ##       ##     ## ##     ## ##     ## ##       ##     ##  */
+/* ########  ######   ##       ##     ## ########  ##     ## ######   ##     ##  */
+/* ##   ##   ##       ##       ##     ## ##   ##   ##     ## ##       ##     ##  */
+/* ##    ##  ##       ##    ## ##     ## ##    ##  ##     ## ##       ##     ##  */
+/* ##     ## ########  ######   #######  ##     ## ########  ######## ########   */
+
+
+static bool recorded_iter_next(struct hdr_iter* iter)
+{
+    while (basic_iter_next(iter))
+    {
+        if (iter->count != 0)
+        {
+            update_iterated_values(iter, iter->value);
+
+            iter->specifics.recorded.count_added_in_this_iteration_step = iter->count;
+            return true;
+        }
+    }
+
+    return false;
+}
+
+void hdr_iter_recorded_init(struct hdr_iter* iter, const struct hdr_histogram* h)
+{
+    hdr_iter_init(iter, h);
+
+    iter->specifics.recorded.count_added_in_this_iteration_step = 0;
+
+    iter->_next_fp = recorded_iter_next;
+}
+
+/* ##       #### ##    ## ########    ###    ########  */
+/* ##        ##  ###   ## ##         ## ##   ##     ## */
+/* ##        ##  ####  ## ##        ##   ##  ##     ## */
+/* ##        ##  ## ## ## ######   ##     ## ########  */
+/* ##        ##  ##  #### ##       ######### ##   ##   */
+/* ##        ##  ##   ### ##       ##     ## ##    ##  */
+/* ######## #### ##    ## ######## ##     ## ##     ## */
+
+
+static bool iter_linear_next(struct hdr_iter* iter)
+{
+    struct hdr_iter_linear* linear = &iter->specifics.linear;
+
+    linear->count_added_in_this_iteration_step = 0;
+
+    if (has_next(iter) ||
+        next_value_greater_than_reporting_level_upper_bound(
+            iter, linear->next_value_reporting_level_lowest_equivalent))
+    {
+        do
+        {
+            if (iter->value >= linear->next_value_reporting_level_lowest_equivalent)
+            {
+                update_iterated_values(iter, linear->next_value_reporting_level);
+
+                linear->next_value_reporting_level += linear->value_units_per_bucket;
+                linear->next_value_reporting_level_lowest_equivalent =
+                    lowest_equivalent_value(iter->h, linear->next_value_reporting_level);
+
+                return true;
+            }
+
+            if (!move_next(iter))
+            {
+                return true;
+            }
+
+            linear->count_added_in_this_iteration_step += iter->count;
+        }
+        while (true);
+    }
+
+    return false;
+}
+
+
+void hdr_iter_linear_init(struct hdr_iter* iter, const struct hdr_histogram* h, int64_t value_units_per_bucket)
+{
+    hdr_iter_init(iter, h);
+
+    iter->specifics.linear.count_added_in_this_iteration_step = 0;
+    iter->specifics.linear.value_units_per_bucket = value_units_per_bucket;
+    iter->specifics.linear.next_value_reporting_level = value_units_per_bucket;
+    iter->specifics.linear.next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(h, value_units_per_bucket);
+
+    iter->_next_fp = iter_linear_next;
+}
+
+void hdr_iter_linear_set_value_units_per_bucket(struct hdr_iter* iter, int64_t value_units_per_bucket)
+{
+    iter->specifics.linear.value_units_per_bucket = value_units_per_bucket;
+}
+
+/* ##        #######   ######      ###    ########  #### ######## ##     ## ##     ## ####  ######  */
+/* ##       ##     ## ##    ##    ## ##   ##     ##  ##     ##    ##     ## ###   ###  ##  ##    ## */
+/* ##       ##     ## ##         ##   ##  ##     ##  ##     ##    ##     ## #### ####  ##  ##       */
+/* ##       ##     ## ##   #### ##     ## ########   ##     ##    ######### ## ### ##  ##  ##       */
+/* ##       ##     ## ##    ##  ######### ##   ##    ##     ##    ##     ## ##     ##  ##  ##       */
+/* ##       ##     ## ##    ##  ##     ## ##    ##   ##     ##    ##     ## ##     ##  ##  ##    ## */
+/* ########  #######   ######   ##     ## ##     ## ####    ##    ##     ## ##     ## ####  ######  */
+
+static bool log_iter_next(struct hdr_iter *iter)
+{
+    struct hdr_iter_log* logarithmic = &iter->specifics.log;
+
+    logarithmic->count_added_in_this_iteration_step = 0;
+
+    if (has_next(iter) ||
+        next_value_greater_than_reporting_level_upper_bound(
+            iter, logarithmic->next_value_reporting_level_lowest_equivalent))
+    {
+        do
+        {
+            if (iter->value >= logarithmic->next_value_reporting_level_lowest_equivalent)
+            {
+                update_iterated_values(iter, logarithmic->next_value_reporting_level);
+
+                logarithmic->next_value_reporting_level *= (int64_t)logarithmic->log_base;
+                logarithmic->next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(iter->h, logarithmic->next_value_reporting_level);
+
+                return true;
+            }
+
+            if (!move_next(iter))
+            {
+                return true;
+            }
+
+            logarithmic->count_added_in_this_iteration_step += iter->count;
+        }
+        while (true);
+    }
+
+    return false;
+}
+
+void hdr_iter_log_init(
+        struct hdr_iter* iter,
+        const struct hdr_histogram* h,
+        int64_t value_units_first_bucket,
+        double log_base)
+{
+    hdr_iter_init(iter, h);
+    iter->specifics.log.count_added_in_this_iteration_step = 0;
+    iter->specifics.log.log_base = log_base;
+    iter->specifics.log.next_value_reporting_level = value_units_first_bucket;
+    iter->specifics.log.next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(h, value_units_first_bucket);
+
+    iter->_next_fp = log_iter_next;
+}
+
+/* Printing. */
+
+static const char* format_head_string(format_type format)
+{
+    switch (format)
+    {
+        case CSV:
+            return "%s,%s,%s,%s\n";
+        case CLASSIC:
+        default:
+            return "%12s %12s %12s %12s\n\n";
+    }
+}
+
+static const char CLASSIC_FOOTER[] =
+    "#[Mean    = %12.3f, StdDeviation   = %12.3f]\n"
+    "#[Max     = %12.3f, Total count    = %12" PRIu64 "]\n"
+    "#[Buckets = %12d, SubBuckets     = %12d]\n";
+
+int hdr_percentiles_print(
+        struct hdr_histogram* h, FILE* stream, int32_t ticks_per_half_distance,
+        double value_scale, format_type format)
+{
+    char line_format[25];
+    const char* head_format;
+    int rc = 0;
+    struct hdr_iter iter;
+    struct hdr_iter_percentiles * percentiles;
+
+    format_line_string(line_format, 25, h->significant_figures, format);
+    head_format = format_head_string(format);
+
+    hdr_iter_percentile_init(&iter, h, ticks_per_half_distance);
+
+    if (fprintf(
+            stream, head_format,
+            "Value", "Percentile", "TotalCount", "1/(1-Percentile)") < 0)
+    {
+        rc = EIO;
+        goto cleanup;
+    }
+
+    percentiles = &iter.specifics.percentiles;
+    while (hdr_iter_next(&iter))
+    {
+        double  value               = iter.highest_equivalent_value / value_scale;
+        double  percentile          = percentiles->percentile / 100.0;
+        int64_t total_count         = iter.cumulative_count;
+        double  inverted_percentile = (1.0 / (1.0 - percentile));
+
+        if (fprintf(
+                stream, line_format, value, percentile, total_count, inverted_percentile) < 0)
+        {
+            rc = EIO;
+            goto cleanup;
+        }
+    }
+
+    if (CLASSIC == format)
+    {
+        double mean   = hdr_mean(h)   / value_scale;
+        double stddev = hdr_stddev(h) / value_scale;
+        double max    = hdr_max(h)    / value_scale;
+
+        if (fprintf(
+                stream, CLASSIC_FOOTER,  mean, stddev, max,
+                h->total_count, h->bucket_count, h->sub_bucket_count) < 0)
+        {
+            rc = EIO;
+            goto cleanup;
+        }
+    }
+
+    cleanup:
+    return rc;
+}
diff --git a/deps/hdr_histogram/hdr_histogram.h b/deps/hdr_histogram/hdr_histogram.h
new file mode 100644
index 000000000..c26c56b20
--- /dev/null
+++ b/deps/hdr_histogram/hdr_histogram.h
@@ -0,0 +1,509 @@
+/**
+ * hdr_histogram.h
+ * Written by Michael Barker and released to the public domain,
+ * as explained at http://creativecommons.org/publicdomain/zero/1.0/
+ *
+ * The source for the hdr_histogram utilises a few C99 constructs, specifically
+ * the use of stdint/stdbool and inline variable declaration.
+ */
+
+#ifndef HDR_HISTOGRAM_H
+#define HDR_HISTOGRAM_H 1
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+
+struct hdr_histogram
+{
+    int64_t lowest_trackable_value;
+    int64_t highest_trackable_value;
+    int32_t unit_magnitude;
+    int32_t significant_figures;
+    int32_t sub_bucket_half_count_magnitude;
+    int32_t sub_bucket_half_count;
+    int64_t sub_bucket_mask;
+    int32_t sub_bucket_count;
+    int32_t bucket_count;
+    int64_t min_value;
+    int64_t max_value;
+    int32_t normalizing_index_offset;
+    double conversion_ratio;
+    int32_t counts_len;
+    int64_t total_count;
+    int64_t* counts;
+};
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Allocate the memory and initialise the hdr_histogram.
+ *
+ * Due to the size of the histogram being the result of some reasonably
+ * involved math on the input parameters this function it is tricky to stack allocate.
+ * The histogram should be released with hdr_close
+ *
+ * @param lowest_trackable_value The smallest possible value to be put into the
+ * histogram.
+ * @param highest_trackable_value The largest possible value to be put into the
+ * histogram.
+ * @param significant_figures The level of precision for this histogram, i.e. the number
+ * of figures in a decimal number that will be maintained.  E.g. a value of 3 will mean
+ * the results from the histogram will be accurate up to the first three digits.  Must
+ * be a value between 1 and 5 (inclusive).
+ * @param result Output parameter to capture allocated histogram.
+ * @return 0 on success, EINVAL if lowest_trackable_value is < 1 or the
+ * significant_figure value is outside of the allowed range, ENOMEM if malloc
+ * failed.
+ */
+int hdr_init(
+    int64_t lowest_trackable_value,
+    int64_t highest_trackable_value,
+    int significant_figures,
+    struct hdr_histogram** result);
+
+/**
+ * Free the memory and close the hdr_histogram.
+ *
+ * @param h The histogram you want to close.
+ */
+void hdr_close(struct hdr_histogram* h);
+
+/**
+ * Allocate the memory and initialise the hdr_histogram.  This is the equivalent of calling
+ * hdr_init(1, highest_trackable_value, significant_figures, result);
+ *
+ * @deprecated use hdr_init.
+ */
+int hdr_alloc(int64_t highest_trackable_value, int significant_figures, struct hdr_histogram** result);
+
+
+/**
+ * Reset a histogram to zero - empty out a histogram and re-initialise it
+ *
+ * If you want to re-use an existing histogram, but reset everything back to zero, this
+ * is the routine to use.
+ *
+ * @param h The histogram you want to reset to empty.
+ *
+ */
+void hdr_reset(struct hdr_histogram* h);
+
+/**
+ * Get the memory size of the hdr_histogram.
+ *
+ * @param h "This" pointer
+ * @return The amount of memory used by the hdr_histogram in bytes
+ */
+size_t hdr_get_memory_size(struct hdr_histogram* h);
+
+/**
+ * Records a value in the histogram, will round this value of to a precision at or better
+ * than the significant_figure specified at construction time.
+ *
+ * @param h "This" pointer
+ * @param value Value to add to the histogram
+ * @return false if the value is larger than the highest_trackable_value and can't be recorded,
+ * true otherwise.
+ */
+bool hdr_record_value(struct hdr_histogram* h, int64_t value);
+
+/**
+ * Records a value in the histogram, will round this value of to a precision at or better
+ * than the significant_figure specified at construction time.
+ *
+ * Will record this value atomically, however the whole structure may appear inconsistent
+ * when read concurrently with this update.  Do NOT mix calls to this method with calls
+ * to non-atomic updates.
+ *
+ * @param h "This" pointer
+ * @param value Value to add to the histogram
+ * @return false if the value is larger than the highest_trackable_value and can't be recorded,
+ * true otherwise.
+ */
+bool hdr_record_value_atomic(struct hdr_histogram* h, int64_t value);
+
+/**
+ * Records count values in the histogram, will round this value of to a
+ * precision at or better than the significant_figure specified at construction
+ * time.
+ *
+ * @param h "This" pointer
+ * @param value Value to add to the histogram
+ * @param count Number of 'value's to add to the histogram
+ * @return false if any value is larger than the highest_trackable_value and can't be recorded,
+ * true otherwise.
+ */
+bool hdr_record_values(struct hdr_histogram* h, int64_t value, int64_t count);
+
+/**
+ * Records count values in the histogram, will round this value of to a
+ * precision at or better than the significant_figure specified at construction
+ * time.
+ *
+ * Will record this value atomically, however the whole structure may appear inconsistent
+ * when read concurrently with this update.  Do NOT mix calls to this method with calls
+ * to non-atomic updates.
+ *
+ * @param h "This" pointer
+ * @param value Value to add to the histogram
+ * @param count Number of 'value's to add to the histogram
+ * @return false if any value is larger than the highest_trackable_value and can't be recorded,
+ * true otherwise.
+ */
+bool hdr_record_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count);
+
+/**
+ * Record a value in the histogram and backfill based on an expected interval.
+ *
+ * Records a value in the histogram, will round this value of to a precision at or better
+ * than the significant_figure specified at contruction time.  This is specifically used
+ * for recording latency.  If the value is larger than the expected_interval then the
+ * latency recording system has experienced co-ordinated omission.  This method fills in the
+ * values that would have occured had the client providing the load not been blocked.
+
+ * @param h "This" pointer
+ * @param value Value to add to the histogram
+ * @param expected_interval The delay between recording values.
+ * @return false if the value is larger than the highest_trackable_value and can't be recorded,
+ * true otherwise.
+ */
+bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t expexcted_interval);
+
+/**
+ * Record a value in the histogram and backfill based on an expected interval.
+ *
+ * Records a value in the histogram, will round this value of to a precision at or better
+ * than the significant_figure specified at contruction time.  This is specifically used
+ * for recording latency.  If the value is larger than the expected_interval then the
+ * latency recording system has experienced co-ordinated omission.  This method fills in the
+ * values that would have occured had the client providing the load not been blocked.
+ *
+ * Will record this value atomically, however the whole structure may appear inconsistent
+ * when read concurrently with this update.  Do NOT mix calls to this method with calls
+ * to non-atomic updates.
+ *
+ * @param h "This" pointer
+ * @param value Value to add to the histogram
+ * @param expected_interval The delay between recording values.
+ * @return false if the value is larger than the highest_trackable_value and can't be recorded,
+ * true otherwise.
+ */
+bool hdr_record_corrected_value_atomic(struct hdr_histogram* h, int64_t value, int64_t expexcted_interval);
+
+/**
+ * Record a value in the histogram 'count' times.  Applies the same correcting logic
+ * as 'hdr_record_corrected_value'.
+ *
+ * @param h "This" pointer
+ * @param value Value to add to the histogram
+ * @param count Number of 'value's to add to the histogram
+ * @param expected_interval The delay between recording values.
+ * @return false if the value is larger than the highest_trackable_value and can't be recorded,
+ * true otherwise.
+ */
+bool hdr_record_corrected_values(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval);
+
+/**
+ * Record a value in the histogram 'count' times.  Applies the same correcting logic
+ * as 'hdr_record_corrected_value'.
+ *
+ * Will record this value atomically, however the whole structure may appear inconsistent
+ * when read concurrently with this update.  Do NOT mix calls to this method with calls
+ * to non-atomic updates.
+ *
+ * @param h "This" pointer
+ * @param value Value to add to the histogram
+ * @param count Number of 'value's to add to the histogram
+ * @param expected_interval The delay between recording values.
+ * @return false if the value is larger than the highest_trackable_value and can't be recorded,
+ * true otherwise.
+ */
+bool hdr_record_corrected_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval);
+
+/**
+ * Adds all of the values from 'from' to 'this' histogram.  Will return the
+ * number of values that are dropped when copying.  Values will be dropped
+ * if they around outside of h.lowest_trackable_value and
+ * h.highest_trackable_value.
+ *
+ * @param h "This" pointer
+ * @param from Histogram to copy values from.
+ * @return The number of values dropped when copying.
+ */
+int64_t hdr_add(struct hdr_histogram* h, const struct hdr_histogram* from);
+
+/**
+ * Adds all of the values from 'from' to 'this' histogram.  Will return the
+ * number of values that are dropped when copying.  Values will be dropped
+ * if they around outside of h.lowest_trackable_value and
+ * h.highest_trackable_value.
+ *
+ * @param h "This" pointer
+ * @param from Histogram to copy values from.
+ * @return The number of values dropped when copying.
+ */
+int64_t hdr_add_while_correcting_for_coordinated_omission(
+    struct hdr_histogram* h, struct hdr_histogram* from, int64_t expected_interval);
+
+/**
+ * Get minimum value from the histogram.  Will return 2^63-1 if the histogram
+ * is empty.
+ *
+ * @param h "This" pointer
+ */
+int64_t hdr_min(const struct hdr_histogram* h);
+
+/**
+ * Get maximum value from the histogram.  Will return 0 if the histogram
+ * is empty.
+ *
+ * @param h "This" pointer
+ */
+int64_t hdr_max(const struct hdr_histogram* h);
+
+/**
+ * Get the value at a specific percentile.
+ *
+ * @param h "This" pointer.
+ * @param percentile The percentile to get the value for
+ */
+int64_t hdr_value_at_percentile(const struct hdr_histogram* h, double percentile);
+
+/**
+ * Gets the standard deviation for the values in the histogram.
+ *
+ * @param h "This" pointer
+ * @return The standard deviation
+ */
+double hdr_stddev(const struct hdr_histogram* h);
+
+/**
+ * Gets the mean for the values in the histogram.
+ *
+ * @param h "This" pointer
+ * @return The mean
+ */
+double hdr_mean(const struct hdr_histogram* h);
+
+/**
+ * Determine if two values are equivalent with the histogram's resolution.
+ * Where "equivalent" means that value samples recorded for any two
+ * equivalent values are counted in a common total count.
+ *
+ * @param h "This" pointer
+ * @param a first value to compare
+ * @param b second value to compare
+ * @return 'true' if values are equivalent with the histogram's resolution.
+ */
+bool hdr_values_are_equivalent(const struct hdr_histogram* h, int64_t a, int64_t b);
+
+/**
+ * Get the lowest value that is equivalent to the given value within the histogram's resolution.
+ * Where "equivalent" means that value samples recorded for any two
+ * equivalent values are counted in a common total count.
+ *
+ * @param h "This" pointer
+ * @param value The given value
+ * @return The lowest value that is equivalent to the given value within the histogram's resolution.
+ */
+int64_t hdr_lowest_equivalent_value(const struct hdr_histogram* h, int64_t value);
+
+/**
+ * Get the count of recorded values at a specific value
+ * (to within the histogram resolution at the value level).
+ *
+ * @param h "This" pointer
+ * @param value The value for which to provide the recorded count
+ * @return The total count of values recorded in the histogram within the value range that is
+ * {@literal >=} lowestEquivalentValue(<i>value</i>) and {@literal <=} highestEquivalentValue(<i>value</i>)
+ */
+int64_t hdr_count_at_value(const struct hdr_histogram* h, int64_t value);
+
+int64_t hdr_count_at_index(const struct hdr_histogram* h, int32_t index);
+
+int64_t hdr_value_at_index(const struct hdr_histogram* h, int32_t index);
+
+struct hdr_iter_percentiles
+{
+    bool seen_last_value;
+    int32_t ticks_per_half_distance;
+    double percentile_to_iterate_to;
+    double percentile;
+};
+
+struct hdr_iter_recorded
+{
+    int64_t count_added_in_this_iteration_step;
+};
+
+struct hdr_iter_linear
+{
+    int64_t value_units_per_bucket;
+    int64_t count_added_in_this_iteration_step;
+    int64_t next_value_reporting_level;
+    int64_t next_value_reporting_level_lowest_equivalent;
+};
+
+struct hdr_iter_log
+{
+    double log_base;
+    int64_t count_added_in_this_iteration_step;
+    int64_t next_value_reporting_level;
+    int64_t next_value_reporting_level_lowest_equivalent;
+};
+
+/**
+ * The basic iterator.  This is a generic structure
+ * that supports all of the types of iteration.  Use
+ * the appropriate initialiser to get the desired
+ * iteration.
+ *
+ * @
+ */
+struct hdr_iter
+{
+    const struct hdr_histogram* h;
+    /** raw index into the counts array */
+    int32_t counts_index;
+    /** snapshot of the length at the time the iterator is created */
+    int64_t total_count;
+    /** value directly from array for the current counts_index */
+    int64_t count;
+    /** sum of all of the counts up to and including the count at this index */
+    int64_t cumulative_count;
+    /** The current value based on counts_index */
+    int64_t value;
+    int64_t highest_equivalent_value;
+    int64_t lowest_equivalent_value;
+    int64_t median_equivalent_value;
+    int64_t value_iterated_from;
+    int64_t value_iterated_to;
+
+    union
+    {
+        struct hdr_iter_percentiles percentiles;
+        struct hdr_iter_recorded recorded;
+        struct hdr_iter_linear linear;
+        struct hdr_iter_log log;
+    } specifics;
+
+    bool (* _next_fp)(struct hdr_iter* iter);
+
+};
+
+/**
+ * Initalises the basic iterator.
+ *
+ * @param itr 'This' pointer
+ * @param h The histogram to iterate over
+ */
+void hdr_iter_init(struct hdr_iter* iter, const struct hdr_histogram* h);
+
+/**
+ * Initialise the iterator for use with percentiles.
+ */
+void hdr_iter_percentile_init(struct hdr_iter* iter, const struct hdr_histogram* h, int32_t ticks_per_half_distance);
+
+/**
+ * Initialise the iterator for use with recorded values.
+ */
+void hdr_iter_recorded_init(struct hdr_iter* iter, const struct hdr_histogram* h);
+
+/**
+ * Initialise the iterator for use with linear values.
+ */
+void hdr_iter_linear_init(
+    struct hdr_iter* iter,
+    const struct hdr_histogram* h,
+    int64_t value_units_per_bucket);
+
+/**
+ * Update the iterator value units per bucket
+ */
+void hdr_iter_linear_set_value_units_per_bucket(struct hdr_iter* iter, int64_t value_units_per_bucket);
+
+/**
+ * Initialise the iterator for use with logarithmic values
+ */
+void hdr_iter_log_init(
+    struct hdr_iter* iter,
+    const struct hdr_histogram* h,
+    int64_t value_units_first_bucket,
+    double log_base);
+
+/**
+ * Iterate to the next value for the iterator.  If there are no more values
+ * available return faluse.
+ *
+ * @param itr 'This' pointer
+ * @return 'false' if there are no values remaining for this iterator.
+ */
+bool hdr_iter_next(struct hdr_iter* iter);
+
+typedef enum
+{
+    CLASSIC,
+    CSV
+} format_type;
+
+/**
+ * Print out a percentile based histogram to the supplied stream.  Note that
+ * this call will not flush the FILE, this is left up to the user.
+ *
+ * @param h 'This' pointer
+ * @param stream The FILE to write the output to
+ * @param ticks_per_half_distance The number of iteration steps per half-distance to 100%
+ * @param value_scale Scale the output values by this amount
+ * @param format_type Format to use, e.g. CSV.
+ * @return 0 on success, error code on failure.  EIO if an error occurs writing
+ * the output.
+ */
+int hdr_percentiles_print(
+    struct hdr_histogram* h, FILE* stream, int32_t ticks_per_half_distance,
+    double value_scale, format_type format);
+
+/**
+* Internal allocation methods, used by hdr_dbl_histogram.
+*/
+struct hdr_histogram_bucket_config
+{
+    int64_t lowest_trackable_value;
+    int64_t highest_trackable_value;
+    int64_t unit_magnitude;
+    int64_t significant_figures;
+    int32_t sub_bucket_half_count_magnitude;
+    int32_t sub_bucket_half_count;
+    int64_t sub_bucket_mask;
+    int32_t sub_bucket_count;
+    int32_t bucket_count;
+    int32_t counts_len;
+};
+
+int hdr_calculate_bucket_config(
+    int64_t lowest_trackable_value,
+    int64_t highest_trackable_value,
+    int significant_figures,
+    struct hdr_histogram_bucket_config* cfg);
+
+void hdr_init_preallocated(struct hdr_histogram* h, struct hdr_histogram_bucket_config* cfg);
+
+int64_t hdr_size_of_equivalent_value_range(const struct hdr_histogram* h, int64_t value);
+
+int64_t hdr_next_non_equivalent_value(const struct hdr_histogram* h, int64_t value);
+
+int64_t hdr_median_equivalent_value(const struct hdr_histogram* h, int64_t value);
+
+/**
+ * Used to reset counters after importing data manuallying into the histogram, used by the logging code
+ * and other custom serialisation tools.
+ */
+void hdr_reset_internal_counters(struct hdr_histogram* h);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/Makefile b/src/Makefile
index 3ca7b1685..a8d2aa518 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -16,7 +16,7 @@ release_hdr := $(shell sh -c './mkreleasehdr.sh')
 uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
 uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not')
 OPTIMIZATION?=-O2
-DEPENDENCY_TARGETS=hiredis linenoise lua
+DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram
 NODEPS:=clean distclean
 
 # Default settings
@@ -149,7 +149,7 @@ endif
 endif
 endif
 # Include paths to dependencies
-FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src
+FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram
 
 # Determine systemd support and/or build preference (defaulting to auto-detection)
 BUILD_WITH_SYSTEMD=no
@@ -300,7 +300,7 @@ $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ)
 
 # redis-benchmark
 $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ)
-	$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a $(FINAL_LIBS)
+	$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/hdr_histogram/hdr_histogram.o $(FINAL_LIBS)
 
 dict-benchmark: dict.c zmalloc.c sds.c siphash.c
 	$(REDIS_CC) $(FINAL_CFLAGS) $^ -D DICT_BENCHMARK_MAIN -o $@ $(FINAL_LIBS)
diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index a86079156..a221ebdd2 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -51,12 +51,17 @@
 #include "zmalloc.h"
 #include "atomicvar.h"
 #include "crc16_slottable.h"
+#include "hdr_histogram.h"
 
 #define UNUSED(V) ((void) V)
 #define RANDPTR_INITIAL_SIZE 8
-#define MAX_LATENCY_PRECISION 3
+#define DEFAULT_LATENCY_PRECISION 3
+#define MAX_LATENCY_PRECISION 4
 #define MAX_THREADS 500
 #define CLUSTER_SLOTS 16384
+#define CONFIG_LATENCY_HISTOGRAM_MIN_VALUE 10L          /* >= 10 usecs */
+#define CONFIG_LATENCY_HISTOGRAM_MAX_VALUE 3000000L          /* <= 30 secs(us precision) */
+#define CONFIG_LATENCY_HISTOGRAM_INSTANT_MAX_VALUE 3000000L   /* <= 3 secs(us precision) */
 
 #define CLIENT_GET_EVENTLOOP(c) \
     (c->thread_id >= 0 ? config.threads[c->thread_id]->el : config.el)
@@ -75,6 +80,9 @@ static struct config {
     int requests;
     int requests_issued;
     int requests_finished;
+    int previous_requests_finished;
+    int last_printed_bytes;
+    long long previous_tick;
     int keysize;
     int datasize;
     int randomkeys;
@@ -103,6 +111,8 @@ static struct config {
     int cluster_node_count;
     struct clusterNode **cluster_nodes;
     struct redisConfig *redis_config;
+    struct hdr_histogram* latency_histogram;
+    struct hdr_histogram* current_sec_latency_histogram;
     int is_fetching_slots;
     int is_updating_slots;
     int slots_last_update;
@@ -534,8 +544,23 @@ static void readHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
                 }
                 int requests_finished = 0;
                 atomicGetIncr(config.requests_finished, requests_finished, 1);
-                if (requests_finished < config.requests)
-                    config.latency[requests_finished] = c->latency;
+                if (requests_finished < config.requests){
+                        if (config.num_threads == 0) {
+                            hdr_record_value(
+                            config.latency_histogram,  // Histogram to record to
+                            (long)c->latency<=CONFIG_LATENCY_HISTOGRAM_MAX_VALUE ? (long)c->latency : CONFIG_LATENCY_HISTOGRAM_MAX_VALUE);  // Value to record
+                            hdr_record_value(
+                            config.current_sec_latency_histogram,  // Histogram to record to
+                            (long)c->latency<=CONFIG_LATENCY_HISTOGRAM_INSTANT_MAX_VALUE ? (long)c->latency : CONFIG_LATENCY_HISTOGRAM_INSTANT_MAX_VALUE);  // Value to record
+                        } else {
+                            hdr_record_value_atomic(
+                            config.latency_histogram,  // Histogram to record to
+                            (long)c->latency<=CONFIG_LATENCY_HISTOGRAM_MAX_VALUE ? (long)c->latency : CONFIG_LATENCY_HISTOGRAM_MAX_VALUE);  // Value to record
+                            hdr_record_value_atomic(
+                            config.current_sec_latency_histogram,  // Histogram to record to
+                            (long)c->latency<=CONFIG_LATENCY_HISTOGRAM_INSTANT_MAX_VALUE ? (long)c->latency : CONFIG_LATENCY_HISTOGRAM_INSTANT_MAX_VALUE);  // Value to record
+                        }
+                }
                 c->pending--;
                 if (c->pending == 0) {
                     clientDone(c);
@@ -794,27 +819,18 @@ static void createMissingClients(client c) {
     }
 }
 
-static int compareLatency(const void *a, const void *b) {
-    return (*(long long*)a)-(*(long long*)b);
-}
-
-static int ipow(int base, int exp) {
-    int result = 1;
-    while (exp) {
-        if (exp & 1) result *= base;
-        exp /= 2;
-        base *= base;
-    }
-    return result;
-}
-
 static void showLatencyReport(void) {
-    int i, curlat = 0;
-    int usbetweenlat = ipow(10, MAX_LATENCY_PRECISION-config.precision);
-    float perc, reqpersec;
 
-    reqpersec = (float)config.requests_finished/((float)config.totlatency/1000);
+    const float reqpersec = (float)config.requests_finished/((float)config.totlatency/1000.0f);
+    const float p0 = ((float) hdr_min(config.latency_histogram))/1000.0f;
+    const float p50 = hdr_value_at_percentile(config.latency_histogram, 50.0 )/1000.0f;
+    const float p95 = hdr_value_at_percentile(config.latency_histogram, 95.0 )/1000.0f;
+    const float p99 = hdr_value_at_percentile(config.latency_histogram, 99.0 )/1000.0f;
+    const float p100 = ((float) hdr_max(config.latency_histogram))/1000.0f;
+    const float avg = hdr_mean(config.latency_histogram)/1000.0f;
+
     if (!config.quiet && !config.csv) {
+        printf("%*s\r", config.last_printed_bytes, " "); // ensure there is a clean line
         printf("====== %s ======\n", config.title);
         printf("  %d requests completed in %.2f seconds\n", config.requests_finished,
             (float)config.totlatency/1000);
@@ -847,31 +863,52 @@ static void showLatencyReport(void) {
             printf("  threads: %d\n", config.num_threads);
 
         printf("\n");
-
-        qsort(config.latency,config.requests,sizeof(long long),compareLatency);
-        for (i = 0; i < config.requests; i++) {
-            if (config.latency[i]/usbetweenlat != curlat ||
-                i == (config.requests-1))
-            {
-                /* After the 2 milliseconds latency to have percentages split
-                 * by decimals will just add a lot of noise to the output. */
-                if (config.latency[i] >= 2000) {
-                    config.precision = 0;
-                    usbetweenlat = ipow(10,
-                        MAX_LATENCY_PRECISION-config.precision);
-                }
-
-                curlat = config.latency[i]/usbetweenlat;
-                perc = ((float)(i+1)*100)/config.requests;
-                printf("%.2f%% <= %.*f milliseconds\n", perc, config.precision,
-                    curlat/pow(10.0, config.precision));
+        printf("Latency by percentile distribution:\n");
+        struct hdr_iter iter;
+        long long previous_cumulative_count = -1;
+        const long long total_count = config.latency_histogram->total_count;
+        hdr_iter_percentile_init(&iter, config.latency_histogram, 1);
+        struct hdr_iter_percentiles *percentiles = &iter.specifics.percentiles;
+        while (hdr_iter_next(&iter))
+        {
+            const double value = iter.highest_equivalent_value / 1000.0f;
+            const double percentile = percentiles->percentile;
+            const long long cumulative_count = iter.cumulative_count;
+            if( previous_cumulative_count != cumulative_count || cumulative_count == total_count ){
+                printf("%3.3f%% <= %.3f milliseconds (cumulative count %lld)\n", percentile, value, cumulative_count);
             }
+            previous_cumulative_count = cumulative_count;
         }
-        printf("%.2f requests per second\n\n", reqpersec);
+        printf("\n");
+        printf("Cumulative distribution of latencies:\n");
+        previous_cumulative_count = -1;
+        hdr_iter_linear_init(&iter, config.latency_histogram, 100);
+        while (hdr_iter_next(&iter))
+        {
+            const double value = iter.highest_equivalent_value / 1000.0f;
+            const long long cumulative_count = iter.cumulative_count;
+            const double percentile = ((double)cumulative_count/(double)total_count)*100.0;
+            if( previous_cumulative_count != cumulative_count || cumulative_count == total_count ){
+                printf("%3.3f%% <= %.3f milliseconds (cumulative count %lld)\n", percentile, value, cumulative_count);
+            }
+            /* After the 2 milliseconds latency to have percentages split
+             * by decimals will just add a lot of noise to the output. */
+            if(iter.highest_equivalent_value > 2000){
+                hdr_iter_linear_set_value_units_per_bucket(&iter,1000);
+            }
+            previous_cumulative_count = cumulative_count;
+        }
+        printf("\n");
+        printf("Summary:\n");
+        printf("  throughput summary: %.2f requests per second\n", reqpersec);
+        printf("  latency summary (msec):\n");
+        printf("    %9s %9s %9s %9s %9s %9s\n", "avg", "min", "p50", "p95", "p99", "max");
+        printf("    %9.3f %9.3f %9.3f %9.3f %9.3f %9.3f\n", avg, p0, p50, p95, p99, p100);
     } else if (config.csv) {
-        printf("\"%s\",\"%.2f\"\n", config.title, reqpersec);
+        printf("\"%s\",\"%.2f\",\"%.3f\",\"%.3f\",\"%.3f\",\"%.3f\",\"%.3f\",\"%.3f\"\n", config.title, reqpersec, avg, p0, p50, p95, p99, p100);
     } else {
-        printf("%s: %.2f requests per second\n", config.title, reqpersec);
+        printf("%*s\r", config.last_printed_bytes, " "); // ensure there is a clean line
+        printf("%s: %.2f requests per second, p50=%.3f msec\n", config.title, reqpersec, p50);
     }
 }
 
@@ -904,6 +941,18 @@ static void benchmark(char *title, char *cmd, int len) {
     config.title = title;
     config.requests_issued = 0;
     config.requests_finished = 0;
+    config.previous_requests_finished = 0;
+    config.last_printed_bytes = 0;
+    hdr_init(
+        CONFIG_LATENCY_HISTOGRAM_MIN_VALUE,  // Minimum value
+        CONFIG_LATENCY_HISTOGRAM_MAX_VALUE,  // Maximum value
+        config.precision,  // Number of significant figures
+        &config.latency_histogram);  // Pointer to initialise
+    hdr_init(
+        CONFIG_LATENCY_HISTOGRAM_MIN_VALUE,  // Minimum value
+        CONFIG_LATENCY_HISTOGRAM_INSTANT_MAX_VALUE,  // Maximum value
+        config.precision,  // Number of significant figures
+        &config.current_sec_latency_histogram);  // Pointer to initialise
 
     if (config.num_threads) initBenchmarkThreads();
 
@@ -919,6 +968,9 @@ static void benchmark(char *title, char *cmd, int len) {
     showLatencyReport();
     freeAllClients();
     if (config.threads) freeBenchmarkThreads();
+    if (config.current_sec_latency_histogram) hdr_close(config.current_sec_latency_histogram);
+    if (config.latency_histogram) hdr_close(config.latency_histogram);
+
 }
 
 /* Thread functions. */
@@ -1387,7 +1439,7 @@ int parseOptions(int argc, const char **argv) {
         } else if (!strcmp(argv[i],"--precision")) {
             if (lastarg) goto invalid;
             config.precision = atoi(argv[++i]);
-            if (config.precision < 0) config.precision = 0;
+            if (config.precision < 0) config.precision = DEFAULT_LATENCY_PRECISION;
             if (config.precision > MAX_LATENCY_PRECISION) config.precision = MAX_LATENCY_PRECISION;
         } else if (!strcmp(argv[i],"--threads")) {
              if (lastarg) goto invalid;
@@ -1476,9 +1528,12 @@ int showThroughput(struct aeEventLoop *eventLoop, long long id, void *clientData
     UNUSED(clientData);
     int liveclients = 0;
     int requests_finished = 0;
+    int previous_requests_finished = 0;
+    long long current_tick = mstime();
     atomicGet(config.liveclients, liveclients);
     atomicGet(config.requests_finished, requests_finished);
-
+    atomicGet(config.previous_requests_finished, previous_requests_finished);
+    
     if (liveclients == 0 && requests_finished != config.requests) {
         fprintf(stderr,"All clients disconnected... aborting.\n");
         exit(1);
@@ -1493,9 +1548,14 @@ int showThroughput(struct aeEventLoop *eventLoop, long long id, void *clientData
         fflush(stdout);
 	return 250;
     }
-    float dt = (float)(mstime()-config.start)/1000.0;
-    float rps = (float)requests_finished/dt;
-    printf("%s: %.2f\r", config.title, rps);
+    const float dt = (float)(current_tick-config.start)/1000.0;
+    const float rps = (float)requests_finished/dt;
+    const float instantaneous_dt = (float)(current_tick-config.previous_tick)/1000.0;
+    const float instantaneous_rps = (float)(requests_finished-previous_requests_finished)/instantaneous_dt;
+    config.previous_tick = current_tick;
+    atomicSet(config.previous_requests_finished,requests_finished);
+    config.last_printed_bytes = printf("%s: rps=%.1f (overall: %.1f) avg_msec=%.3f (overall: %.3f)\r", config.title, instantaneous_rps, rps, hdr_mean(config.current_sec_latency_histogram)/1000.0f, hdr_mean(config.latency_histogram)/1000.0f);
+    hdr_reset(config.current_sec_latency_histogram);
     fflush(stdout);
     return 250; /* every 250ms */
 }
@@ -1540,7 +1600,6 @@ int main(int argc, const char **argv) {
     config.csv = 0;
     config.loop = 0;
     config.idlemode = 0;
-    config.latency = NULL;
     config.clients = listCreate();
     config.hostip = "127.0.0.1";
     config.hostport = 6379;
@@ -1548,7 +1607,7 @@ int main(int argc, const char **argv) {
     config.tests = NULL;
     config.dbnum = 0;
     config.auth = NULL;
-    config.precision = 1;
+    config.precision = DEFAULT_LATENCY_PRECISION;
     config.num_threads = 0;
     config.threads = NULL;
     config.cluster_mode = 0;
@@ -1564,8 +1623,6 @@ int main(int argc, const char **argv) {
     argc -= i;
     argv += i;
 
-    config.latency = zmalloc(sizeof(long long)*config.requests);
-
     if (config.cluster_mode) {
         /* Fetch cluster configuration. */
         if (!fetchClusterConfiguration() || !config.cluster_nodes) {
@@ -1611,7 +1668,6 @@ int main(int argc, const char **argv) {
         if (config.redis_config == NULL)
             fprintf(stderr, "WARN: could not fetch server CONFIG\n");
     }
-
     if (config.num_threads > 0) {
         pthread_mutex_init(&(config.requests_issued_mutex), NULL);
         pthread_mutex_init(&(config.requests_finished_mutex), NULL);
@@ -1639,7 +1695,9 @@ int main(int argc, const char **argv) {
         else aeMain(config.el);
         /* and will wait for every */
     }
-
+    if(config.csv){
+        printf("\"test\",\"rps\",\"avg_latency_ms\",\"min_latency_ms\",\"p50_latency_ms\",\"p95_latency_ms\",\"p99_latency_ms\",\"max_latency_ms\"\n");
+    }
     /* Run benchmark with command in the remainder of the arguments. */
     if (argc) {
         sds title = sdsnew(argv[0]);
@@ -1650,6 +1708,8 @@ int main(int argc, const char **argv) {
 
         do {
             len = redisFormatCommandArgv(&cmd,argc,argv,NULL);
+            // adjust the datasize to the parsed command
+            config.datasize = len;
             benchmark(title,cmd,len);
             free(cmd);
         } while(config.loop);

From daef1f00c26632a839ba7b0234100067566266de Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 27 Aug 2020 08:09:39 +0300
Subject: [PATCH 0607/1098] Add test coverage for CLIENT UNBLOCK (#7712)

plus minor other fixes to list.tcl
---
 tests/unit/type/list.tcl | 53 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 52 insertions(+), 1 deletion(-)

diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl
index 5ea2b9cd1..95f18b086 100644
--- a/tests/unit/type/list.tcl
+++ b/tests/unit/type/list.tcl
@@ -1,3 +1,11 @@
+proc wait_for_blocked_client {} {
+    wait_for_condition 50 100 {
+        [s blocked_clients] ne 0
+    } else {
+        fail "no blocked clients"
+    }
+}
+
 start_server {
     tags {"list"}
     overrides {
@@ -52,7 +60,7 @@ start_server {
 
     test {LPOS when RANK is greater than matches} {
         r DEL mylist
-        r LPUSH l a
+        r LPUSH mylist a
         assert {[r LPOS mylist b COUNT 10 RANK 5] eq {}}
     }
 
@@ -217,6 +225,8 @@ start_server {
         r del list
 
         $rd blpop list 0
+        after 100 ;# Make sure rd is blocked before MULTI
+
         r multi
         r lpush list a
         r del list
@@ -887,4 +897,45 @@ start_server {
         $rd2 close
         r ping
     } {PONG}
+
+    test "client unblock tests" {
+        r del l
+        set rd [redis_deferring_client]
+        $rd client id
+        set id [$rd read]
+
+        # test default args
+        $rd blpop l 0
+        wait_for_blocked_client
+        r client unblock $id
+        assert_equal {} [$rd read]
+
+        # test with timeout
+        $rd blpop l 0
+        wait_for_blocked_client
+        r client unblock $id TIMEOUT
+        assert_equal {} [$rd read]
+
+        # test with error
+        $rd blpop l 0
+        wait_for_blocked_client
+        r client unblock $id ERROR
+        catch {[$rd read]} e
+        assert_equal $e "UNBLOCKED client unblocked via CLIENT UNBLOCK"
+
+        # test with invalid client id
+        catch {[r client unblock asd]} e
+        assert_equal $e "ERR value is not an integer or out of range"
+
+        # test with non blocked client
+        set myid [r client id]
+        catch {[r client unblock $myid]} e
+        assert_equal $e {invalid command name "0"}
+
+        # finally, see the this client and list are still functional
+        $rd blpop l 0
+        wait_for_blocked_client
+        r lpush l foo
+        assert_equal {l foo} [$rd read]
+    } {}
 }

From b7289e912cbe1a011a5569cd67929e83731b9660 Mon Sep 17 00:00:00 2001
From: valentinogeron <valentino@redislabs.com>
Date: Thu, 27 Aug 2020 09:19:24 +0300
Subject: [PATCH 0608/1098] EXEC with only read commands should not be rejected
 when OOM (#7696)

If the server gets MULTI command followed by only read
commands, and right before it gets the EXEC it reaches OOM,
the client will get OOM response.

So, from now on, it will get OOM response only if there was
at least one command that was tagged with `use-memory` flag
---
 src/server.c         | 21 +++++++++++++--------
 tests/unit/multi.tcl | 38 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 8 deletions(-)

diff --git a/src/server.c b/src/server.c
index 70307ff0b..42fa87898 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3625,14 +3625,19 @@ int processCommand(client *c) {
          * into a slave, that may be the active client, to be freed. */
         if (server.current_client == NULL) return C_ERR;
 
-        /* It was impossible to free enough memory, and the command the client
-         * is trying to execute is denied during OOM conditions or the client
-         * is in MULTI/EXEC context? Error. */
-        if (out_of_memory &&
-            (is_denyoom_command ||
-             (c->flags & CLIENT_MULTI &&
-              c->cmd->proc != discardCommand)))
-        {
+        int reject_cmd_on_oom = is_denyoom_command;
+        /* If client is in MULTI/EXEC context, queuing may consume an unlimited
+         * amount of memory, so we want to stop that.
+         * However, we never want to reject DISCARD, or even EXEC (unless it
+         * contains denied commands, in which case is_denyoom_command is already
+         * set. */
+        if (c->flags & CLIENT_MULTI &&
+            c->cmd->proc != execCommand &&
+            c->cmd->proc != discardCommand) {
+            reject_cmd_on_oom = 1;
+        }
+
+        if (out_of_memory && reject_cmd_on_oom) {
             rejectCommand(c, shared.oomerr);
             return C_OK;
         }
diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl
index 44a822ba6..817d509c5 100644
--- a/tests/unit/multi.tcl
+++ b/tests/unit/multi.tcl
@@ -466,4 +466,42 @@ start_server {tags {"multi"}} {
         assert { $xx == 1 }
         $r1 close;
     }
+
+    test {EXEC with only read commands should not be rejected when OOM} {
+        set r2 [redis_client]
+
+        r set x value
+        r multi
+        r get x
+        r ping
+
+        # enforcing OOM
+        $r2 config set maxmemory 1
+
+        # finish the multi transaction with exec
+        assert { [r exec] == {value PONG} }
+
+        # releasing OOM
+        $r2 config set maxmemory 0
+        $r2 close
+    }
+
+    test {EXEC with at least one use-memory command should fail} {
+        set r2 [redis_client]
+
+        r multi
+        r set x 1
+        r get x
+
+        # enforcing OOM
+        $r2 config set maxmemory 1
+
+        # finish the multi transaction with exec
+        catch {r exec} e
+        assert_match {EXECABORT*OOM*} $e
+
+        # releasing OOM
+        $r2 config set maxmemory 0
+        $r2 close
+    }
 }

From 8bdcbbb085888d98b765cc7899009670e019b9d0 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 27 Aug 2020 11:09:32 +0300
Subject: [PATCH 0609/1098] Update memory metrics for INFO during loading
 (#7690)

During a long AOF or RDB loading, the memory stats were not updated, and
INFO would return stale data, specifically about fragmentation and RSS.
In the past some of these were sampled directly inside the INFO command,
but were moved to cron as an optimization.

This commit introduces a concept of loadingCron which should take
some of the responsibilities of serverCron.
It attempts to limit it's rate to approximately the server Hz, but may
not be very accurate.

In order to avoid too many system call, we use the cached ustime, and
also make sure to update it in both AOF loading and RDB loading inside
processEventsWhileBlocked (it seems AOF loading was missing it).
---
 src/aof.c        |  1 +
 src/module.c     |  2 +-
 src/networking.c |  4 +++
 src/rdb.c        |  5 +--
 src/server.c     | 85 ++++++++++++++++++++++++++++++------------------
 src/server.h     |  1 +
 6 files changed, 62 insertions(+), 36 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index cbc0989d0..3f33bda3a 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -773,6 +773,7 @@ int loadAppendOnlyFile(char *filename) {
         if (!(loops++ % 1000)) {
             loadingProgress(ftello(fp));
             processEventsWhileBlocked();
+            loadingCron();
             processModuleLoadingProgressEvent(1);
         }
 
diff --git a/src/module.c b/src/module.c
index ce1f1edc9..de66af506 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7258,7 +7258,7 @@ void moduleUnsubscribeAllServerEvents(RedisModule *module) {
 }
 
 void processModuleLoadingProgressEvent(int is_aof) {
-    long long now = ustime();
+    long long now = server.ustime;
     static long long next_event = 0;
     if (now >= next_event) {
         /* Fire the loading progress modules end event. */
diff --git a/src/networking.c b/src/networking.c
index 24424ab90..295f75283 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2887,6 +2887,10 @@ int clientsArePaused(void) {
 void processEventsWhileBlocked(void) {
     int iterations = 4; /* See the function top-comment. */
 
+    /* Update our cached time since it is used to create and update the last
+     * interaction time with clients and for other important things. */
+    updateCachedTime(0);
+
     /* Note: when we are processing events while blocked (for instance during
      * busy Lua scripts), we set a global flag. When such flag is set, we
      * avoid handling the read part of clients using threaded I/O.
diff --git a/src/rdb.c b/src/rdb.c
index 54a169cd8..eab77fb9e 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2069,14 +2069,11 @@ void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) {
     if (server.loading_process_events_interval_bytes &&
         (r->processed_bytes + len)/server.loading_process_events_interval_bytes > r->processed_bytes/server.loading_process_events_interval_bytes)
     {
-        /* The DB can take some non trivial amount of time to load. Update
-         * our cached time since it is used to create and update the last
-         * interaction time with clients and for other important things. */
-        updateCachedTime(0);
         if (server.masterhost && server.repl_state == REPL_STATE_TRANSFER)
             replicationSendNewlineToMaster();
         loadingProgress(r->processed_bytes);
         processEventsWhileBlocked();
+        loadingCron();
         processModuleLoadingProgressEvent(0);
     }
 }
diff --git a/src/server.c b/src/server.c
index 42fa87898..6363a9b3b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1839,6 +1839,41 @@ void checkChildrenDone(void) {
     }
 }
 
+/* Called from serverCron and loadingCron to update cached memory metrics. */
+void cronUpdateMemoryStats() {
+    /* Record the max memory used since the server was started. */
+    if (zmalloc_used_memory() > server.stat_peak_memory)
+        server.stat_peak_memory = zmalloc_used_memory();
+
+    run_with_period(100) {
+        /* Sample the RSS and other metrics here since this is a relatively slow call.
+         * We must sample the zmalloc_used at the same time we take the rss, otherwise
+         * the frag ratio calculate may be off (ratio of two samples at different times) */
+        server.cron_malloc_stats.process_rss = zmalloc_get_rss();
+        server.cron_malloc_stats.zmalloc_used = zmalloc_used_memory();
+        /* Sampling the allcator info can be slow too.
+         * The fragmentation ratio it'll show is potentically more accurate
+         * it excludes other RSS pages such as: shared libraries, LUA and other non-zmalloc
+         * allocations, and allocator reserved pages that can be pursed (all not actual frag) */
+        zmalloc_get_allocator_info(&server.cron_malloc_stats.allocator_allocated,
+                                   &server.cron_malloc_stats.allocator_active,
+                                   &server.cron_malloc_stats.allocator_resident);
+        /* in case the allocator isn't providing these stats, fake them so that
+         * fragmention info still shows some (inaccurate metrics) */
+        if (!server.cron_malloc_stats.allocator_resident) {
+            /* LUA memory isn't part of zmalloc_used, but it is part of the process RSS,
+             * so we must desuct it in order to be able to calculate correct
+             * "allocator fragmentation" ratio */
+            size_t lua_memory = lua_gc(server.lua,LUA_GCCOUNT,0)*1024LL;
+            server.cron_malloc_stats.allocator_resident = server.cron_malloc_stats.process_rss - lua_memory;
+        }
+        if (!server.cron_malloc_stats.allocator_active)
+            server.cron_malloc_stats.allocator_active = server.cron_malloc_stats.allocator_resident;
+        if (!server.cron_malloc_stats.allocator_allocated)
+            server.cron_malloc_stats.allocator_allocated = server.cron_malloc_stats.zmalloc_used;
+    }
+}
+
 /* This is our timer interrupt, called server.hz times per second.
  * Here is where we do a number of things that need to be done asynchronously.
  * For instance:
@@ -1907,37 +1942,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
      * LRU_CLOCK_RESOLUTION define. */
     server.lruclock = getLRUClock();
 
-    /* Record the max memory used since the server was started. */
-    if (zmalloc_used_memory() > server.stat_peak_memory)
-        server.stat_peak_memory = zmalloc_used_memory();
-
-    run_with_period(100) {
-        /* Sample the RSS and other metrics here since this is a relatively slow call.
-         * We must sample the zmalloc_used at the same time we take the rss, otherwise
-         * the frag ratio calculate may be off (ratio of two samples at different times) */
-        server.cron_malloc_stats.process_rss = zmalloc_get_rss();
-        server.cron_malloc_stats.zmalloc_used = zmalloc_used_memory();
-        /* Sampling the allcator info can be slow too.
-         * The fragmentation ratio it'll show is potentically more accurate
-         * it excludes other RSS pages such as: shared libraries, LUA and other non-zmalloc
-         * allocations, and allocator reserved pages that can be pursed (all not actual frag) */
-        zmalloc_get_allocator_info(&server.cron_malloc_stats.allocator_allocated,
-                                   &server.cron_malloc_stats.allocator_active,
-                                   &server.cron_malloc_stats.allocator_resident);
-        /* in case the allocator isn't providing these stats, fake them so that
-         * fragmention info still shows some (inaccurate metrics) */
-        if (!server.cron_malloc_stats.allocator_resident) {
-            /* LUA memory isn't part of zmalloc_used, but it is part of the process RSS,
-             * so we must desuct it in order to be able to calculate correct
-             * "allocator fragmentation" ratio */
-            size_t lua_memory = lua_gc(server.lua,LUA_GCCOUNT,0)*1024LL;
-            server.cron_malloc_stats.allocator_resident = server.cron_malloc_stats.process_rss - lua_memory;
-        }
-        if (!server.cron_malloc_stats.allocator_active)
-            server.cron_malloc_stats.allocator_active = server.cron_malloc_stats.allocator_resident;
-        if (!server.cron_malloc_stats.allocator_allocated)
-            server.cron_malloc_stats.allocator_allocated = server.cron_malloc_stats.zmalloc_used;
-    }
+    cronUpdateMemoryStats();
 
     /* We received a SIGTERM, shutting down here in a safe way, as it is
      * not ok doing so inside the signal handler. */
@@ -2103,6 +2108,24 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
     return 1000/server.hz;
 }
 
+/* This function fill in the role of serverCron during RDB or AOF loading.
+ * It attempts to do its duties at a similar rate as the configured server.hz,
+ * and updates cronloops variable so that similarly to serverCron, the
+ * run_with_period can be used. */
+void loadingCron() {
+    long long now = server.ustime;
+    static long long next_event = 0;
+    if (now >= next_event) {
+        cronUpdateMemoryStats();
+        
+        /* Increment cronloop so that run_with_period works. */
+        server.cronloops++;
+
+        /* Decide when the next event should fire. */
+        next_event = now + 1000000 / server.hz;
+    }
+}
+
 extern int ProcessingEventsWhileBlocked;
 
 /* This function gets called every time Redis is entering the
diff --git a/src/server.h b/src/server.h
index 9badea9e8..97f663076 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1696,6 +1696,7 @@ int listenToPort(int port, int *fds, int *count);
 void pauseClients(mstime_t duration);
 int clientsArePaused(void);
 void processEventsWhileBlocked(void);
+void loadingCron(void);
 int handleClientsWithPendingWrites(void);
 int handleClientsWithPendingWritesUsingThreads(void);
 int handleClientsWithPendingReadsUsingThreads(void);

From 9fcd9e191e6f54276688fb7c74e1d5c3c4be9a75 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 27 Aug 2020 12:54:01 +0300
Subject: [PATCH 0610/1098] Fix rejectCommand trims newline in shared error
 objects, hung clients (#7714)

65a3307bc (released in 6.0.6) has a side effect, when processCommand
rejects a command with pre-made shared object error string, it trims the
newlines from the end of the string. if that string is later used with
addReply, the newline will be missing, breaking the protocol, and
leaving the client hung.

It seems that the only scenario which this happens is when replying with
-LOADING to some command, and later using that reply from the CONFIG
SET command (still during loading). this will result in hung client.

Refactoring the code in order to avoid trimming these newlines from
shared string objects, and do the newline trimming only in other cases
where it's needed.

Co-authored-by: Guy Benoish <guy.benoish@redislabs.com>
---
 src/multi.c      |  3 ++-
 src/networking.c | 47 ++++++++++++++++++++++++++++++-----------------
 src/server.c     | 12 ++++++++----
 src/server.h     |  3 ++-
 4 files changed, 42 insertions(+), 23 deletions(-)

diff --git a/src/multi.c b/src/multi.c
index 35ddf92af..a99c308be 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -127,7 +127,8 @@ void execCommandPropagateExec(client *c) {
 /* Aborts a transaction, with a specific error message.
  * The transaction is always aboarted with -EXECABORT so that the client knows
  * the server exited the multi state, but the actual reason for the abort is
- * included too. */
+ * included too.
+ * Note: 'error' may or may not end with \r\n. see addReplyErrorFormat. */
 void execCommandAbort(client *c, sds error) {
     discardTransaction(c);
 
diff --git a/src/networking.c b/src/networking.c
index 295f75283..4a58cec40 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -357,14 +357,18 @@ void addReplyProto(client *c, const char *s, size_t len) {
  *
  * If the error code is already passed in the string 's', the error
  * code provided is used, otherwise the string "-ERR " for the generic
- * error code is automatically added. */
+ * error code is automatically added.
+ * Note that 's' must NOT end with \r\n. */
 void addReplyErrorLength(client *c, const char *s, size_t len) {
     /* If the string already starts with "-..." then the error code
      * is provided by the caller. Otherwise we use "-ERR". */
     if (!len || s[0] != '-') addReplyProto(c,"-ERR ",5);
     addReplyProto(c,s,len);
     addReplyProto(c,"\r\n",2);
+}
 
+/* Do some actions after an error reply was sent (Log if needed, updates stats, etc.) */
+void afterErrorReply(client *c, const char *s, size_t len) {
     /* Sometimes it could be normal that a slave replies to a master with
      * an error and this function gets called. Actually the error will never
      * be sent because addReply*() against master clients has no effect...
@@ -390,10 +394,11 @@ void addReplyErrorLength(client *c, const char *s, size_t len) {
             from = "master";
         }
 
+        if (len > 4096) len = 4096;
         char *cmdname = c->lastcmd ? c->lastcmd->name : "<unknown>";
         serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error "
-                             "to its %s: '%s' after processing the command "
-                             "'%s'", from, to, s, cmdname);
+                             "to its %s: '%.*s' after processing the command "
+                             "'%s'", from, to, (int)len, s, cmdname);
         if (ctype == CLIENT_TYPE_MASTER && server.repl_backlog &&
             server.repl_backlog_histlen > 0)
         {
@@ -403,31 +408,39 @@ void addReplyErrorLength(client *c, const char *s, size_t len) {
     }
 }
 
+/* The 'err' object is expected to start with -ERRORCODE and end with \r\n.
+ * Unlike addReplyErrorSds and others alike which rely on addReplyErrorLength. */
+void addReplyErrorObject(client *c, robj *err) {
+    addReply(c, err);
+    afterErrorReply(c, err->ptr, sdslen(err->ptr)-2); /* Ignore trailing \r\n */
+}
+
+/* See addReplyErrorLength for expectations from the input string. */
 void addReplyError(client *c, const char *err) {
     addReplyErrorLength(c,err,strlen(err));
+    afterErrorReply(c,err,strlen(err));
 }
 
-/* See addReplyErrorLength.
- * Makes sure there are no newlines in the string, otherwise invalid protocol
- * is emitted. */
-void addReplyErrorSafe(client *c, char *s, size_t len) {
-    size_t j;
-    /* Trim any newlines at the end (ones will be added by addReplyErrorLength) */
-    while (s[len-1] == '\r' || s[len-1] == '\n')
-        len--;
-    /* Replace any newlines in the rest of the string with spaces. */
-    for (j = 0; j < len; j++) {
-        if (s[j] == '\r' || s[j] == '\n') s[j] = ' ';
-    }
-    addReplyErrorLength(c,s,len);
+/* See addReplyErrorLength for expectations from the input string. */
+void addReplyErrorSds(client *c, sds err) {
+    addReplyErrorLength(c,err,sdslen(err));
+    afterErrorReply(c,err,sdslen(err));
 }
 
+/* See addReplyErrorLength for expectations from the formatted string.
+ * The formatted string is safe to contain \r and \n anywhere. */
 void addReplyErrorFormat(client *c, const char *fmt, ...) {
     va_list ap;
     va_start(ap,fmt);
     sds s = sdscatvprintf(sdsempty(),fmt,ap);
     va_end(ap);
-    addReplyErrorSafe(c, s, sdslen(s));
+    /* Trim any newlines at the end (ones will be added by addReplyErrorLength) */
+    s = sdstrim(s, "\r\n");
+    /* Make sure there are no newlines in the middle of the string, otherwise
+     * invalid protocol is emitted. */
+    s = sdsmapchars(s, "\r\n", "  ",  2);
+    addReplyErrorLength(c,s,sdslen(s));
+    afterErrorReply(c,s,sdslen(s));
     sdsfree(s);
 }
 
diff --git a/src/server.c b/src/server.c
index 6363a9b3b..4da4aeeec 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3498,14 +3498,15 @@ void call(client *c, int flags) {
 /* Used when a command that is ready for execution needs to be rejected, due to
  * varios pre-execution checks. it returns the appropriate error to the client.
  * If there's a transaction is flags it as dirty, and if the command is EXEC,
- * it aborts the transaction. */
+ * it aborts the transaction.
+ * Note: 'reply' is expected to end with \r\n */
 void rejectCommand(client *c, robj *reply) {
     flagTransaction(c);
     if (c->cmd && c->cmd->proc == execCommand) {
         execCommandAbort(c, reply->ptr);
     } else {
         /* using addReplyError* rather than addReply so that the error can be logged. */
-        addReplyErrorSafe(c, reply->ptr, sdslen(reply->ptr));
+        addReplyErrorObject(c, reply);
     }
 }
 
@@ -3515,10 +3516,13 @@ void rejectCommandFormat(client *c, const char *fmt, ...) {
     va_start(ap,fmt);
     sds s = sdscatvprintf(sdsempty(),fmt,ap);
     va_end(ap);
+    /* Make sure there are no newlines in the string, otherwise invalid protocol
+     * is emitted (The args come from the user, they may contain any character). */
+    sdsmapchars(s, "\r\n", "  ",  2);
     if (c->cmd && c->cmd->proc == execCommand) {
         execCommandAbort(c, s);
     } else {
-        addReplyErrorSafe(c, s, sdslen(s));
+        addReplyErrorSds(c, s);
     }
     sdsfree(s);
 }
@@ -3688,7 +3692,7 @@ int processCommand(client *c) {
             rejectCommand(c, shared.bgsaveerr);
         else
             rejectCommandFormat(c,
-                "-MISCONF Errors writing to the AOF file: %s\r\n",
+                "-MISCONF Errors writing to the AOF file: %s",
                 strerror(server.aof_last_write_errno));
         return C_OK;
     }
diff --git a/src/server.h b/src/server.h
index 97f663076..53b4fcce3 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1657,7 +1657,8 @@ void addReplyBulkLongLong(client *c, long long ll);
 void addReply(client *c, robj *obj);
 void addReplySds(client *c, sds s);
 void addReplyBulkSds(client *c, sds s);
-void addReplyErrorSafe(client *c, char *s, size_t len);
+void addReplyErrorObject(client *c, robj *err);
+void addReplyErrorSds(client *c, sds err);
 void addReplyError(client *c, const char *err);
 void addReplyStatus(client *c, const char *status);
 void addReplyDouble(client *c, double d);

From c01e94a4319c416c4c231ffbea9e778d52424e30 Mon Sep 17 00:00:00 2001
From: Jim Brunner <brunnerj@amazon.com>
Date: Fri, 28 Aug 2020 01:54:10 -0700
Subject: [PATCH 0611/1098] Use H/W Monotonic clock and updates to AE (#7644)

Update adds a general source for retrieving a monotonic time.
In addition, AE has been updated to utilize the new monotonic
clock for timer processing.

This performance improvement is **not** enabled in a default build due to various H/W compatibility
concerns, see README.md for details. It does however change the default use of gettimeofday with
clock_gettime and somewhat improves performance.

This update provides the following
1. An interface for retrieving a monotonic clock. getMonotonicUs returns a uint64_t (aka monotime)
   with the number of micro-seconds from an arbitrary point. No more messing with tv_sec/tv_usec.
   Simple routines are provided for measuring elapsed milli-seconds or elapsed micro-seconds (the
   most common use case for a monotonic timer). No worries about time moving backwards.
2. High-speed assembler implementation for x86 and ARM. The standard method for retrieving the
   monotonic clock is POSIX.1b (1993): clock_gettime(CLOCK_MONOTONIC, timespec*). However, most
   modern processors provide a constant speed instruction clock which can be retrieved in a fraction
   of the time that it takes to call clock_gettime. For x86, this is provided by the RDTSC
   instruction. For ARM, this is provided by the CNTVCT_EL0 instruction. As a compile-time option,
   these high-speed timers can be chosen. (Default is POSIX clock_gettime.)
3. Refactor of event loop timers. The timer processing in ae.c has been refactored to use the new
   monotonic clock interface. This results in simpler/cleaner logic and improved performance.
---
 README.md       |  12 ++++
 src/Makefile    |   6 +-
 src/ae.c        | 114 +++++++++-----------------------
 src/ae.h        |   6 +-
 src/fmacros.h   |   6 ++
 src/monotonic.c | 170 ++++++++++++++++++++++++++++++++++++++++++++++++
 src/monotonic.h |  52 +++++++++++++++
 src/server.c    |   3 +
 8 files changed, 279 insertions(+), 90 deletions(-)
 create mode 100644 src/monotonic.c
 create mode 100644 src/monotonic.h

diff --git a/README.md b/README.md
index d8f8f7880..46b89c392 100644
--- a/README.md
+++ b/README.md
@@ -106,6 +106,18 @@ To compile against jemalloc on Mac OS X systems, use:
 
     % make MALLOC=jemalloc
 
+Monotonic clock
+---------------
+
+By default, Redis will build using the POSIX clock_gettime function as the
+monotonic clock source.  On most modern systems, the internal processor clock
+can be used to improve performance.  Cautions can be found here: 
+    http://oliveryang.net/2015/09/pitfalls-of-TSC-usage/
+
+To build with support for the processor's internal instruction clock, use:
+
+    % make CFLAGS="-DUSE_PROCESSOR_CLOCK"
+
 Verbose build
 -------------
 
diff --git a/src/Makefile b/src/Makefile
index a8d2aa518..9ea4b654f 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -228,11 +228,11 @@ endif
 
 REDIS_SERVER_NAME=redis-server
 REDIS_SENTINEL_NAME=redis-sentinel
-REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o
+REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o
 REDIS_CLI_NAME=redis-cli
-REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o
+REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o
 REDIS_BENCHMARK_NAME=redis-benchmark
-REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o
+REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o monotonic.o
 REDIS_CHECK_RDB_NAME=redis-check-rdb
 REDIS_CHECK_AOF_NAME=redis-check-aof
 
diff --git a/src/ae.c b/src/ae.c
index 689a27d16..94459acff 100644
--- a/src/ae.c
+++ b/src/ae.c
@@ -30,6 +30,8 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "ae.h"
+
 #include <stdio.h>
 #include <sys/time.h>
 #include <sys/types.h>
@@ -40,7 +42,6 @@
 #include <time.h>
 #include <errno.h>
 
-#include "ae.h"
 #include "zmalloc.h"
 #include "config.h"
 
@@ -60,16 +61,18 @@
     #endif
 #endif
 
+
 aeEventLoop *aeCreateEventLoop(int setsize) {
     aeEventLoop *eventLoop;
     int i;
 
+    monotonicInit();    /* just in case the calling app didn't initialize */
+
     if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) goto err;
     eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize);
     eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize);
     if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err;
     eventLoop->setsize = setsize;
-    eventLoop->lastTime = time(NULL);
     eventLoop->timeEventHead = NULL;
     eventLoop->timeEventNextId = 0;
     eventLoop->stop = 0;
@@ -199,29 +202,6 @@ int aeGetFileEvents(aeEventLoop *eventLoop, int fd) {
     return fe->mask;
 }
 
-static void aeGetTime(long *seconds, long *milliseconds)
-{
-    struct timeval tv;
-
-    gettimeofday(&tv, NULL);
-    *seconds = tv.tv_sec;
-    *milliseconds = tv.tv_usec/1000;
-}
-
-static void aeAddMillisecondsToNow(long long milliseconds, long *sec, long *ms) {
-    long cur_sec, cur_ms, when_sec, when_ms;
-
-    aeGetTime(&cur_sec, &cur_ms);
-    when_sec = cur_sec + milliseconds/1000;
-    when_ms = cur_ms + milliseconds%1000;
-    if (when_ms >= 1000) {
-        when_sec ++;
-        when_ms -= 1000;
-    }
-    *sec = when_sec;
-    *ms = when_ms;
-}
-
 long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
         aeTimeProc *proc, void *clientData,
         aeEventFinalizerProc *finalizerProc)
@@ -232,7 +212,7 @@ long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds,
     te = zmalloc(sizeof(*te));
     if (te == NULL) return AE_ERR;
     te->id = id;
-    aeAddMillisecondsToNow(milliseconds,&te->when_sec,&te->when_ms);
+    te->when = getMonotonicUs() + milliseconds * 1000;
     te->timeProc = proc;
     te->finalizerProc = finalizerProc;
     te->clientData = clientData;
@@ -258,10 +238,8 @@ int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id)
     return AE_ERR; /* NO event with the specified ID found */
 }
 
-/* Search the first timer to fire.
- * This operation is useful to know how many time the select can be
- * put in sleep without to delay any event.
- * If there are no timers NULL is returned.
+/* How many milliseconds until the first timer should fire.
+ * If there are no timers, -1 is returned.
  *
  * Note that's O(N) since time events are unsorted.
  * Possible optimizations (not needed by Redis so far, but...):
@@ -269,19 +247,20 @@ int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id)
  *    Much better but still insertion or deletion of timers is O(N).
  * 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)).
  */
-static aeTimeEvent *aeSearchNearestTimer(aeEventLoop *eventLoop)
-{
+static long msUntilEarliestTimer(aeEventLoop *eventLoop) {
     aeTimeEvent *te = eventLoop->timeEventHead;
-    aeTimeEvent *nearest = NULL;
+    if (te == NULL) return -1;
 
-    while(te) {
-        if (!nearest || te->when_sec < nearest->when_sec ||
-                (te->when_sec == nearest->when_sec &&
-                 te->when_ms < nearest->when_ms))
-            nearest = te;
+    aeTimeEvent *earliest = NULL;
+    while (te) {
+        if (!earliest || te->when < earliest->when)
+            earliest = te;
         te = te->next;
     }
-    return nearest;
+
+    monotime now = getMonotonicUs();
+    return (now >= earliest->when)
+            ? 0 : (long)((earliest->when - now) / 1000);
 }
 
 /* Process time events */
@@ -289,29 +268,11 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
     int processed = 0;
     aeTimeEvent *te;
     long long maxId;
-    time_t now = time(NULL);
-
-    /* If the system clock is moved to the future, and then set back to the
-     * right value, time events may be delayed in a random way. Often this
-     * means that scheduled operations will not be performed soon enough.
-     *
-     * Here we try to detect system clock skews, and force all the time
-     * events to be processed ASAP when this happens: the idea is that
-     * processing events earlier is less dangerous than delaying them
-     * indefinitely, and practice suggests it is. */
-    if (now < eventLoop->lastTime) {
-        te = eventLoop->timeEventHead;
-        while(te) {
-            te->when_sec = 0;
-            te = te->next;
-        }
-    }
-    eventLoop->lastTime = now;
 
     te = eventLoop->timeEventHead;
     maxId = eventLoop->timeEventNextId-1;
+    monotime now = getMonotonicUs();
     while(te) {
-        long now_sec, now_ms;
         long long id;
 
         /* Remove events scheduled for deletion. */
@@ -330,8 +291,10 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
                 eventLoop->timeEventHead = te->next;
             if (te->next)
                 te->next->prev = te->prev;
-            if (te->finalizerProc)
+            if (te->finalizerProc) {
                 te->finalizerProc(eventLoop, te->clientData);
+                now = getMonotonicUs();
+            }
             zfree(te);
             te = next;
             continue;
@@ -346,10 +309,8 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
             te = te->next;
             continue;
         }
-        aeGetTime(&now_sec, &now_ms);
-        if (now_sec > te->when_sec ||
-            (now_sec == te->when_sec && now_ms >= te->when_ms))
-        {
+
+        if (te->when <= now) {
             int retval;
 
             id = te->id;
@@ -357,8 +318,9 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
             retval = te->timeProc(eventLoop, id, te->clientData);
             te->refcount--;
             processed++;
+            now = getMonotonicUs();
             if (retval != AE_NOMORE) {
-                aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms);
+                te->when = now + retval * 1000;
             } else {
                 te->id = AE_DELETED_EVENT_ID;
             }
@@ -397,30 +359,16 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
     if (eventLoop->maxfd != -1 ||
         ((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) {
         int j;
-        aeTimeEvent *shortest = NULL;
         struct timeval tv, *tvp;
+        long msUntilTimer = -1;
 
         if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT))
-            shortest = aeSearchNearestTimer(eventLoop);
-        if (shortest) {
-            long now_sec, now_ms;
+            msUntilTimer = msUntilEarliestTimer(eventLoop);
 
-            aeGetTime(&now_sec, &now_ms);
+        if (msUntilTimer >= 0) {
+            tv.tv_sec = msUntilTimer / 1000;
+            tv.tv_usec = (msUntilTimer % 1000) * 1000;
             tvp = &tv;
-
-            /* How many milliseconds we need to wait for the next
-             * time event to fire? */
-            long long ms =
-                (shortest->when_sec - now_sec)*1000 +
-                shortest->when_ms - now_ms;
-
-            if (ms > 0) {
-                tvp->tv_sec = ms/1000;
-                tvp->tv_usec = (ms % 1000)*1000;
-            } else {
-                tvp->tv_sec = 0;
-                tvp->tv_usec = 0;
-            }
         } else {
             /* If we have to check for events but need to return
              * ASAP because of AE_DONT_WAIT we need to set the timeout
diff --git a/src/ae.h b/src/ae.h
index d1b7f34bf..4a1edb68f 100644
--- a/src/ae.h
+++ b/src/ae.h
@@ -33,7 +33,7 @@
 #ifndef __AE_H__
 #define __AE_H__
 
-#include <time.h>
+#include "monotonic.h"
 
 #define AE_OK 0
 #define AE_ERR -1
@@ -79,8 +79,7 @@ typedef struct aeFileEvent {
 /* Time event structure */
 typedef struct aeTimeEvent {
     long long id; /* time event identifier. */
-    long when_sec; /* seconds */
-    long when_ms; /* milliseconds */
+    monotime when;
     aeTimeProc *timeProc;
     aeEventFinalizerProc *finalizerProc;
     void *clientData;
@@ -101,7 +100,6 @@ typedef struct aeEventLoop {
     int maxfd;   /* highest file descriptor currently registered */
     int setsize; /* max number of file descriptors tracked */
     long long timeEventNextId;
-    time_t lastTime;     /* Used to detect system clock skew */
     aeFileEvent *events; /* Registered events */
     aeFiredEvent *fired; /* Fired events */
     aeTimeEvent *timeEventHead;
diff --git a/src/fmacros.h b/src/fmacros.h
index 6e56c759d..089dc8de7 100644
--- a/src/fmacros.h
+++ b/src/fmacros.h
@@ -58,4 +58,10 @@
 #define _LARGEFILE_SOURCE
 #define _FILE_OFFSET_BITS 64
 
+#ifdef __linux__
+/* features.h uses the defines above to set feature specific defines.  */
+#include <linux/version.h>
+#include <features.h>
+#endif
+
 #endif
diff --git a/src/monotonic.c b/src/monotonic.c
new file mode 100644
index 000000000..5bb4f03bf
--- /dev/null
+++ b/src/monotonic.c
@@ -0,0 +1,170 @@
+#include "monotonic.h"
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <time.h>
+
+#undef NDEBUG
+#include <assert.h>
+
+
+/* The function pointer for clock retrieval.  */
+monotime (*getMonotonicUs)(void) = NULL;
+
+static char monotonic_info_string[32];
+
+
+/* Using the processor clock (aka TSC on x86) can provide improved performance
+ * throughout Redis wherever the monotonic clock is used.  The processor clock
+ * is significantly faster than calling 'clock_getting' (POSIX).  While this is
+ * generally safe on modern systems, this link provides additional information
+ * about use of the x86 TSC: http://oliveryang.net/2015/09/pitfalls-of-TSC-usage
+ *
+ * To use the processor clock, either uncomment this line, or build with
+ *   CFLAGS="-DUSE_PROCESSOR_CLOCK"
+#define USE_PROCESSOR_CLOCK
+ */
+
+
+#if defined(USE_PROCESSOR_CLOCK) && defined(__x86_64__) && defined(__linux__)
+#include <regex.h>
+#include <x86intrin.h>
+
+static long mono_ticksPerMicrosecond = 0;
+
+static monotime getMonotonicUs_x86() {
+    return __rdtsc() / mono_ticksPerMicrosecond;
+}
+
+static void monotonicInit_x86linux() {
+    const int bufflen = 256;
+    char buf[bufflen];
+    regex_t cpuGhzRegex, constTscRegex;
+    const size_t nmatch = 2;
+    regmatch_t pmatch[nmatch];
+    int constantTsc = 0;
+    int rc;
+
+    /* Determine the number of TSC ticks in a micro-second.  This is
+     * a constant value matching the standard speed of the processor.
+     * On modern processors, this speed remains constant even though
+     * the actual clock speed varies dynamically for each core.  */
+    rc = regcomp(&cpuGhzRegex, "^model name\\s+:.*@ ([0-9.]+)GHz", REG_EXTENDED);
+    assert(rc == 0);
+
+    /* Also check that the constant_tsc flag is present.  (It should be
+     * unless this is a really old CPU.  */
+    rc = regcomp(&constTscRegex, "^flags\\s+:.* constant_tsc", REG_EXTENDED);
+    assert(rc == 0);
+
+    FILE *cpuinfo = fopen("/proc/cpuinfo", "r");
+    if (cpuinfo != NULL) {
+        while (fgets(buf, bufflen, cpuinfo) != NULL) {
+            if (regexec(&cpuGhzRegex, buf, nmatch, pmatch, 0) == 0) {
+                buf[pmatch[1].rm_eo] = '\0';
+                double ghz = atof(&buf[pmatch[1].rm_so]);
+                mono_ticksPerMicrosecond = (long)(ghz * 1000);
+                break;
+            }
+        }
+        while (fgets(buf, bufflen, cpuinfo) != NULL) {
+            if (regexec(&constTscRegex, buf, nmatch, pmatch, 0) == 0) {
+                constantTsc = 1;
+                break;
+            }
+        }
+
+        fclose(cpuinfo);
+    }
+    regfree(&cpuGhzRegex);
+    regfree(&constTscRegex);
+
+    if (mono_ticksPerMicrosecond == 0) {
+        fprintf(stderr, "monotonic: x86 linux, unable to determine clock rate");
+        return;
+    }
+    if (!constantTsc) {
+        fprintf(stderr, "monotonic: x86 linux, 'constant_tsc' flag not present");
+        return;
+    }
+
+    snprintf(monotonic_info_string, sizeof(monotonic_info_string),
+            "X86 TSC @ %ld ticks/us", mono_ticksPerMicrosecond);
+    getMonotonicUs = getMonotonicUs_x86;
+}
+#endif
+
+
+#if defined(USE_PROCESSOR_CLOCK) && defined(__aarch64__)
+static long mono_ticksPerMicrosecond = 0;
+
+/* Read the clock value.  */
+static inline uint64_t __cntvct() {
+    uint64_t virtual_timer_value;
+    __asm__ volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value));
+    return virtual_timer_value;
+}
+
+/* Read the Count-timer Frequency.  */
+static inline uint32_t cntfrq_hz() {
+    uint64_t virtual_freq_value;
+    __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(virtual_freq_value));
+    return (uint32_t)virtual_freq_value;    /* top 32 bits are reserved */
+}
+
+static monotime getMonotonicUs_aarch64() {
+    return __cntvct() / mono_ticksPerMicrosecond;
+}
+
+static void monotonicInit_aarch64() {
+    mono_ticksPerMicrosecond = (long)cntfrq_hz() / 1000L / 1000L;
+    if (mono_ticksPerMicrosecond == 0) {
+        fprintf(stderr, "monotonic: aarch64, unable to determine clock rate");
+        return;
+    }
+
+    snprintf(monotonic_info_string, sizeof(monotonic_info_string),
+            "ARM CNTVCT @ %ld ticks/us", mono_ticksPerMicrosecond);
+    getMonotonicUs = getMonotonicUs_aarch64;
+}
+#endif
+
+
+static monotime getMonotonicUs_posix() {
+    /* clock_gettime() is specified in POSIX.1b (1993).  Even so, some systems
+     * did not support this until much later.  CLOCK_MONOTONIC is technically
+     * optional and may not be supported - but it appears to be universal.
+     * If this is not supported, provide a system-specific alternate version.  */
+    struct timespec ts;
+    clock_gettime(CLOCK_MONOTONIC, &ts);
+    return ((uint64_t)ts.tv_sec) * 1000000 + ts.tv_nsec / 1000;
+}
+
+static void monotonicInit_posix() {
+    /* Ensure that CLOCK_MONOTONIC is supported.  This should be supported
+     * on any reasonably current OS.  If the assertion below fails, provide
+     * an appropriate alternate implementation.  */
+    struct timespec ts;
+    int rc = clock_gettime(CLOCK_MONOTONIC, &ts);
+    assert(rc == 0);
+
+    snprintf(monotonic_info_string, sizeof(monotonic_info_string),
+            "POSIX clock_gettime");
+    getMonotonicUs = getMonotonicUs_posix;
+}
+
+
+
+const char * monotonicInit() {
+    #if defined(USE_PROCESSOR_CLOCK) && defined(__x86_64__) && defined(__linux__)
+    if (getMonotonicUs == NULL) monotonicInit_x86linux();
+    #endif
+
+    #if defined(USE_PROCESSOR_CLOCK) && defined(__aarch64__)
+    if (getMonotonicUs == NULL) monotonicInit_aarch64();
+    #endif
+
+    if (getMonotonicUs == NULL) monotonicInit_posix();
+
+    return monotonic_info_string;
+}
diff --git a/src/monotonic.h b/src/monotonic.h
new file mode 100644
index 000000000..4e82f9d53
--- /dev/null
+++ b/src/monotonic.h
@@ -0,0 +1,52 @@
+#ifndef __MONOTONIC_H
+#define __MONOTONIC_H
+/* The monotonic clock is an always increasing clock source.  It is unrelated to
+ * the actual time of day and should only be used for relative timings.  The
+ * monotonic clock is also not guaranteed to be chronologically precise; there
+ * may be slight skew/shift from a precise clock.
+ *
+ * Depending on system architecture, the monotonic time may be able to be
+ * retrieved much faster than a normal clock source by using an instruction
+ * counter on the CPU.  On x86 architectures (for example), the RDTSC
+ * instruction is a very fast clock source for this purpose.
+ */
+
+#include "fmacros.h"
+#include <stdint.h>
+#include <unistd.h>
+
+/* A counter in micro-seconds.  The 'monotime' type is provided for variables
+ * holding a monotonic time.  This will help distinguish & document that the
+ * variable is associated with the monotonic clock and should not be confused
+ * with other types of time.*/
+typedef uint64_t monotime;
+
+/* Retrieve counter of micro-seconds relative to an arbitrary point in time.  */
+extern monotime (*getMonotonicUs)(void);
+
+
+/* Call once at startup to initialize the monotonic clock.  Though this only
+ * needs to be called once, it may be called additional times without impact.
+ * Returns a printable string indicating the type of clock initialized.
+ * (The returned string is static and doesn't need to be freed.)  */
+const char * monotonicInit();
+
+
+/* Functions to measure elapsed time.  Example:
+ *     monotime myTimer;
+ *     elapsedStart(&myTimer);
+ *     while (elapsedMs(myTimer) < 10) {} // loops for 10ms
+ */
+static inline void elapsedStart(monotime *start_time) {
+    *start_time = getMonotonicUs();
+}
+
+static inline uint64_t elapsedUs(monotime start_time) {
+    return getMonotonicUs() - start_time;
+}
+
+static inline uint64_t elapsedMs(monotime start_time) {
+    return elapsedUs(start_time) / 1000;
+}
+
+#endif
diff --git a/src/server.c b/src/server.c
index 4da4aeeec..d3f5c34f8 100644
--- a/src/server.c
+++ b/src/server.c
@@ -28,6 +28,7 @@
  */
 
 #include "server.h"
+#include "monotonic.h"
 #include "cluster.h"
 #include "slowlog.h"
 #include "bio.h"
@@ -2874,6 +2875,8 @@ void initServer(void) {
 
     createSharedObjects();
     adjustOpenFilesLimit();
+    const char *clk_msg = monotonicInit();
+    serverLog(LL_NOTICE, "monotonic clock: %s", clk_msg);
     server.el = aeCreateEventLoop(server.maxclients+CONFIG_FDSET_INCR);
     if (server.el == NULL) {
         serverLog(LL_WARNING,

From ee4a15aae08486077eb604d3d82e4f350bfcc041 Mon Sep 17 00:00:00 2001
From: maohuazhu <34360891+neal-zhu@users.noreply.github.com>
Date: Fri, 28 Aug 2020 09:22:35 -0500
Subject: [PATCH 0612/1098] Optimize __ziplistCascadeUpdate algorithm (#6886)

The previous algorithm is of O(n^2) time complexity.
It would have run through the ziplist entries one by one, each time doing a `realloc` and a
`memmove` (moving the entire tail of the ziplist).

The new algorithm is O(n), it runs over all the records once, computing the size of the `realloc`
needed, then does one `realloc`, and run thought the records again doing many smaller `memmove`s,
each time moving just one record.

So this change reduces many reallocs, and moves each record just once.

Co-authored-by: zhumaohua <zhumaohua@megvii.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/ziplist.c | 247 ++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 201 insertions(+), 46 deletions(-)

diff --git a/src/ziplist.c b/src/ziplist.c
index 13881c117..d4bdb4e78 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -614,62 +614,99 @@ unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
  * The pointer "p" points to the first entry that does NOT need to be
  * updated, i.e. consecutive fields MAY need an update. */
 unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p) {
-    size_t curlen = intrev32ifbe(ZIPLIST_BYTES(zl)), rawlen, rawlensize;
-    size_t offset, noffset, extra;
-    unsigned char *np;
-    zlentry cur, next;
+    zlentry cur;
+    size_t prevlen, prevlensize, prevoffset; /* Informat of the last changed entry. */
+    size_t firstentrylen; /* Used to handle insert at head. */
+    size_t rawlen, curlen = intrev32ifbe(ZIPLIST_BYTES(zl));
+    size_t extra = 0, cnt = 0, offset;
+    size_t delta = 4; /* Extra bytes needed to update a entry's prevlen (5-1). */
+    unsigned char *tail = zl + intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl));
 
+    /* Empty ziplist */
+    if (p[0] == ZIP_END) return zl;
+
+    zipEntry(p, &cur);
+    firstentrylen = prevlen = cur.headersize + cur.len;
+    prevlensize = zipStorePrevEntryLength(NULL, prevlen);
+    prevoffset = p - zl;
+    p += prevlen;
+
+    /* Iterate ziplist to find out how many extra bytes do we need to update it. */
     while (p[0] != ZIP_END) {
         zipEntry(p, &cur);
-        rawlen = cur.headersize + cur.len;
-        rawlensize = zipStorePrevEntryLength(NULL,rawlen);
-
-        /* Abort if there is no next entry. */
-        if (p[rawlen] == ZIP_END) break;
-        zipEntry(p+rawlen, &next);
 
         /* Abort when "prevlen" has not changed. */
-        if (next.prevrawlen == rawlen) break;
+        if (cur.prevrawlen == prevlen) break;
 
-        if (next.prevrawlensize < rawlensize) {
-            /* The "prevlen" field of "next" needs more bytes to hold
-             * the raw length of "cur". */
-            offset = p-zl;
-            extra = rawlensize-next.prevrawlensize;
-            zl = ziplistResize(zl,curlen+extra);
-            p = zl+offset;
-
-            /* Current pointer and offset for next element. */
-            np = p+rawlen;
-            noffset = np-zl;
-
-            /* Update tail offset when next element is not the tail element. */
-            if ((zl+intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))) != np) {
-                ZIPLIST_TAIL_OFFSET(zl) =
-                    intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+extra);
-            }
-
-            /* Move the tail to the back. */
-            memmove(np+rawlensize,
-                np+next.prevrawlensize,
-                curlen-noffset-next.prevrawlensize-1);
-            zipStorePrevEntryLength(np,rawlen);
-
-            /* Advance the cursor */
-            p += rawlen;
-            curlen += extra;
-        } else {
-            if (next.prevrawlensize > rawlensize) {
-                /* This would result in shrinking, which we want to avoid.
-                 * So, set "rawlen" in the available bytes. */
-                zipStorePrevEntryLengthLarge(p+rawlen,rawlen);
+        /* Abort when entry's "prevlensize" is big enough. */
+        if (cur.prevrawlensize >= prevlensize) {
+            if (cur.prevrawlensize == prevlensize) {
+                zipStorePrevEntryLength(p, prevlen);
             } else {
-                zipStorePrevEntryLength(p+rawlen,rawlen);
+                /* This would result in shrinking, which we want to avoid.
+                 * So, set "prevlen" in the available bytes. */
+                zipStorePrevEntryLengthLarge(p, prevlen);
             }
-
-            /* Stop here, as the raw length of "next" has not changed. */
             break;
         }
+
+        /* cur.prevrawlen means cur is the former head entry. */
+        assert(cur.prevrawlen == 0 || cur.prevrawlen + delta == prevlen);
+
+        /* Update prev entry's info and advance the cursor. */
+        rawlen = cur.headersize + cur.len;
+        prevlen = rawlen + delta; 
+        prevlensize = zipStorePrevEntryLength(NULL, prevlen);
+        prevoffset = p - zl;
+        p += rawlen;
+        extra += delta;
+        cnt++;
+    }
+
+    /* Extra bytes is zero all update has been done(or no need to update). */
+    if (extra == 0) return zl;
+
+    /* Update tail offset after loop. */
+    if (tail == zl + prevoffset) {
+        /* When the the last entry we need to update is also the tail, update tail offset
+         * unless this is the only entry that was updated (so the tail offset didn't change). */
+        if (extra - delta != 0) {
+            ZIPLIST_TAIL_OFFSET(zl) =
+                intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+extra-delta);
+        }
+    } else {
+        /* Update the tail offset in cases where the last entry we updated is not the tail. */
+        ZIPLIST_TAIL_OFFSET(zl) =
+            intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+extra);
+    }
+
+    /* Now "p" points at the first unchanged byte in original ziplist,
+     * move data after that to new ziplist. */
+    offset = p - zl;
+    zl = ziplistResize(zl, curlen + extra);
+    p = zl + offset;
+    memmove(p + extra, p, curlen - offset - 1);
+    p += extra;
+
+    /* Iterate all entries that need to be updated tail to head. */
+    while (cnt) {
+        zipEntry(zl + prevoffset, &cur);
+        rawlen = cur.headersize + cur.len;
+        /* Move entry to tail and reset prevlen. */
+        memmove(p - (rawlen - cur.prevrawlensize), 
+                zl + prevoffset + cur.prevrawlensize, 
+                rawlen - cur.prevrawlensize);
+        p -= (rawlen + delta);
+        if (cur.prevrawlen == 0) {
+            /* "cur" is the previous head entry, update its prevlen with firstentrylen. */
+            zipStorePrevEntryLength(p, firstentrylen);
+        } else {
+            /* An entry's prevlen can only increment 4 bytes. */
+            zipStorePrevEntryLength(p, cur.prevrawlen+delta);
+        }
+        /* Foward to previous entry. */
+        prevoffset -= cur.prevrawlen;
+        cnt--;
     }
     return zl;
 }
@@ -1379,6 +1416,30 @@ static void verify(unsigned char *zl, zlentry *e) {
     }
 }
 
+static unsigned char *insertHelper(unsigned char *zl, char ch, size_t len, unsigned char *pos) {
+    assert(len <= ZIP_BIG_PREVLEN);
+    unsigned char data[ZIP_BIG_PREVLEN] = {0};
+    memset(data, ch, len);
+    return ziplistInsert(zl, pos, data, len);
+}
+
+static int compareHelper(unsigned char *zl, char ch, size_t len, int index) {
+    assert(len <= ZIP_BIG_PREVLEN);
+    unsigned char data[ZIP_BIG_PREVLEN] = {0};
+    memset(data, ch, len);
+    unsigned char *p = ziplistIndex(zl, index);
+    assert(p != NULL);
+    return ziplistCompare(p, data, len);
+}
+
+static size_t strEntryBytesSmall(size_t slen) {
+    return slen + zipStorePrevEntryLength(NULL, 0) + zipStoreEntryEncoding(NULL, 0, slen);
+}
+
+static size_t strEntryBytesLarge(size_t slen) {
+    return slen + zipStorePrevEntryLength(NULL, ZIP_BIG_PREVLEN) + zipStoreEntryEncoding(NULL, 0, slen);
+}
+
 int ziplistTest(int argc, char **argv) {
     unsigned char *zl, *p;
     unsigned char *entry;
@@ -1927,6 +1988,100 @@ int ziplistTest(int argc, char **argv) {
         stress(ZIPLIST_TAIL,100000,16384,256);
     }
 
+    printf("Stress __ziplistCascadeUpdate:\n");
+    {
+        char data[ZIP_BIG_PREVLEN];
+        zl = ziplistNew();
+        for (int i = 0; i < 100000; i++) {
+            zl = ziplistPush(zl, (unsigned char*)data, ZIP_BIG_PREVLEN-4, ZIPLIST_TAIL);
+        }
+        unsigned long long start = usec();
+        zl = ziplistPush(zl, (unsigned char*)data, ZIP_BIG_PREVLEN-3, ZIPLIST_HEAD);
+        printf("Done. usec=%lld\n\n", usec()-start);
+        zfree(zl);
+    }
+
+    printf("Edge cases of __ziplistCascadeUpdate:\n");
+    {
+        /* Inserting a entry with data length greater than ZIP_BIG_PREVLEN-4 
+         * will leads to cascade update. */
+        size_t s1 = ZIP_BIG_PREVLEN-4, s2 = ZIP_BIG_PREVLEN-3;
+        zl = ziplistNew();
+
+        zlentry e[4] = {{.prevrawlensize = 0, .prevrawlen = 0, .lensize = 0,
+                         .len = 0, .headersize = 0, .encoding = 0, .p = NULL}};
+
+        zl = insertHelper(zl, 'a', s1, ZIPLIST_ENTRY_HEAD(zl));
+        verify(zl, e);
+
+        assert(e[0].prevrawlensize == 1 && e[0].prevrawlen == 0);
+        assert(compareHelper(zl, 'a', s1, 0));
+        ziplistRepr(zl);
+
+        /* No expand. */
+        zl = insertHelper(zl, 'b', s1, ZIPLIST_ENTRY_HEAD(zl));
+        verify(zl, e);
+
+        assert(e[0].prevrawlensize == 1 && e[0].prevrawlen == 0);
+        assert(compareHelper(zl, 'b', s1, 0));
+
+        assert(e[1].prevrawlensize == 1 && e[1].prevrawlen == strEntryBytesSmall(s1));
+        assert(compareHelper(zl, 'a', s1, 1));
+
+        ziplistRepr(zl);
+
+        /* Expand(tail included). */
+        zl = insertHelper(zl, 'c', s2, ZIPLIST_ENTRY_HEAD(zl));
+        verify(zl, e);
+
+        assert(e[0].prevrawlensize == 1 && e[0].prevrawlen == 0);
+        assert(compareHelper(zl, 'c', s2, 0));
+
+        assert(e[1].prevrawlensize == 5 && e[1].prevrawlen == strEntryBytesSmall(s2));
+        assert(compareHelper(zl, 'b', s1, 1));
+
+        assert(e[2].prevrawlensize == 5 && e[2].prevrawlen == strEntryBytesLarge(s1));
+        assert(compareHelper(zl, 'a', s1, 2));
+
+        ziplistRepr(zl);
+
+        /* Expand(only previous head entry). */
+        zl = insertHelper(zl, 'd', s2, ZIPLIST_ENTRY_HEAD(zl));
+        verify(zl, e);
+
+        assert(e[0].prevrawlensize == 1 && e[0].prevrawlen == 0);
+        assert(compareHelper(zl, 'd', s2, 0));
+
+        assert(e[1].prevrawlensize == 5 && e[1].prevrawlen == strEntryBytesSmall(s2));
+        assert(compareHelper(zl, 'c', s2, 1));
+
+        assert(e[2].prevrawlensize == 5 && e[2].prevrawlen == strEntryBytesLarge(s2));
+        assert(compareHelper(zl, 'b', s1, 2));
+
+        assert(e[3].prevrawlensize == 5 && e[3].prevrawlen == strEntryBytesLarge(s1));
+        assert(compareHelper(zl, 'a', s1, 3));
+
+        ziplistRepr(zl);
+
+        /* Delete from mid. */
+        unsigned char *p = ziplistIndex(zl, 2);
+        zl = ziplistDelete(zl, &p);
+        verify(zl, e);
+
+        assert(e[0].prevrawlensize == 1 && e[0].prevrawlen == 0);
+        assert(compareHelper(zl, 'd', s2, 0));
+
+        assert(e[1].prevrawlensize == 5 && e[1].prevrawlen == strEntryBytesSmall(s2));
+        assert(compareHelper(zl, 'c', s2, 1));
+
+        assert(e[2].prevrawlensize == 5 && e[2].prevrawlen == strEntryBytesLarge(s2));
+        assert(compareHelper(zl, 'a', s1, 2));
+
+        ziplistRepr(zl);
+
+        zfree(zl);
+    }
+
     return 0;
 }
 #endif

From b61b663895f16d9f559a14c408c225062254a57b Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 30 Aug 2020 12:23:47 +0300
Subject: [PATCH 0613/1098] Fix oom-score-adj on older distros. (#7724)

Don't assume `ps` handles `-h` to display output without headers and
manually trim headers line from output.
---
 tests/unit/oom-score-adj.tcl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/unit/oom-score-adj.tcl b/tests/unit/oom-score-adj.tcl
index 894a70fb2..993004602 100644
--- a/tests/unit/oom-score-adj.tcl
+++ b/tests/unit/oom-score-adj.tcl
@@ -16,8 +16,8 @@ if {$system_name eq {linux}} {
 
         proc get_child_pid {} {
             set pid [srv 0 pid]
-            set fd [open "|ps --ppid $pid -o pid -h" "r"]
-            set child_pid [string trim [read $fd]]
+            set fd [open "|ps --ppid $pid -o pid" "r"]
+            set child_pid [string trim [lindex [split [read $fd] \n] 1]]
             close $fd
 
             return $child_pid

From 571571ca192ec0b7cc66ca61cd6794dcb6a9d8bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Leo=C5=A1=20Liter=C3=A1k?=
 <literakl@users.noreply.github.com>
Date: Mon, 31 Aug 2020 11:44:09 +0200
Subject: [PATCH 0614/1098] Update README.md with instructions how to build
 with systemd support (#7730)

#7728 - update instructions for systemd support
---
 README.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/README.md b/README.md
index 46b89c392..6c0dbf525 100644
--- a/README.md
+++ b/README.md
@@ -40,6 +40,11 @@ libssl-dev on Debian/Ubuntu) and run:
 
     % make BUILD_TLS=yes
 
+To build with systemd support, you'll need systemd development libraries (such 
+as libsystemd-dev on Debian/Ubuntu or systemd-devel on CentOS) and run:
+
+    % make BUILD_WITH_SYSTEMD=yes USE_SYSTEMD=yes
+
 You can run a 32 bit Redis binary using:
 
     % make 32bit

From d75ad774a92bd7de0b9448be3d622d7a13b7af27 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 31 Aug 2020 20:42:46 +0300
Subject: [PATCH 0615/1098] Backport Lua 5.2.2 stack overflow fix. (#7733)

This fixes the issue described in CVE-2014-5461. At this time we cannot
confirm that the original issue has a real impact on Redis, but it is
included as an extra safety measure.
---
 deps/lua/src/ldo.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deps/lua/src/ldo.c b/deps/lua/src/ldo.c
index 514f7a2a3..939940a4c 100644
--- a/deps/lua/src/ldo.c
+++ b/deps/lua/src/ldo.c
@@ -274,7 +274,7 @@ int luaD_precall (lua_State *L, StkId func, int nresults) {
     CallInfo *ci;
     StkId st, base;
     Proto *p = cl->p;
-    luaD_checkstack(L, p->maxstacksize);
+    luaD_checkstack(L, p->maxstacksize + p->numparams);
     func = restorestack(L, funcr);
     if (!p->is_vararg) {  /* no varargs? */
       base = func + 1;

From b35d6e5cffb895ab6cbbd9e5e56ef00299b1359e Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 1 Sep 2020 10:02:14 +0300
Subject: [PATCH 0616/1098] Fix double-make issue with make && make install.
 (#7734)

All user-supplied variables that affect the build should be explicitly
persisted.

Fixes #7254
---
 src/Makefile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Makefile b/src/Makefile
index 9ea4b654f..9ff344c6a 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -255,6 +255,8 @@ persist-settings: distclean
 	echo WARN=$(WARN) >> .make-settings
 	echo OPT=$(OPT) >> .make-settings
 	echo MALLOC=$(MALLOC) >> .make-settings
+	echo BUILD_TLS=$(BUILD_TLS) >> .make-settings
+	echo USE_SYSTEMD=$(USE_SYSTEMD) >> .make-settings
 	echo CFLAGS=$(CFLAGS) >> .make-settings
 	echo LDFLAGS=$(LDFLAGS) >> .make-settings
 	echo REDIS_CFLAGS=$(REDIS_CFLAGS) >> .make-settings

From 747b4004eaf49b40a25f9b4f78f57b6328f015c7 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 1 Sep 2020 21:31:37 +0300
Subject: [PATCH 0617/1098] fix README about BUILD_WITH_SYSTEMD usage (#7739)

BUILD_WITH_SYSTEMD is an internal variable. Users should use USE_SYSTEMD=yes.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 6c0dbf525..e80b81c52 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ libssl-dev on Debian/Ubuntu) and run:
 To build with systemd support, you'll need systemd development libraries (such 
 as libsystemd-dev on Debian/Ubuntu or systemd-devel on CentOS) and run:
 
-    % make BUILD_WITH_SYSTEMD=yes USE_SYSTEMD=yes
+    % make USE_SYSTEMD=yes
 
 You can run a 32 bit Redis binary using:
 

From f22f64f0db82cce666714808a9c994304f97904a Mon Sep 17 00:00:00 2001
From: Thandayuthapani <thanushpani1@gmail.com>
Date: Wed, 2 Sep 2020 18:53:49 +0530
Subject: [PATCH 0618/1098] Add masters/replicas options to redis-cli --cluster
 call command (#6491)

* Add master/slave option in --cluster call command

* Update src/redis-cli.c

* Update src/redis-cli.c

Co-authored-by: Itamar Haber <itamar@redislabs.com>
---
 src/redis-cli.c | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index aefef2c5a..f60077b72 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -126,6 +126,8 @@
 #define CLUSTER_MANAGER_CMD_FLAG_COLOR          1 << 8
 #define CLUSTER_MANAGER_CMD_FLAG_CHECK_OWNERS   1 << 9
 #define CLUSTER_MANAGER_CMD_FLAG_FIX_WITH_UNREACHABLE_MASTERS 1 << 10
+#define CLUSTER_MANAGER_CMD_FLAG_MASTERS_ONLY   1 << 11
+#define CLUSTER_MANAGER_CMD_FLAG_SLAVES_ONLY    1 << 12
 
 #define CLUSTER_MANAGER_OPT_GETFRIENDS  1 << 0
 #define CLUSTER_MANAGER_OPT_COLD        1 << 1
@@ -1614,6 +1616,12 @@ static int parseOptions(int argc, char **argv) {
             i = j;
         } else if (!strcmp(argv[i],"--cluster") && lastarg) {
             usage();
+        } else if ((!strcmp(argv[i],"--cluster-only-masters"))) {
+            config.cluster_manager_command.flags |=
+                    CLUSTER_MANAGER_CMD_FLAG_MASTERS_ONLY;
+        } else if ((!strcmp(argv[i],"--cluster-only-replicas"))) {
+            config.cluster_manager_command.flags |=
+                    CLUSTER_MANAGER_CMD_FLAG_SLAVES_ONLY;
         } else if (!strcmp(argv[i],"--cluster-replicas") && !lastarg) {
             config.cluster_manager_command.replicas = atoi(argv[++i]);
         } else if (!strcmp(argv[i],"--cluster-master-id") && !lastarg) {
@@ -2403,7 +2411,7 @@ clusterManagerCommandDef clusterManagerCommands[] = {
      "new_host:new_port existing_host:existing_port", "slave,master-id <arg>"},
     {"del-node", clusterManagerCommandDeleteNode, 2, "host:port node_id",NULL},
     {"call", clusterManagerCommandCall, -2,
-        "host:port command arg arg .. arg", NULL},
+        "host:port command arg arg .. arg", "only-masters,only-replicas"},
     {"set-timeout", clusterManagerCommandSetTimeout, 2,
      "host:port milliseconds", NULL},
     {"import", clusterManagerCommandImport, 1, "host:port",
@@ -6506,6 +6514,10 @@ static int clusterManagerCommandCall(int argc, char **argv) {
     listRewind(cluster_manager.nodes, &li);
     while ((ln = listNext(&li)) != NULL) {
         clusterManagerNode *n = ln->value;
+        if ((config.cluster_manager_command.flags & CLUSTER_MANAGER_CMD_FLAG_MASTERS_ONLY)
+              && (n->replicate != NULL)) continue;  // continue if node is slave
+        if ((config.cluster_manager_command.flags & CLUSTER_MANAGER_CMD_FLAG_SLAVES_ONLY)
+              && (n->replicate == NULL)) continue;   // continue if node is master
         if (!n->context && !clusterManagerNodeConnect(n)) continue;
         redisReply *reply = NULL;
         redisAppendCommandArgv(n->context, argc, (const char **) argv, argvlen);
@@ -8282,4 +8294,3 @@ int main(int argc, char **argv) {
         return noninteractive(argc,convertToSds(argc,argv));
     }
 }
-

From 8b0747d65734ba1128da7479f8b961e530b82916 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 2 Sep 2020 17:18:09 +0300
Subject: [PATCH 0619/1098] Print server startup messages after daemonization
 (#7743)

When redis isn't configured to have a log file, having these prints
before damonization puts them in the calling process stdout rather than
/dev/null
---
 src/server.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/server.c b/src/server.c
index d3f5c34f8..94c3040f5 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5284,6 +5284,10 @@ int main(int argc, char **argv) {
         sdsfree(options);
     }
 
+    server.supervised = redisIsSupervised(server.supervised_mode);
+    int background = server.daemonize && !server.supervised;
+    if (background) daemonize();
+
     serverLog(LL_WARNING, "oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo");
     serverLog(LL_WARNING,
         "Redis version=%s, bits=%d, commit=%s, modified=%d, pid=%d, just started",
@@ -5299,11 +5303,7 @@ int main(int argc, char **argv) {
         serverLog(LL_WARNING, "Configuration loaded");
     }
 
-    server.supervised = redisIsSupervised(server.supervised_mode);
-    int background = server.daemonize && !server.supervised;
-    if (background) daemonize();
     readOOMScoreAdj();
-
     initServer();
     if (background || server.pidfile) createPidFile();
     redisSetProcTitle(argv[0]);

From d52ce4ea1aa51457aed1d63a5bf784f94b2768c3 Mon Sep 17 00:00:00 2001
From: Pierre Jambet <pierre.jambet@gmail.com>
Date: Wed, 2 Sep 2020 16:27:48 -0400
Subject: [PATCH 0620/1098] Fix error message for the DEBUG ZIPLIST command
 (#7745)

DEBUG ZIPLIST <key> currently returns the following error string if the
key is not a ziplist: "ERR Not an sds encoded string.". This looks like
an accidental copy/paste error from the error returned in the else if
branch above where this string is returned if the key is not an sds
string. The command was added in
ac61f9062583d67dd43f7d698824464d1e30d84b and looking at the commit,
nothing indicates that it is not an accidental typo.

The error string now returns a correct error: "Not a ziplist encoded
object", which accurately describes the error.
---
 src/debug.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/debug.c b/src/debug.c
index b05dc344e..373938e6e 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -588,7 +588,7 @@ NULL
                 == NULL) return;
 
         if (o->encoding != OBJ_ENCODING_ZIPLIST) {
-            addReplyError(c,"Not an sds encoded string.");
+            addReplyError(c,"Not a ziplist encoded object.");
         } else {
             ziplistRepr(o->ptr);
             addReplyStatus(c,"Ziplist structure printed on stdout");

From 9ef8d2f67155d6aa69b8344fc0b63e6dab8ce9f8 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 3 Sep 2020 08:47:29 +0300
Subject: [PATCH 0621/1098] Run active defrag while blocked / loading (#7726)

During long running scripts or loading RDB/AOF, we may need to do some
defragging. Since processEventsWhileBlocked is called periodically at
unknown intervals, and many cron jobs either depend on run_with_period
(including active defrag), or rely on being called at server.hz rate
(i.e. active defrag knows ho much time to run by looking at server.hz),
the whileBlockedCron may have to run a loop triggering the cron jobs in it
(currently only active defrag) several times.

Other changes:
- Adding a test for defrag during aof loading.
- Changing key-load-delay config to take negative values for fractions
  of a microsecond sleep
---
 src/aof.c                    |  3 +-
 src/config.c                 |  4 +--
 src/debug.c                  |  9 ++++++
 src/networking.c             |  3 ++
 src/rdb.c                    |  8 +++--
 src/scripting.c              |  2 ++
 src/server.c                 | 61 ++++++++++++++++++++++++++++++------
 src/server.h                 | 11 +++++--
 tests/support/util.tcl       | 11 +++++++
 tests/unit/memefficiency.tcl | 57 ++++++++++++++++++++++++++-------
 10 files changed, 139 insertions(+), 30 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index 3f33bda3a..07f9f8d08 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -773,7 +773,6 @@ int loadAppendOnlyFile(char *filename) {
         if (!(loops++ % 1000)) {
             loadingProgress(ftello(fp));
             processEventsWhileBlocked();
-            loadingCron();
             processModuleLoadingProgressEvent(1);
         }
 
@@ -859,7 +858,7 @@ int loadAppendOnlyFile(char *filename) {
         fakeClient->cmd = NULL;
         if (server.aof_load_truncated) valid_up_to = ftello(fp);
         if (server.key_load_delay)
-            usleep(server.key_load_delay);
+            debugDelay(server.key_load_delay);
     }
 
     /* This point can only be reached when EOF is reached without errors.
diff --git a/src/config.c b/src/config.c
index 5b73d9e28..0e1aa01e9 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2319,8 +2319,8 @@ standardConfig configs[] = {
     createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_timeout, 60, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_ping_slave_period, 10, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, 0, INTEGER_CONFIG, NULL, NULL),
-    createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL),
-    createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, 0, INTEGER_CONFIG, NULL, NULL),
+    createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL),
+    createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.key_load_delay, 0, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, 1, INTEGER_CONFIG, NULL, NULL), /* From 1 to 10. */
     createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG, NULL, updateHZ),
     createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves),
diff --git a/src/debug.c b/src/debug.c
index 373938e6e..5dd9ec849 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1804,3 +1804,12 @@ void disableWatchdog(void) {
     sigaction(SIGALRM, &act, NULL);
     server.watchdog_period = 0;
 }
+
+/* Positive input is sleep time in microseconds. Negative input is fractions
+ * of microseconds, i.e. -10 means 100 nanoseconds. */
+void debugDelay(int usec) {
+    /* Since even the shortest sleep results in context switch and system call,
+     * the way we achive short sleeps is by statistically sleeping less often. */
+    if (usec < 0) usec = (rand() % -usec) == 0 ? 1: 0;
+    if (usec) usleep(usec);
+}
diff --git a/src/networking.c b/src/networking.c
index 4a58cec40..71e30cfa1 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2920,6 +2920,9 @@ void processEventsWhileBlocked(void) {
         long long events = server.events_processed_while_blocked - startval;
         if (!events) break;
     }
+
+    whileBlockedCron();
+
     ProcessingEventsWhileBlocked = 0;
 }
 
diff --git a/src/rdb.c b/src/rdb.c
index eab77fb9e..d4e44be5d 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1084,7 +1084,7 @@ int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime) {
 
     /* Delay return if required (for testing) */
     if (server.rdb_key_save_delay)
-        usleep(server.rdb_key_save_delay);
+        debugDelay(server.rdb_key_save_delay);
 
     return 1;
 }
@@ -1997,6 +1997,7 @@ void startLoading(size_t size, int rdbflags) {
     server.loading_start_time = time(NULL);
     server.loading_loaded_bytes = 0;
     server.loading_total_bytes = size;
+    blockingOperationStarts();
 
     /* Fire the loading modules start event. */
     int subevent;
@@ -2030,6 +2031,7 @@ void loadingProgress(off_t pos) {
 /* Loading finished */
 void stopLoading(int success) {
     server.loading = 0;
+    blockingOperationEnds();
     rdbFileBeingLoaded = NULL;
 
     /* Fire the loading modules end event. */
@@ -2073,7 +2075,6 @@ void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) {
             replicationSendNewlineToMaster();
         loadingProgress(r->processed_bytes);
         processEventsWhileBlocked();
-        loadingCron();
         processModuleLoadingProgressEvent(0);
     }
 }
@@ -2336,7 +2337,8 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
 
         /* Loading the database more slowly is useful in order to test
          * certain edge cases. */
-        if (server.key_load_delay) usleep(server.key_load_delay);
+        if (server.key_load_delay)
+            debugDelay(server.key_load_delay);
 
         /* Reset the state that is key-specified and is populated by
          * opcodes before the key, so that we start from scratch again. */
diff --git a/src/scripting.c b/src/scripting.c
index bccbcf637..be44ec67f 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -1424,6 +1424,7 @@ void luaMaskCountHook(lua_State *lua, lua_Debug *ar) {
             "Script SHA1 is: %s",
             elapsed, server.lua_cur_script);
         server.lua_timedout = 1;
+        blockingOperationStarts();
         /* Once the script timeouts we reenter the event loop to permit others
          * to call SCRIPT KILL or SHUTDOWN NOSAVE if needed. For this reason
          * we need to mask the client executing the script from the event loop.
@@ -1575,6 +1576,7 @@ void evalGenericCommand(client *c, int evalsha) {
     if (delhook) lua_sethook(lua,NULL,0,0); /* Disable hook */
     if (server.lua_timedout) {
         server.lua_timedout = 0;
+        blockingOperationEnds();
         /* Restore the client that was protected when the script timeout
          * was detected. */
         unprotectClient(c);
diff --git a/src/server.c b/src/server.c
index 94c3040f5..5e9c85585 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2109,22 +2109,62 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
     return 1000/server.hz;
 }
 
-/* This function fill in the role of serverCron during RDB or AOF loading.
+
+void blockingOperationStarts() {
+    updateCachedTime(0);
+    server.blocked_last_cron = server.mstime;
+}
+
+void blockingOperationEnds() {
+    server.blocked_last_cron = 0;
+}
+
+/* This function fill in the role of serverCron during RDB or AOF loading, and
+ * also during blocked scripts.
  * It attempts to do its duties at a similar rate as the configured server.hz,
  * and updates cronloops variable so that similarly to serverCron, the
  * run_with_period can be used. */
-void loadingCron() {
-    long long now = server.ustime;
-    static long long next_event = 0;
-    if (now >= next_event) {
-        cronUpdateMemoryStats();
-        
+void whileBlockedCron() {
+    /* Here we may want to perform some cron jobs (normally done server.hz times
+     * per second). */
+
+    /* Since this function depends on a call to blockingOperationStarts, let's
+     * make sure it was done. */
+    serverAssert(server.blocked_last_cron);
+
+    /* In case we where called too soon, leave right away. This way one time
+     * jobs after the loop below don't need an if. and we don't bother to start
+     * latency monitor if this function is called too often. */
+    if (server.blocked_last_cron >= server.mstime)
+        return;
+
+    mstime_t latency;
+    latencyStartMonitor(latency);
+
+    /* In some cases we may be called with big intervals, so we may need to do
+     * extra work here. This is because some of the functions in serverCron rely
+     * on the fact that it is performed every 10 ms or so. For instance, if
+     * activeDefragCycle needs to utilize 25% cpu, it will utilize 2.5ms, so we
+     * need to call it multiple times. */
+    long hz_ms = 1000/server.hz;
+    while (server.blocked_last_cron < server.mstime) {
+
+        /* Defrag keys gradually. */
+        activeDefragCycle();
+
+        server.blocked_last_cron += hz_ms;
         /* Increment cronloop so that run_with_period works. */
         server.cronloops++;
-
-        /* Decide when the next event should fire. */
-        next_event = now + 1000000 / server.hz;
     }
+
+    /* Other cron jobs do not need to be done in a loop. No need to check
+     * server.blocked_last_cron since we have an early exit at the top. */
+
+    /* Update memory stats during loading (excluding blocked scripts) */
+    if (server.loading) cronUpdateMemoryStats();
+
+    latencyEndMonitor(latency);
+    latencyAddSampleIfNeeded("while-blocked-cron",latency);
 }
 
 extern int ProcessingEventsWhileBlocked;
@@ -2830,6 +2870,7 @@ void resetServerStats(void) {
     server.stat_net_output_bytes = 0;
     server.stat_unexpected_error_replies = 0;
     server.aof_delayed_fsync = 0;
+    server.blocked_last_cron = 0;
 }
 
 void initServer(void) {
diff --git a/src/server.h b/src/server.h
index 53b4fcce3..6dac914c6 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1265,9 +1265,11 @@ struct redisServer {
     char *rdb_pipe_buff;            /* In diskless replication, this buffer holds data */
     int rdb_pipe_bufflen;           /* that was read from the the rdb pipe. */
     int rdb_key_save_delay;         /* Delay in microseconds between keys while
-                                     * writing the RDB. (for testings) */
+                                     * writing the RDB. (for testings). negative
+                                     * value means fractions of microsecons (on average). */
     int key_load_delay;             /* Delay in microseconds between keys while
-                                     * loading aof or rdb. (for testings) */
+                                     * loading aof or rdb. (for testings). negative
+                                     * value means fractions of microsecons (on average). */
     /* Pipe and data structures for child -> parent info sharing. */
     int child_info_pipe[2];         /* Pipe used to write the child_info_data. */
     struct {
@@ -1393,6 +1395,7 @@ struct redisServer {
     int daylight_active;        /* Currently in daylight saving time. */
     mstime_t mstime;            /* 'unixtime' in milliseconds. */
     ustime_t ustime;            /* 'unixtime' in microseconds. */
+    long long blocked_last_cron; /* Indicate the mstime of the last time we did cron jobs from a blocking operation */
     /* Pubsub */
     dict *pubsub_channels;  /* Map channels to list of subscribed clients */
     list *pubsub_patterns;  /* A list of pubsub_patterns */
@@ -1698,6 +1701,9 @@ void pauseClients(mstime_t duration);
 int clientsArePaused(void);
 void processEventsWhileBlocked(void);
 void loadingCron(void);
+void whileBlockedCron();
+void blockingOperationStarts();
+void blockingOperationEnds();
 int handleClientsWithPendingWrites(void);
 int handleClientsWithPendingWritesUsingThreads(void);
 int handleClientsWithPendingReadsUsingThreads(void);
@@ -2464,6 +2470,7 @@ int memtest_preserving_test(unsigned long *m, size_t bytes, int passes);
 void mixDigest(unsigned char *digest, void *ptr, size_t len);
 void xorDigest(unsigned char *digest, void *ptr, size_t len);
 int populateCommandTableParseFlags(struct redisCommand *c, char *strflags);
+void debugDelay(int usec);
 
 /* TLS stuff */
 void tlsInit(void);
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index 8340ad207..14b17f4e7 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -455,3 +455,14 @@ proc start_bg_complex_data {host port db ops} {
 proc stop_bg_complex_data {handle} {
     catch {exec /bin/kill -9 $handle}
 }
+
+proc populate {num prefix size} {
+    set rd [redis_deferring_client]
+    for {set j 0} {$j < $num} {incr j} {
+        $rd set $prefix$j [string repeat A $size]
+    }
+    for {set j 0} {$j < $num} {incr j} {
+        $rd read
+    }
+    $rd close
+}
diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl
index 390bad192..fe4e44d3e 100644
--- a/tests/unit/memefficiency.tcl
+++ b/tests/unit/memefficiency.tcl
@@ -37,10 +37,9 @@ start_server {tags {"memefficiency"}} {
 }
 
 run_solo {defrag} {
-start_server {tags {"defrag"}} {
+start_server {tags {"defrag"} overrides {appendonly yes auto-aof-rewrite-percentage 0 save ""}} {
     if {[string match {*jemalloc*} [s mem_allocator]]} {
         test "Active defrag" {
-            r config set save "" ;# prevent bgsave from interfereing with save below
             r config set hz 100
             r config set activedefrag no
             r config set active-defrag-threshold-lower 5
@@ -49,9 +48,9 @@ start_server {tags {"defrag"}} {
             r config set active-defrag-ignore-bytes 2mb
             r config set maxmemory 100mb
             r config set maxmemory-policy allkeys-lru
-            r debug populate 700000 asdf1 150
-            r debug populate 170000 asdf2 300
-            r ping ;# trigger eviction following the previous population
+
+            populate 700000 asdf1 150
+            populate 170000 asdf2 300
             after 120 ;# serverCron only updates the info once in 100ms
             set frag [s allocator_frag_ratio]
             if {$::verbose} {
@@ -107,19 +106,57 @@ start_server {tags {"defrag"}} {
                 # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75,
                 # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher
                 assert {$max_latency <= 30}
-            } else {
-                set _ ""
             }
             # verify the data isn't corrupted or changed
             set newdigest [r debug digest]
             assert {$digest eq $newdigest}
             r save ;# saving an rdb iterates over all the data / pointers
-        } {OK}
+
+            # if defrag is supported, test AOF loading too
+            if {[r config get activedefrag] eq "activedefrag yes"} {
+                # reset stats and load the AOF file
+                r config resetstat
+                r config set key-load-delay -50 ;# sleep on average 1/50 usec
+                r debug loadaof
+                r config set activedefrag no
+                # measure hits and misses right after aof loading
+                set misses [s active_defrag_misses]
+                set hits [s active_defrag_hits]
+
+                after 120 ;# serverCron only updates the info once in 100ms
+                set frag [s allocator_frag_ratio]
+                set max_latency 0
+                foreach event [r latency latest] {
+                    lassign $event eventname time latency max
+                    if {$eventname == "loading-cron"} {
+                        set max_latency $max
+                    }
+                }
+                if {$::verbose} {
+                    puts "AOF loading:"
+                    puts "frag $frag"
+                    puts "hits: $hits"
+                    puts "misses: $misses"
+                    puts "max latency $max_latency"
+                    puts [r latency latest]
+                    puts [r latency history loading-cron]
+                }
+                # make sure we had defrag hits during AOF loading
+                assert {$hits > 100000}
+                # make sure the defragger did enough work to keep the fragmentation low during loading.
+                # we cannot check that it went all the way down, since we don't wait for full defrag cycle to complete.
+                assert {$frag < 1.4}
+                # since the AOF contains simple (fast) SET commands (and the cron during loading runs every 1000 commands),
+                # it'll still not block the loading for long periods of time.
+                assert {$max_latency <= 30}
+            }
+        }
+        r config set appendonly no
+        r config set key-load-delay 0
 
         test "Active defrag big keys" {
             r flushdb
             r config resetstat
-            r config set save "" ;# prevent bgsave from interfereing with save below
             r config set hz 100
             r config set activedefrag no
             r config set active-defrag-max-scan-fields 1000
@@ -247,7 +284,6 @@ start_server {tags {"defrag"}} {
         test "Active defrag big list" {
             r flushdb
             r config resetstat
-            r config set save "" ;# prevent bgsave from interfereing with save below
             r config set hz 100
             r config set activedefrag no
             r config set active-defrag-max-scan-fields 1000
@@ -354,7 +390,6 @@ start_server {tags {"defrag"}} {
             start_server {tags {"defrag"}} {
                 r flushdb
                 r config resetstat
-                r config set save "" ;# prevent bgsave from interfereing with save below
                 r config set hz 100
                 r config set activedefrag no
                 r config set active-defrag-max-scan-fields 1000

From 12f798dc18976839f3dfcb835433b6db63f8922f Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Thu, 3 Sep 2020 15:31:18 +0800
Subject: [PATCH 0622/1098] fix wrong comments in redis.conf, change default
 always-show-logo (#5695)

1. default value of always-show-logo was not consistent with the default in the code
2. comment about cluster-replica-no-failover is wrong since we can only do manually failover upon replicas
3. improve description about always-show-logo
---
 redis.conf | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/redis.conf b/redis.conf
index 35ee752e1..8f26a8f3e 100644
--- a/redis.conf
+++ b/redis.conf
@@ -285,12 +285,13 @@ logfile ""
 databases 16
 
 # By default Redis shows an ASCII art logo only when started to log to the
-# standard output and if the standard output is a TTY. Basically this means
-# that normally a logo is displayed only in interactive sessions.
+# standard output and if the standard output is a TTY and syslog logging is
+# disabled. Basically this means that normally a logo is displayed only in
+# interactive sessions.
 #
 # However it is possible to force the pre-4.0 behavior and always show a
 # ASCII art logo in startup logs by setting the following option to yes.
-always-show-logo yes
+always-show-logo no
 
 ################################ SNAPSHOTTING  ################################
 #
@@ -1322,7 +1323,7 @@ lua-time-limit 5000
 # cluster-require-full-coverage yes
 
 # This option, when set to yes, prevents replicas from trying to failover its
-# master during master failures. However the master can still perform a
+# master during master failures. However the replica can still perform a
 # manual failover, if forced to do so.
 #
 # This is useful in different scenarios, especially in the case of multiple

From 58e5feb3f49c50b9c18f38fd8f6cad2317c02265 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 3 Sep 2020 18:15:48 +0300
Subject: [PATCH 0623/1098] redis-cli: fix writeConn() buffer handling. (#7749)

Fix issues with writeConn() which resulted with corruption of the stream by leaving an extra byte in the buffer. The trigger for this is partial writes or write errors which were not experienced on Linux but reported on macOS.
---
 src/redis-cli.c | 43 +++++++++++++++++++++++++++++++++++++------
 1 file changed, 37 insertions(+), 6 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index f60077b72..944ba44ec 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -6905,21 +6905,52 @@ static ssize_t writeConn(redisContext *c, const char *buf, size_t buf_len)
 {
     int done = 0;
 
+    /* Append data to buffer which is *usually* expected to be empty
+     * but we don't assume that, and write.
+     */
     c->obuf = sdscatlen(c->obuf, buf, buf_len);
     if (redisBufferWrite(c, &done) == REDIS_ERR) {
-        sdsrange(c->obuf, 0, -(buf_len+1));
         if (!(c->flags & REDIS_BLOCK))
             errno = EAGAIN;
+
+        /* On error, we assume nothing was written and we roll back the
+         * buffer to its original state.
+         */
+        if (sdslen(c->obuf) > buf_len)
+            sdsrange(c->obuf, 0, -(buf_len+1));
+        else
+            sdsclear(c->obuf);
+
         return -1;
     }
 
-    size_t left = sdslen(c->obuf);
-    sdsclear(c->obuf);
-    if (!done) {
-        return buf_len - left;
+    /* If we're done, free up everything. We may have written more than
+     * buf_len (if c->obuf was not initially empty) but we don't have to
+     * tell.
+     */
+    if (done) {
+        sdsclear(c->obuf);
+        return buf_len;
     }
 
-    return buf_len;
+    /* Write was successful but we have some leftovers which we should
+     * remove from the buffer.
+     *
+     * Do we still have data that was there prior to our buf? If so,
+     * restore buffer to it's original state and report no new data was
+     * writen.
+     */
+    if (sdslen(c->obuf) > buf_len) {
+        sdsrange(c->obuf, 0, -(buf_len+1));
+        return 0;
+    }
+
+    /* At this point we're sure no prior data is left. We flush the buffer
+     * and report how much we've written.
+     */
+    size_t left = sdslen(c->obuf);
+    sdsclear(c->obuf);
+    return buf_len - left;
 }
 
 /* Read raw bytes through a redisContext. The read operation is not greedy

From 9d527d076b17851b87bc95aa34cca8fa5a91d41b Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 31 Aug 2020 10:23:09 +0300
Subject: [PATCH 0624/1098] test infra - write test name to logfile

---
 tests/support/server.tcl | 14 ++++++++++++++
 tests/support/test.tcl   | 13 +++++++++++++
 tests/support/util.tcl   |  8 ++++++++
 3 files changed, 35 insertions(+)

diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index 0afe89f7c..14e59e55c 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -301,6 +301,13 @@ proc start_server {options {code undefined}} {
     set stdout [format "%s/%s" [dict get $config "dir"] "stdout"]
     set stderr [format "%s/%s" [dict get $config "dir"] "stderr"]
 
+    # if we're inside a test, write the test name to the server log file
+    if {[info exists ::cur_test]} {
+        set fd [open $stdout "a+"]
+        puts $fd "### Starting server for test $::cur_test"
+        close $fd
+    }
+
     # We need a loop here to retry with different ports.
     set server_started 0
     while {$server_started == 0} {
@@ -443,6 +450,13 @@ proc restart_server {level wait_ready} {
     set stderr [dict get $srv "stderr"]
     set config_file [dict get $srv "config_file"]
 
+    # if we're inside a test, write the test name to the server log file
+    if {[info exists ::cur_test]} {
+        set fd [open $stdout "a+"]
+        puts $fd "### Restarting server for test $::cur_test"
+        close $fd
+    }
+
     set prev_ready_count [exec grep -i "Ready to accept" | wc -l < $stdout]
 
     set pid [spawn_server $config_file $stdout $stderr]
diff --git a/tests/support/test.tcl b/tests/support/test.tcl
index a5573f583..d266eba41 100644
--- a/tests/support/test.tcl
+++ b/tests/support/test.tcl
@@ -143,6 +143,18 @@ proc test {name code {okpattern undefined}} {
     set details {}
     lappend details "$name in $::curfile"
 
+    # set a cur_test global to be logged into new servers that are spown
+    # and log the test name in all existing servers
+    set ::cur_test "$name in $::curfile"
+    if {!$::external} {
+        foreach srv $::servers {
+            set stdout [dict get $srv stdout]
+            set fd [open $stdout "a+"]
+            puts $fd "### Starting test $::cur_test"
+            close $fd
+        }
+    }
+
     send_data_packet $::test_server_fd testing $name
 
     if {[catch {set retval [uplevel 1 $code]} error]} {
@@ -183,4 +195,5 @@ proc test {name code {okpattern undefined}} {
             send_data_packet $::test_server_fd err "Detected a memory leak in test '$name': $output"
         }
     }
+    unset ::cur_test
 }
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index 14b17f4e7..bf590d402 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -138,6 +138,14 @@ proc wait_for_log_messages {srv_idx patterns from_line maxtries delay} {
     }
 }
 
+# write line to server log file
+proc write_log_line {srv_idx msg} {
+    set logfile [srv $srv_idx stdout]
+    set fd [open $logfile "a+"]
+    puts $fd "### $msg"
+    close $fd
+}
+
 # Random integer between 0 and max (excluded).
 proc randomInt {max} {
     expr {int(rand()*$max)}

From e3e69c25fd05b608f5ea8d612bc0e377922a6115 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 31 Aug 2020 10:44:43 +0300
Subject: [PATCH 0625/1098] test infra - reduce disk space usage

this is important when running a test with --loop
---
 tests/integration/rdb.tcl        |  8 ++++----
 tests/support/server.tcl         | 30 ++++++++++++++++++++++++++----
 tests/unit/moduleapi/testrdb.tcl |  6 +++---
 3 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl
index 9cd970350..58dc6c968 100644
--- a/tests/integration/rdb.tcl
+++ b/tests/integration/rdb.tcl
@@ -25,7 +25,7 @@ start_server [list overrides [list "dir" $server_path "dbfilename" "encodings.rd
 
 set server_path [tmpdir "server.rdb-startup-test"]
 
-start_server [list overrides [list "dir" $server_path]] {
+start_server [list overrides [list "dir" $server_path] keep_persistence true] {
     test {Server started empty with non-existing RDB file} {
         r debug digest
     } {0000000000000000000000000000000000000000}
@@ -33,13 +33,13 @@ start_server [list overrides [list "dir" $server_path]] {
     r save
 }
 
-start_server [list overrides [list "dir" $server_path]] {
+start_server [list overrides [list "dir" $server_path] keep_persistence true] {
     test {Server started empty with empty RDB file} {
         r debug digest
     } {0000000000000000000000000000000000000000}
 }
 
-start_server [list overrides [list "dir" $server_path]] {
+start_server [list overrides [list "dir" $server_path] keep_persistence true] {
     test {Test RDB stream encoding} {
         for {set j 0} {$j < 1000} {incr j} {
             if {rand() < 0.9} {
@@ -64,7 +64,7 @@ set defaults {}
 proc start_server_and_kill_it {overrides code} {
     upvar defaults defaults srv srv server_path server_path
     set config [concat $defaults $overrides]
-    set srv [start_server [list overrides $config]]
+    set srv [start_server [list overrides $config keep_persistence true]]
     uplevel 1 $code
     kill_server $srv
 }
diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index 14e59e55c..6775b125a 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -31,6 +31,16 @@ proc check_valgrind_errors stderr {
     }
 }
 
+proc clean_persistence config {
+    # we may wanna keep the logs for later, but let's clean the persistence
+    # files right away, since they can accumulate and take up a lot of space
+    set config [dict get $config "config"]
+    set rdb [format "%s/%s" [dict get $config "dir"] "dump.rdb"]
+    set aof [format "%s/%s" [dict get $config "dir"] "appendonly.aof"]
+    catch {exec rm -rf $rdb}
+    catch {exec rm -rf $aof}
+}
+
 proc kill_server config {
     # nothing to kill when running against external server
     if {$::external} return
@@ -238,19 +248,27 @@ proc start_server {options {code undefined}} {
     set baseconfig "default.conf"
     set overrides {}
     set tags {}
+    set keep_persistence false
 
     # parse options
     foreach {option value} $options {
         switch $option {
             "config" {
-                set baseconfig $value }
+                set baseconfig $value
+            }
             "overrides" {
-                set overrides $value }
+                set overrides $value
+            }
             "tags" {
                 set tags $value
-                set ::tags [concat $::tags $value] }
+                set ::tags [concat $::tags $value]
+            }
+            "keep_persistence" {
+                set keep_persistence $value
+            }
             default {
-                error "Unknown option $option" }
+                error "Unknown option $option"
+            }
         }
     }
 
@@ -436,6 +454,10 @@ proc start_server {options {code undefined}} {
 
         set ::tags [lrange $::tags 0 end-[llength $tags]]
         kill_server $srv
+        if {!$keep_persistence} {
+            clean_persistence $srv
+        }
+        set _ ""
     } else {
         set ::tags [lrange $::tags 0 end-[llength $tags]]
         set _ $srv
diff --git a/tests/unit/moduleapi/testrdb.tcl b/tests/unit/moduleapi/testrdb.tcl
index 0d26b0a26..5ba102284 100644
--- a/tests/unit/moduleapi/testrdb.tcl
+++ b/tests/unit/moduleapi/testrdb.tcl
@@ -12,7 +12,7 @@ tags "modules" {
 
     test {modules global are lost without aux} {
         set server_path [tmpdir "server.module-testrdb"]
-        start_server [list overrides [list loadmodule "$testmodule" "dir" $server_path]] {
+        start_server [list overrides [list loadmodule "$testmodule" "dir" $server_path] keep_persistence true] {
             r testrdb.set.before global1
             assert_equal "global1" [r testrdb.get.before]
         }
@@ -23,7 +23,7 @@ tags "modules" {
 
     test {modules are able to persist globals before and after} {
         set server_path [tmpdir "server.module-testrdb"]
-        start_server [list overrides [list loadmodule "$testmodule 2" "dir" $server_path]] {
+        start_server [list overrides [list loadmodule "$testmodule 2" "dir" $server_path] keep_persistence true] {
             r testrdb.set.before global1
             r testrdb.set.after global2
             assert_equal "global1" [r testrdb.get.before]
@@ -38,7 +38,7 @@ tags "modules" {
 
     test {modules are able to persist globals just after} {
         set server_path [tmpdir "server.module-testrdb"]
-        start_server [list overrides [list loadmodule "$testmodule 1" "dir" $server_path]] {
+        start_server [list overrides [list loadmodule "$testmodule 1" "dir" $server_path] keep_persistence true] {
             r testrdb.set.after global2
             assert_equal "global2" [r testrdb.get.after]
         }

From 677d14c2137ab50fa25c8163d20b14bc563261c7 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 31 Aug 2020 11:05:30 +0300
Subject: [PATCH 0626/1098] test infra - improve test skipping ability

- skip full units
- skip a single test (not just a list of tests)
- when skipping tag, skip spinning up servers, not just the tests
- skip tags when running against an external server too
- allow using multiple tags (split them)
---
 tests/support/server.tcl | 69 ++++++++++++++++++++++++++++------------
 tests/support/test.tcl   | 11 +------
 tests/test_helper.tcl    | 47 +++++++++++++++++++++++----
 3 files changed, 91 insertions(+), 36 deletions(-)

diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index 6775b125a..8ab8cf66a 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -160,7 +160,19 @@ proc server_is_up {host port retrynum} {
 
 # doesn't really belong here, but highly coupled to code in start_server
 proc tags {tags code} {
+    # If we 'tags' contain multiple tags, quoted and seperated by spaces,
+    # we want to get rid of the quotes in order to have a proper list
+    set tags [string map { \" "" } $tags]
     set ::tags [concat $::tags $tags]
+    # We skip unwanted tags
+    foreach tag $::denytags {
+        if {[lsearch $::tags $tag] >= 0} {
+            incr ::num_aborted
+            send_data_packet $::test_server_fd ignore "Tag: $tag"
+            set ::tags [lrange $::tags 0 end-[llength $tags]]
+            return
+        }
+    }
     uplevel 1 $code
     set ::tags [lrange $::tags 0 end-[llength $tags]]
 }
@@ -226,24 +238,6 @@ proc wait_server_started {config_file stdout pid} {
 }
 
 proc start_server {options {code undefined}} {
-    # If we are running against an external server, we just push the
-    # host/port pair in the stack the first time
-    if {$::external} {
-        if {[llength $::servers] == 0} {
-            set srv {}
-            dict set srv "host" $::host
-            dict set srv "port" $::port
-            set client [redis $::host $::port 0 $::tls]
-            dict set srv "client" $client
-            $client select 9
-
-            # append the server to the stack
-            lappend ::servers $srv
-        }
-        uplevel 1 $code
-        return
-    }
-
     # setup defaults
     set baseconfig "default.conf"
     set overrides {}
@@ -260,8 +254,10 @@ proc start_server {options {code undefined}} {
                 set overrides $value
             }
             "tags" {
-                set tags $value
-                set ::tags [concat $::tags $value]
+                # If we 'tags' contain multiple tags, quoted and seperated by spaces,
+                # we want to get rid of the quotes in order to have a proper list
+                set tags [string map { \" "" } $value]
+                set ::tags [concat $::tags $tags]
             }
             "keep_persistence" {
                 set keep_persistence $value
@@ -272,6 +268,39 @@ proc start_server {options {code undefined}} {
         }
     }
 
+    # We skip unwanted tags
+    foreach tag $::denytags {
+        if {[lsearch $::tags $tag] >= 0} {
+            incr ::num_aborted
+            send_data_packet $::test_server_fd ignore "Tag: $tag"
+            set ::tags [lrange $::tags 0 end-[llength $tags]]
+            return
+        }
+    }
+
+    # If we are running against an external server, we just push the
+    # host/port pair in the stack the first time
+    if {$::external} {
+        if {[llength $::servers] == 0} {
+            set srv {}
+            dict set srv "host" $::host
+            dict set srv "port" $::port
+            set client [redis $::host $::port 0 $::tls]
+            dict set srv "client" $client
+            $client select 9
+
+            set config {}
+            dict set config "port" $::port
+            dict set srv "config" $config
+
+            # append the server to the stack
+            lappend ::servers $srv
+        }
+        uplevel 1 $code
+        set ::tags [lrange $::tags 0 end-[llength $tags]]
+        return
+    }
+
     set data [split [exec cat "tests/assets/$baseconfig"] "\n"]
     set config {}
     if {$::tls} {
diff --git a/tests/support/test.tcl b/tests/support/test.tcl
index d266eba41..773461abb 100644
--- a/tests/support/test.tcl
+++ b/tests/support/test.tcl
@@ -99,16 +99,7 @@ proc wait_for_condition {maxtries delay e _else_ elsescript} {
     }
 }
 
-proc test {name code {okpattern undefined}} {
-    # abort if tagged with a tag to deny
-    foreach tag $::denytags {
-        if {[lsearch $::tags $tag] >= 0} {
-            incr ::num_aborted
-            send_data_packet $::test_server_fd ignore $name
-            return
-        }
-    }
-
+proc test {name code {okpattern undefined} {options undefined}} {
     # abort if test name in skiptests
     if {[lsearch $::skiptests $name] >= 0} {
         incr ::num_skipped
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index d0f962762..4a470ec30 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -85,6 +85,7 @@ set ::verbose 0
 set ::quiet 0
 set ::denytags {}
 set ::skiptests {}
+set ::skipunits {}
 set ::allowtags {}
 set ::only_tests {}
 set ::single_tests {}
@@ -423,6 +424,12 @@ proc lpop {listVar {count 1}} {
     set ele
 }
 
+proc lremove {listVar value} {
+    upvar 1 $listVar var
+    set idx [lsearch -exact $var $value]
+    set var [lreplace $var $idx $idx]
+}
+
 # A new client is idle. Remove it from the list of active clients and
 # if there are still test units to run, launch them.
 proc signal_idle_client fd {
@@ -521,11 +528,13 @@ proc print_help_screen {} {
         "--list-tests       List all the available test units."
         "--only <test>      Just execute the specified test by test name. this option can be repeated."
         "--skip-till <unit> Skip all units until (and including) the specified one."
+        "--skipunit <unit>  Skip one unit."
         "--clients <num>    Number of test clients (default 16)."
         "--timeout <sec>    Test timeout in seconds (default 10 min)."
         "--force-failure    Force the execution of a test that always fails."
         "--config <k> <v>   Extra config file argument."
         "--skipfile <file>  Name of a file containing test names that should be skipped (one per line)."
+        "--skiptest <name>  Name of a file containing test names that should be skipped (one per line)."
         "--dont-clean       Don't delete redis log files after the run."
         "--stop             Blocks once the first test fails."
         "--loop             Execute the specified set of tests forever."
@@ -563,6 +572,9 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
         set file_data [read $fp]
         close $fp
         set ::skiptests [split $file_data "\n"]
+    } elseif {$opt eq {--skiptest}} {
+        lappend ::skiptests $arg
+        incr j
     } elseif {$opt eq {--valgrind}} {
         set ::valgrind 1
     } elseif {$opt eq {--stack-logging}} {
@@ -601,6 +613,9 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
     } elseif {$opt eq {--only}} {
         lappend ::only_tests $arg
         incr j
+    } elseif {$opt eq {--skipunit}} {
+        lappend ::skipunits $arg
+        incr j
     } elseif {$opt eq {--skip-till}} {
         set ::skip_till $arg
         incr j
@@ -638,13 +653,23 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
     }
 }
 
-# If --skil-till option was given, we populate the list of single tests
+set filtered_tests {}
+
+# Set the filtered tests to be the short list (single_tests) if exists.
+# Otherwise, we start filtering all_tests
+if {[llength $::single_tests] > 0} {
+    set filtered_tests $::single_tests
+} else {
+    set filtered_tests $::all_tests
+}
+
+# If --skip-till option was given, we populate the list of single tests
 # to run with everything *after* the specified unit.
 if {$::skip_till != ""} {
     set skipping 1
     foreach t $::all_tests {
-        if {$skipping == 0} {
-            lappend ::single_tests $t
+        if {$skipping == 1} {
+            lremove filtered_tests $t
         }
         if {$t == $::skip_till} {
             set skipping 0
@@ -656,10 +681,20 @@ if {$::skip_till != ""} {
     }
 }
 
+# If --skipunits option was given, we populate the list of single tests
+# to run with everything *not* in the skipunits list.
+if {[llength $::skipunits] > 0} {
+    foreach t $::all_tests {
+        if {[lsearch $::skipunits $t] != -1} {
+            lremove filtered_tests $t
+        }
+    }
+}
+
 # Override the list of tests with the specific tests we want to run
-# in case there was some filter, that is --single or --skip-till options.
-if {[llength $::single_tests] > 0} {
-    set ::all_tests $::single_tests
+# in case there was some filter, that is --single, -skipunit or --skip-till options.
+if {[llength $filtered_tests] < [llength $::all_tests]} {
+    set ::all_tests $filtered_tests
 }
 
 proc attach_to_replication_stream {} {

From b65e5aca86b9c2d24b96abc8414a45f9907b6f7d Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 31 Aug 2020 11:16:06 +0300
Subject: [PATCH 0627/1098] test infra - flushall between tests in external
 mode

---
 tests/support/server.tcl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index 8ab8cf66a..9894e6f2a 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -296,6 +296,7 @@ proc start_server {options {code undefined}} {
             # append the server to the stack
             lappend ::servers $srv
         }
+        r flushall
         uplevel 1 $code
         set ::tags [lrange $::tags 0 end-[llength $tags]]
         return

From 1b7ba44e7917082ac6d5523666d3b4ab210dfbad Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 31 Aug 2020 11:20:02 +0300
Subject: [PATCH 0628/1098] test infra - wait_done_loading

reduce code duplication in aof.tcl.
move creation of clients into the test so that it can be skipped
---
 tests/integration/aof.tcl | 44 +++++++--------------------------------
 tests/support/util.tcl    |  8 +++++++
 2 files changed, 16 insertions(+), 36 deletions(-)

diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl
index b82c87d71..d81521374 100644
--- a/tests/integration/aof.tcl
+++ b/tests/integration/aof.tcl
@@ -52,15 +52,9 @@ tags {"aof"} {
             assert_equal 1 [is_alive $srv]
         }
 
-        set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
-
-        wait_for_condition 50 100 {
-            [catch {$client ping} e] == 0
-        } else {
-            fail "Loading DB is taking too much time."
-        }
-
         test "Truncated AOF loaded: we expect foo to be equal to 5" {
+            set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
+            wait_done_loading $client
             assert {[$client get foo] eq "5"}
         }
 
@@ -75,15 +69,9 @@ tags {"aof"} {
             assert_equal 1 [is_alive $srv]
         }
 
-        set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
-
-        wait_for_condition 50 100 {
-            [catch {$client ping} e] == 0
-        } else {
-            fail "Loading DB is taking too much time."
-        }
-
         test "Truncated AOF loaded: we expect foo to be equal to 6 now" {
+            set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
+            wait_done_loading $client
             assert {[$client get foo] eq "6"}
         }
     }
@@ -183,11 +171,7 @@ tags {"aof"} {
 
         test "Fixed AOF: Keyspace should contain values that were parseable" {
             set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
-            wait_for_condition 50 100 {
-                [catch {$client ping} e] == 0
-            } else {
-                fail "Loading DB is taking too much time."
-            }
+            wait_done_loading $client
             assert_equal "hello" [$client get foo]
             assert_equal "" [$client get bar]
         }
@@ -207,11 +191,7 @@ tags {"aof"} {
 
         test "AOF+SPOP: Set should have 1 member" {
             set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
-            wait_for_condition 50 100 {
-                [catch {$client ping} e] == 0
-            } else {
-                fail "Loading DB is taking too much time."
-            }
+            wait_done_loading $client
             assert_equal 1 [$client scard set]
         }
     }
@@ -231,11 +211,7 @@ tags {"aof"} {
 
         test "AOF+SPOP: Set should have 1 member" {
             set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
-            wait_for_condition 50 100 {
-                [catch {$client ping} e] == 0
-            } else {
-                fail "Loading DB is taking too much time."
-            }
+            wait_done_loading $client
             assert_equal 1 [$client scard set]
         }
     }
@@ -254,11 +230,7 @@ tags {"aof"} {
 
         test "AOF+EXPIRE: List should be empty" {
             set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
-            wait_for_condition 50 100 {
-                [catch {$client ping} e] == 0
-            } else {
-                fail "Loading DB is taking too much time."
-            }
+            wait_done_loading $client
             assert_equal 0 [$client llen list]
         }
     }
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index bf590d402..021547854 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -99,6 +99,14 @@ proc wait_for_ofs_sync {r1 r2} {
     }
 }
 
+proc wait_done_loading r {
+    wait_for_condition 50 100 {
+        [catch {$r ping} e] == 0
+    } else {
+        fail "Loading DB is taking too much time."
+    }
+}
+
 # count current log lines in server's stdout
 proc count_log_lines {srv_idx} {
     set _ [exec wc -l < [srv $srv_idx stdout]]

From fe5da2e60d8d6d907062f4789673fbe06fa8773e Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 31 Aug 2020 11:24:17 +0300
Subject: [PATCH 0629/1098] test infra - add durable mode to work around test
 suite crashing

in some cases a command that returns an error possibly due to a timing
issue causes the tcl code to crash and thus prevents the rest of the
tests from running. this adds an option to make the test proceed despite
the crash.
maybe it should be the default mode some day.
---
 tests/support/server.tcl | 28 ++++++++++++++++++++++++++--
 tests/support/test.tcl   |  6 +++++-
 tests/test_helper.tcl    |  4 ++++
 3 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index 9894e6f2a..f2f6ceece 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -297,7 +297,20 @@ proc start_server {options {code undefined}} {
             lappend ::servers $srv
         }
         r flushall
-        uplevel 1 $code
+        if {[catch {set retval [uplevel 1 $code]} error]} {
+            if {$::durable} {
+                set msg [string range $error 10 end]
+                lappend details $msg
+                lappend details $::errorInfo
+                lappend ::tests_failed $details
+
+                incr ::num_failed
+                send_data_packet $::test_server_fd err [join $details "\n"]
+            } else {
+                # Re-raise, let handler up the stack take care of this.
+                error $error $::errorInfo
+            }
+        }
         set ::tags [lrange $::tags 0 end-[llength $tags]]
         return
     }
@@ -468,7 +481,18 @@ proc start_server {options {code undefined}} {
             }
             puts ""
 
-            error $error $backtrace
+            if {$::durable} {
+                set msg [string range $error 10 end]
+                lappend details $msg
+                lappend details $backtrace
+                lappend ::tests_failed $details
+
+                incr ::num_failed
+                send_data_packet $::test_server_fd err [join $details "\n"]
+            } else {
+                # Re-raise, let handler up the stack take care of this.
+                error $error $backtrace
+            }
         }
 
         # fetch srv back from the server list, in case it was restarted by restart_server (new PID)
diff --git a/tests/support/test.tcl b/tests/support/test.tcl
index 773461abb..f5b4c8bef 100644
--- a/tests/support/test.tcl
+++ b/tests/support/test.tcl
@@ -149,9 +149,13 @@ proc test {name code {okpattern undefined} {options undefined}} {
     send_data_packet $::test_server_fd testing $name
 
     if {[catch {set retval [uplevel 1 $code]} error]} {
-        if {[string match "assertion:*" $error]} {
+        set assertion [string match "assertion:*" $error]
+        if {$assertion || $::durable} {
             set msg [string range $error 10 end]
             lappend details $msg
+            if {!$assertion} {
+                lappend details $::errorInfo
+            }
             lappend ::tests_failed $details
 
             incr ::num_failed
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 4a470ec30..fe2d484b8 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -79,6 +79,7 @@ set ::baseport 21111; # initial port for spawned redis servers
 set ::portcount 8000; # we don't wanna use more than 10000 to avoid collision with cluster bus ports
 set ::traceleaks 0
 set ::valgrind 0
+set ::durable 0
 set ::tls 0
 set ::stack_logging 0
 set ::verbose 0
@@ -521,6 +522,7 @@ proc send_data_packet {fd status data} {
 proc print_help_screen {} {
     puts [join {
         "--valgrind         Run the test over valgrind."
+        "--durable          suppress test crashes and keep running"
         "--stack-logging    Enable OSX leaks/malloc stack logging."
         "--accurate         Run slow randomized tests for more iterations."
         "--quiet            Don't show individual tests."
@@ -633,6 +635,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
     } elseif {$opt eq {--clients}} {
         set ::numclients $arg
         incr j
+    } elseif {$opt eq {--durable}} {
+        set ::durable 1
     } elseif {$opt eq {--dont-clean}} {
         set ::dont_clean 1
     } elseif {$opt eq {--wait-server}} {

From 2b998de46078c172c6b19ac3b779318e7992c60a Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 6 Sep 2020 11:11:49 +0300
Subject: [PATCH 0630/1098] Improve valgrind support for cluster tests (#7725)

- redirect valgrind reports to a dedicated file rather than console
- try to avoid killing instances with SIGKILL so that we get the memory
  leak report (killing with SIGTERM before resorting to SIGKILL)
- search for valgrind reports when done, print them and fail the tests
- add --dont-clean option to keep the logs on exit
- fix exit error code when crash is found (would have exited with 0)

changes that affect the normal redis test suite:
- refactor check_valgrind_errors into two functions one to search and
  one to report
- move the search half into util.tcl to serve the cluster tests too
- ignore "address range perms" valgrind warnings which seem non relevant.
---
 tests/instances.tcl      | 65 +++++++++++++++++++++++++++++++++++-----
 tests/support/server.tcl | 18 ++---------
 tests/support/util.tcl   | 23 ++++++++++++++
 3 files changed, 83 insertions(+), 23 deletions(-)

diff --git a/tests/instances.tcl b/tests/instances.tcl
index a43a4cc87..2029bc5f5 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -19,6 +19,7 @@ set ::verbose 0
 set ::valgrind 0
 set ::tls 0
 set ::pause_on_error 0
+set ::dont_clean 0
 set ::simulate_error 0
 set ::failed 0
 set ::sentinel_instances {}
@@ -38,7 +39,7 @@ if {[catch {cd tmp}]} {
 
 # Execute the specified instance of the server specified by 'type', using
 # the provided configuration file. Returns the PID of the process.
-proc exec_instance {type cfgfile} {
+proc exec_instance {type dirname cfgfile} {
     if {$type eq "redis"} {
         set prgname redis-server
     } elseif {$type eq "sentinel"} {
@@ -47,8 +48,9 @@ proc exec_instance {type cfgfile} {
         error "Unknown instance type."
     }
 
+    set errfile [file join $dirname err.txt]
     if {$::valgrind} {
-        set pid [exec valgrind --track-origins=yes --suppressions=../../../src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full ../../../src/${prgname} $cfgfile &]
+        set pid [exec valgrind --track-origins=yes --suppressions=../../../src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full ../../../src/${prgname} $cfgfile 2>> $errfile &]
     } else {
         set pid [exec ../../../src/${prgname} $cfgfile &]
     }
@@ -93,7 +95,7 @@ proc spawn_instance {type base_port count {conf {}}} {
         # Finally exec it and remember the pid for later cleanup.
         set retry 100
         while {$retry} {
-            set pid [exec_instance $type $cfgfile]
+            set pid [exec_instance $type $dirname $cfgfile]
 
             # Check availability
             if {[server_is_up 127.0.0.1 $port 100] == 0} {
@@ -144,16 +146,60 @@ proc log_crashes {} {
                 puts "\n*** Crash report found in $log ***"
                 set found 1
             }
-            if {$found} {puts $line}
+            if {$found} {
+                puts $line
+                incr ::failed
+            }
         }
     }
+
+    set logs [glob */err.txt]
+    foreach log $logs {
+        set res [find_valgrind_errors $log]
+        if {$res != ""} {
+            puts $res
+            incr ::failed
+        }
+    }
+}
+
+proc is_alive pid {
+    if {[catch {exec ps -p $pid} err]} {
+        return 0
+    } else {
+        return 1
+    }
+}
+
+proc stop_instance pid {
+    catch {exec kill $pid}
+    if {$::valgrind} {
+        set max_wait 60000
+    } else {
+        set max_wait 10000
+    }
+    while {[is_alive $pid]} {
+        incr wait 10
+
+        if {$wait >= $max_wait} {
+            puts "Forcing process $pid to exit..."
+            catch {exec kill -KILL $pid}
+        } elseif {$wait % 1000 == 0} {
+            puts "Waiting for process $pid to exit..."
+        }
+        after 10
+    }
 }
 
 proc cleanup {} {
     puts "Cleaning up..."
-    log_crashes
     foreach pid $::pids {
-        catch {exec kill -9 $pid}
+        puts "killing stale instance $pid"
+        stop_instance $pid
+    }
+    log_crashes
+    if {$::dont_clean} {
+        return
     }
     foreach dir $::dirs {
         catch {exec rm -rf $dir}
@@ -178,6 +224,8 @@ proc parse_options {} {
             set ::run_matching "*${val}*"
         } elseif {$opt eq "--pause-on-error"} {
             set ::pause_on_error 1
+        } elseif {$opt eq {--dont-clean}} {
+            set ::dont_clean 1
         } elseif {$opt eq "--fail"} {
             set ::simulate_error 1
         } elseif {$opt eq {--valgrind}} {
@@ -191,6 +239,7 @@ proc parse_options {} {
             set ::tls 1
         } elseif {$opt eq "--help"} {
             puts "--single <pattern>      Only runs tests specified by pattern."
+            puts "--dont-clean            Keep log files on exit."
             puts "--pause-on-error        Pause for manual inspection on error."
             puts "--fail                  Simulate a test failure."
             puts "--valgrind              Run with valgrind."
@@ -486,7 +535,7 @@ proc kill_instance {type id} {
         error "You tried to kill $type $id twice."
     }
 
-    exec kill -9 $pid
+    stop_instance $pid
     set_instance_attrib $type $id pid -1
     set_instance_attrib $type $id link you_tried_to_talk_with_killed_instance
 
@@ -521,7 +570,7 @@ proc restart_instance {type id} {
 
     # Execute the instance with its old setup and append the new pid
     # file for cleanup.
-    set pid [exec_instance $type $cfgfile]
+    set pid [exec_instance $type $dirname $cfgfile]
     set_instance_attrib $type $id pid $pid
     lappend ::pids $pid
 
diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index f2f6ceece..f74d839ad 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -13,21 +13,9 @@ proc start_server_error {config_file error} {
 }
 
 proc check_valgrind_errors stderr {
-    set fd [open $stderr]
-    set buf [read $fd]
-    close $fd
-
-    # look for stack trace and other errors, or the absense of a leak free summary
-    if {[regexp -- { at 0x} $buf] ||
-        [regexp -- {Warning} $buf] ||
-        [regexp -- {Invalid} $buf] ||
-        [regexp -- {Mismatched} $buf] ||
-        [regexp -- {uninitialized} $buf] ||
-        [regexp -- {has a fishy} $buf] ||
-        [regexp -- {overlap} $buf] ||
-        (![regexp -- {definitely lost: 0 bytes} $buf] &&
-         ![regexp -- {no leaks are possible} $buf])} {
-        send_data_packet $::test_server_fd err "Valgrind error: $buf\n"
+    set res [find_valgrind_errors $stderr]
+    if {$res != ""} {
+        send_data_packet $::test_server_fd err "Valgrind error: $res\n"
     }
 }
 
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index 021547854..6d11e5520 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -432,6 +432,29 @@ proc colorstr {color str} {
     }
 }
 
+proc find_valgrind_errors {stderr} {
+    set fd [open $stderr]
+    set buf [read $fd]
+    close $fd
+
+    # Look for stack trace (" at 0x") and other errors (Invalid, Mismatched, etc).
+    # Look for "Warnings", but not the "set address range perms". These don't indicate any real concern.
+    # Look for the absense of a leak free summary (happens when redis isn't terminated properly).
+    if {[regexp -- { at 0x} $buf] ||
+        [regexp -- {^(?=.*Warning)(?:(?!set address range perms).)*$} $buf] ||
+        [regexp -- {Invalid} $buf] ||
+        [regexp -- {Mismatched} $buf] ||
+        [regexp -- {uninitialized} $buf] ||
+        [regexp -- {has a fishy} $buf] ||
+        [regexp -- {overlap} $buf] ||
+        (![regexp -- {definitely lost: 0 bytes} $buf] &&
+         ![regexp -- {no leaks are possible} $buf])} {
+        return $buf
+    }
+
+    return ""
+}
+
 # Execute a background process writing random data for the specified number
 # of seconds to the specified Redis instance.
 proc start_write_load {host port seconds} {

From 573246f73c0d9de6155a7bf5f0cbce98da129afb Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 6 Sep 2020 16:43:57 +0300
Subject: [PATCH 0631/1098] if diskless repl child is killed, make sure to reap
 the pid (#7742)

Starting redis 6.0 and the changes we made to the diskless master to be
suitable for TLS, I made the master avoid reaping (wait3) the pid of the
child until we know all replicas are done reading their rdb.

I did that in order to avoid a state where the rdb_child_pid is -1 but
we don't yet want to start another fork (still busy serving that data to
replicas).

It turns out that the solution used so far was problematic in case the
fork child was being killed (e.g. by the kernel OOM killer), in that
case there's a chance that we currently disabled the read event on the
rdb pipe, since we're waiting for a replica to become writable again.
and in that scenario the master would have never realized the child
exited, and the replica will remain hung too.
Note that there's no mechanism to detect a hung replica while it's in
rdb transfer state.

The solution here is to add another pipe which is used by the parent to
tell the child it is safe to exit. this mean that when the child exits,
for whatever reason, it is safe to reap it.

Besides that, i'm re-introducing an adjustment to REPLCONF ACK which was
part of #6271 (Accelerate diskless master connections) but was dropped
when that PR was rebased after the TLS fork/pipe changes (5a47794).
Now that RdbPipeCleanup no longer calls checkChildrenDone, and the ACK
has chance to detect that the child exited, it should be the one to call
it so that we don't have to wait for cron (server.hz) to do that.
---
 src/rdb.c                         | 45 +++++++++++++++++++++++++------
 src/replication.c                 | 32 ++++++++--------------
 src/server.c                      |  8 ------
 src/server.h                      |  5 ++--
 tests/integration/replication.tcl | 42 +++++++++++++++++++++++++++++
 tests/support/util.tcl            |  9 +++++++
 tests/unit/oom-score-adj.tcl      | 11 +-------
 7 files changed, 103 insertions(+), 49 deletions(-)

diff --git a/src/rdb.c b/src/rdb.c
index d4e44be5d..827ddc4f2 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2449,6 +2449,18 @@ void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) {
     server.rdb_child_pid = -1;
     server.rdb_child_type = RDB_CHILD_TYPE_NONE;
     server.rdb_save_time_start = -1;
+    if (server.rdb_child_exit_pipe!=-1)
+        close(server.rdb_child_exit_pipe);
+    close(server.rdb_pipe_read);
+    server.rdb_child_exit_pipe = -1;
+    server.rdb_pipe_read = -1;
+    zfree(server.rdb_pipe_conns);
+    server.rdb_pipe_conns = NULL;
+    server.rdb_pipe_numconns = 0;
+    server.rdb_pipe_numconns_writing = 0;
+    zfree(server.rdb_pipe_buff);
+    server.rdb_pipe_buff = NULL;
+    server.rdb_pipe_bufflen = 0;
 
     updateSlavesWaitingBgsave((!bysignal && exitcode == 0) ? C_OK : C_ERR, RDB_CHILD_TYPE_SOCKET);
 }
@@ -2484,7 +2496,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
     listNode *ln;
     listIter li;
     pid_t childpid;
-    int pipefds[2];
+    int pipefds[2], rdb_pipe_write, safe_to_exit_pipe;
 
     if (hasActiveChildProcess()) return C_ERR;
 
@@ -2497,10 +2509,20 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
      * of TLS we must let the parent handle a continuous TLS state when the
      * child terminates and parent takes over. */
     if (pipe(pipefds) == -1) return C_ERR;
-    server.rdb_pipe_read = pipefds[0];
-    server.rdb_pipe_write = pipefds[1];
+    server.rdb_pipe_read = pipefds[0]; /* read end */
+    rdb_pipe_write = pipefds[1]; /* write end */
     anetNonBlock(NULL, server.rdb_pipe_read);
 
+    /* create another pipe that is used by the parent to signal to the child
+     * that it can exit. */
+    if (pipe(pipefds) == -1) {
+        close(rdb_pipe_write);
+        close(server.rdb_pipe_read);
+        return C_ERR;
+    }
+    safe_to_exit_pipe = pipefds[0]; /* read end */
+    server.rdb_child_exit_pipe = pipefds[1]; /* write end */
+
     /* Collect the connections of the replicas we want to transfer
      * the RDB to, which are i WAIT_BGSAVE_START state. */
     server.rdb_pipe_conns = zmalloc(sizeof(connection *)*listLength(server.slaves));
@@ -2519,10 +2541,10 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
     openChildInfoPipe();
     if ((childpid = redisFork()) == 0) {
         /* Child */
-        int retval;
+        int retval, dummy;
         rio rdb;
 
-        rioInitWithFd(&rdb,server.rdb_pipe_write);
+        rioInitWithFd(&rdb,rdb_pipe_write);
 
         redisSetProcTitle("redis-rdb-to-slaves");
         redisSetCpuAffinity(server.bgsave_cpulist);
@@ -2536,10 +2558,17 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
         }
 
         rioFreeFd(&rdb);
-        close(server.rdb_pipe_write); /* wake up the reader, tell it we're done. */
+        /* wake up the reader, tell it we're done. */
+        close(rdb_pipe_write);
+        close(server.rdb_child_exit_pipe); /* close write end so that we can detect the close on the parent. */
+        /* hold exit until the parent tells us it's safe. we're not expecting
+         * to read anything, just get the error when the pipe is closed. */
+        dummy = read(safe_to_exit_pipe, pipefds, 1);
+        UNUSED(dummy);
         exitFromChild((retval == C_OK) ? 0 : 1);
     } else {
         /* Parent */
+        close(safe_to_exit_pipe);
         if (childpid == -1) {
             serverLog(LL_WARNING,"Can't save in background: fork: %s",
                 strerror(errno));
@@ -2554,7 +2583,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
                     slave->replstate = SLAVE_STATE_WAIT_BGSAVE_START;
                 }
             }
-            close(server.rdb_pipe_write);
+            close(rdb_pipe_write);
             close(server.rdb_pipe_read);
             zfree(server.rdb_pipe_conns);
             server.rdb_pipe_conns = NULL;
@@ -2567,7 +2596,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
             server.rdb_save_time_start = time(NULL);
             server.rdb_child_pid = childpid;
             server.rdb_child_type = RDB_CHILD_TYPE_SOCKET;
-            close(server.rdb_pipe_write); /* close write in parent so that it can detect the close on the child. */
+            close(rdb_pipe_write); /* close write in parent so that it can detect the close on the child. */
             if (aeCreateFileEvent(server.el, server.rdb_pipe_read, AE_READABLE, rdbPipeReadHandler,NULL) == AE_ERR) {
                 serverPanic("Unrecoverable error creating server.rdb_pipe_read file event.");
             }
diff --git a/src/replication.c b/src/replication.c
index d629a197f..32c41361c 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -912,7 +912,12 @@ void replconfCommand(client *c) {
              * the slave online when the first ACK is received (which
              * confirms slave is online and ready to get more data). This
              * allows for simpler and less CPU intensive EOF detection
-             * when streaming RDB files. */
+             * when streaming RDB files.
+             * There's a chance the ACK got to us before we detected that the
+             * bgsave is done (since that depends on cron ticks), so run a
+             * quick check first (instead of waiting for the next ACK. */
+            if (server.rdb_child_pid != -1 && c->replstate == SLAVE_STATE_WAIT_BGSAVE_END)
+                checkChildrenDone();
             if (c->repl_put_online_on_ack && c->replstate == SLAVE_STATE_ONLINE)
                 putSlaveOnline(c);
             /* Note: this command does not reply anything! */
@@ -1104,24 +1109,6 @@ void rdbPipeWriteHandler(struct connection *conn) {
     rdbPipeWriteHandlerConnRemoved(conn);
 }
 
-/* When the the pipe serving diskless rdb transfer is drained (write end was
- * closed), we can clean up all the temporary variables, and cleanup after the
- * fork child. */
-void RdbPipeCleanup() {
-    close(server.rdb_pipe_read);
-    zfree(server.rdb_pipe_conns);
-    server.rdb_pipe_conns = NULL;
-    server.rdb_pipe_numconns = 0;
-    server.rdb_pipe_numconns_writing = 0;
-    zfree(server.rdb_pipe_buff);
-    server.rdb_pipe_buff = NULL;
-    server.rdb_pipe_bufflen = 0;
-
-    /* Since we're avoiding to detect the child exited as long as the pipe is
-     * not drained, so now is the time to check. */
-    checkChildrenDone();
-}
-
 /* Called in diskless master, when there's data to read from the child's rdb pipe */
 void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask) {
     UNUSED(mask);
@@ -1162,7 +1149,11 @@ void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData,
                 stillUp++;
             }
             serverLog(LL_WARNING,"Diskless rdb transfer, done reading from pipe, %d replicas still up.", stillUp);
-            RdbPipeCleanup();
+            /* Now that the replicas have finished reading, notify the child that it's safe to exit. 
+             * When the server detectes the child has exited, it can mark the replica as online, and
+             * start streaming the replication buffers. */
+            close(server.rdb_child_exit_pipe);
+            server.rdb_child_exit_pipe = -1;
             return;
         }
 
@@ -1203,7 +1194,6 @@ void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData,
         if (stillAlive == 0) {
             serverLog(LL_WARNING,"Diskless rdb transfer, last replica dropped, killing fork child.");
             killRDBChild();
-            RdbPipeCleanup();
         }
         /*  Remove the pipe read handler if at least one write handler was set. */
         if (server.rdb_pipe_numconns_writing || stillAlive == 0) {
diff --git a/src/server.c b/src/server.c
index 5e9c85585..aa3e6b21b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1785,14 +1785,6 @@ void checkChildrenDone(void) {
     int statloc;
     pid_t pid;
 
-    /* If we have a diskless rdb child (note that we support only one concurrent
-     * child), we want to avoid collecting it's exit status and acting on it
-     * as long as we didn't finish to drain the pipe, since then we're at risk
-     * of starting a new fork and a new pipe before we're done with the previous
-     * one. */
-    if (server.rdb_child_pid != -1 && server.rdb_pipe_conns)
-        return;
-
     if ((pid = wait3(&statloc,WNOHANG,NULL)) != 0) {
         int exitcode = WEXITSTATUS(statloc);
         int bysignal = 0;
diff --git a/src/server.h b/src/server.h
index 6dac914c6..05b7d1dd7 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1257,8 +1257,9 @@ struct redisServer {
     int rdb_child_type;             /* Type of save by active child. */
     int lastbgsave_status;          /* C_OK or C_ERR */
     int stop_writes_on_bgsave_err;  /* Don't allow writes if can't BGSAVE */
-    int rdb_pipe_write;             /* RDB pipes used to transfer the rdb */
-    int rdb_pipe_read;              /* data to the parent process in diskless repl. */
+    int rdb_pipe_read;              /* RDB pipe used to transfer the rdb data */
+                                    /* to the parent process in diskless repl. */
+    int rdb_child_exit_pipe;        /* Used by the diskless parent allow child exit. */
     connection **rdb_pipe_conns;    /* Connections which are currently the */
     int rdb_pipe_numconns;          /* target of diskless rdb fork child. */
     int rdb_pipe_numconns_writing;  /* Number of rdb conns with pending writes. */
diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl
index a2f9e75bd..5438b0eba 100644
--- a/tests/integration/replication.tcl
+++ b/tests/integration/replication.tcl
@@ -659,6 +659,48 @@ start_server {tags {"repl"}} {
     }
 }
 
+test "diskless replication child being killed is collected" {
+    # when diskless master is waiting for the replica to become writable
+    # it removes the read event from the rdb pipe so if the child gets killed
+    # the replica will hung. and the master may not collect the pid with wait3
+    start_server {tags {"repl"}} {
+        set master [srv 0 client]
+        set master_host [srv 0 host]
+        set master_port [srv 0 port]
+        set master_pid [srv 0 pid]
+        $master config set repl-diskless-sync yes
+        $master config set repl-diskless-sync-delay 0
+        # put enough data in the db that the rdb file will be bigger than the socket buffers
+        $master debug populate 20000 test 10000
+        $master config set rdbcompression no
+        start_server {} {
+            set replica [srv 0 client]
+            set loglines [count_log_lines 0]
+            $replica config set repl-diskless-load swapdb
+            $replica config set key-load-delay 1000000
+            $replica replicaof $master_host $master_port
+
+            # wait for the replicas to start reading the rdb
+            wait_for_log_messages 0 {"*Loading DB in memory*"} $loglines 800 10
+
+            # wait to be sure the eplica is hung and the master is blocked on write
+            after 500
+
+            # simulate the OOM killer or anyone else kills the child
+            set fork_child_pid [get_child_pid -1]
+            puts "fork child is $fork_child_pid"
+            exec kill -9 $fork_child_pid
+
+            # wait for the parent to notice the child have exited
+            wait_for_condition 50 100 {
+                [s -1 rdb_bgsave_in_progress] == 0
+            } else {
+                fail "rdb child didn't terminate"
+            }
+        }
+    }
+}
+
 test {replicaof right after disconnection} {
     # this is a rare race condition that was reproduced sporadically by the psync2 unit.
     # see details in #7205
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index 6d11e5520..ed4ff713a 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -505,3 +505,12 @@ proc populate {num prefix size} {
     }
     $rd close
 }
+
+proc get_child_pid {idx} {
+    set pid [srv $idx pid]
+    set fd [open "|ps --ppid $pid -o pid" "r"]
+    set child_pid [string trim [lindex [split [read $fd] \n] 1]]
+    close $fd
+
+    return $child_pid
+}
diff --git a/tests/unit/oom-score-adj.tcl b/tests/unit/oom-score-adj.tcl
index 993004602..8eb09a993 100644
--- a/tests/unit/oom-score-adj.tcl
+++ b/tests/unit/oom-score-adj.tcl
@@ -14,15 +14,6 @@ if {$system_name eq {linux}} {
             return $val
         }
 
-        proc get_child_pid {} {
-            set pid [srv 0 pid]
-            set fd [open "|ps --ppid $pid -o pid" "r"]
-            set child_pid [string trim [lindex [split [read $fd] \n] 1]]
-            close $fd
-
-            return $child_pid
-        }
-
         test {CONFIG SET oom-score-adj works as expected} {
             set base [get_oom_score_adj]
 
@@ -47,7 +38,7 @@ if {$system_name eq {linux}} {
             r config set rdb-key-save-delay 100000
             r bgsave
 
-            set child_pid [get_child_pid]
+            set child_pid [get_child_pid 0]
             assert {[get_oom_score_adj $child_pid] == [expr $base + 30]}
         }
 

From 42ba7a1b75aa100ba99d2024420bc658439452fb Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 7 Sep 2020 16:26:11 +0300
Subject: [PATCH 0632/1098] fix broken cluster/sentinel tests by recent commit
 (#7752)

2b998de46 added a file for stderr to keep valgrind log but i forgot to
add a similar thing when valgrind isn't being used.
the result is that `glob */err.txt` fails.
---
 tests/instances.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/instances.tcl b/tests/instances.tcl
index 2029bc5f5..82c35854b 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -52,7 +52,7 @@ proc exec_instance {type dirname cfgfile} {
     if {$::valgrind} {
         set pid [exec valgrind --track-origins=yes --suppressions=../../../src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full ../../../src/${prgname} $cfgfile 2>> $errfile &]
     } else {
-        set pid [exec ../../../src/${prgname} $cfgfile &]
+        set pid [exec ../../../src/${prgname} $cfgfile 2>> $errfile &]
     }
     return $pid
 }

From 2df4cb93acabf10bb0ff39c12030791b0947e719 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 7 Sep 2020 17:30:36 +0300
Subject: [PATCH 0633/1098] Tests: fix unmonitored servers. (#7756)

There is an inherent race condition in port allocation for spawned
servers. If a server fails to start because a port is taken, a new port
is allocated. This fixes a problem where the logs are not truncated and
as a result a large number of unmonitored servers are started.
---
 tests/support/server.tcl | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index f74d839ad..30d0d4045 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -383,6 +383,11 @@ proc start_server {options {code undefined}} {
                 dict set config port $port
             }
             create_server_config_file $config_file $config
+
+            # Truncate log so wait_server_started will not be looking at
+            # output of the failed server.
+            close [open $stdout "w"]
+
             continue; # Try again
         }
 

From b491d477c3062d1fb064a794d74643c82d1c4adc Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 7 Sep 2020 18:06:25 +0300
Subject: [PATCH 0634/1098] Fix cluster consistency-check test (#7754)

This test was failing from time to time see discussion at the bottom of #7635
This was probably due to timing, the DEBUG SLEEP executed by redis-cli
didn't sleep for enough time.

This commit changes:
1) use SET-ACTIVE-EXPIRE instead of DEBUG SLEEP
2) reduce many `after` sleeps with retry loops to speed up the test.
3) add many comment explaining the different steps of the test and
   it's purpose.
4) config appendonly before populating the volatile keys, so that they'll
   be part of the AOF command stream rather than the preamble RDB portion.

other complications: recently kill_instance switched from SIGKILL to
SIGTERM, and this would sometimes fail since there was an AOFRW running
in the background. now we wait for it to end before attempting the kill.
---
 tests/cluster/tests/14-consistency-check.tcl | 86 +++++++++++++-------
 1 file changed, 56 insertions(+), 30 deletions(-)

diff --git a/tests/cluster/tests/14-consistency-check.tcl b/tests/cluster/tests/14-consistency-check.tcl
index 5a80dd0df..ddc0570e6 100644
--- a/tests/cluster/tests/14-consistency-check.tcl
+++ b/tests/cluster/tests/14-consistency-check.tcl
@@ -39,53 +39,79 @@ proc cluster_write_keys_with_expire {id ttl} {
     $cluster close
 }
 
+# make sure that replica who restarts from persistence will load keys
+# that have already expired, critical for correct execution of commands
+# that arrive from the master
 proc test_slave_load_expired_keys {aof} {
     test "Slave expired keys is loaded when restarted: appendonly=$aof" {
         set master_id [find_non_empty_master]
         set replica_id [get_one_of_my_replica $master_id]
 
-        set master_dbsize [R $master_id dbsize]
-        set slave_dbsize [R $replica_id dbsize]
-        assert_equal $master_dbsize $slave_dbsize
-
-        set data_ttl 5
-        cluster_write_keys_with_expire $master_id $data_ttl
-        after 100
-        set replica_dbsize_1 [R $replica_id dbsize]
-        assert {$replica_dbsize_1  > $slave_dbsize}
+        set master_dbsize_0 [R $master_id dbsize]
+        set replica_dbsize_0 [R $replica_id dbsize]
+        assert_equal $master_dbsize_0 $replica_dbsize_0
 
+        # config the replica persistency and rewrite the config file to survive restart
+        # note that this needs to be done before populating the volatile keys since
+        # that triggers and AOFRW, and we rather the AOF file to have SETEX commands
+        # rather than an RDB with volatile keys
         R $replica_id config set appendonly $aof
         R $replica_id config rewrite
 
-        set start_time [clock seconds]
-        set end_time [expr $start_time+$data_ttl+2]
-        R $replica_id save
-        set replica_dbsize_2 [R $replica_id dbsize]
-        assert {$replica_dbsize_2  > $slave_dbsize}
+        # fill with 100 keys with 3 second TTL
+        set data_ttl 3
+        cluster_write_keys_with_expire $master_id $data_ttl
+
+        # wait for replica to be in sync with master
+        wait_for_condition 500 10 {
+            [R $replica_id dbsize] eq [R $master_id dbsize]
+        } else {
+            fail "replica didn't sync"
+        }
+        
+        set replica_dbsize_1 [R $replica_id dbsize]
+        assert {$replica_dbsize_1 > $replica_dbsize_0}
+
+        # make replica create persistence file
+        if {$aof == "yes"} {
+            # we need to wait for the initial AOFRW to be done, otherwise
+            # kill_instance (which now uses SIGTERM will fail ("Writing initial AOF, can't exit")
+            wait_for_condition 100 10 {
+                [RI $replica_id aof_rewrite_in_progress] eq 0
+            } else {
+                fail "keys didn't expire"
+            }
+        } else {
+            R $replica_id save
+        }
+
+        # kill the replica (would stay down until re-started)
         kill_instance redis $replica_id
 
-        set master_port [get_instance_attrib redis $master_id port]
-        exec ../../../src/redis-cli \
-            -h 127.0.0.1 -p $master_port \
-            {*}[rediscli_tls_config "../../../tests"] \
-            debug sleep [expr $data_ttl+3] > /dev/null &
+        # Make sure the master doesn't do active expire (sending DELs to the replica)
+        R $master_id DEBUG SET-ACTIVE-EXPIRE 0
 
-        while {[clock seconds] <= $end_time} {
-            #wait for $data_ttl seconds
-        }
+        # wait for all the keys to get logically expired
+        after [expr $data_ttl*1000]
+
+        # start the replica again (loading an RDB or AOF file)
         restart_instance redis $replica_id
 
-        wait_for_condition 200 50 {
-            [R $replica_id ping] eq {PONG}
-        } else {
-            fail "replica #$replica_id not started"
-        }
-
+        # make sure the keys are still there
         set replica_dbsize_3 [R $replica_id dbsize]
-        assert {$replica_dbsize_3  > $slave_dbsize}
+        assert {$replica_dbsize_3 > $replica_dbsize_0}
+        
+        # restore settings
+        R $master_id DEBUG SET-ACTIVE-EXPIRE 1
+
+        # wait for the master to expire all keys and replica to get the DELs
+        wait_for_condition 500 10 {
+            [R $replica_id dbsize] eq $master_dbsize_0
+        } else {
+            fail "keys didn't expire"
+        }
     }
 }
 
 test_slave_load_expired_keys no
-after 5000
 test_slave_load_expired_keys yes

From f22fa9594d536cb53f83ed8e508c03d4278778b0 Mon Sep 17 00:00:00 2001
From: "bodong.ybd" <bodong.ybd@alibaba-inc.com>
Date: Tue, 8 Sep 2020 10:45:03 +0800
Subject: [PATCH 0635/1098] Tests: Some fixes for macOS

1) cur_test: when restart_server, "no such variable" error occurs
  ./runtest --single integration/rdb
  test {client freed during loading}
      SET ::cur_test
      restart_server
        kill_server
          test "Check for memory leaks (pid $pid)"
          SET ::cur_test
          UNSET ::cur_test
      UNSET ::cur_test // This global variable has been unset.

2) `ps --ppid` not available on macOS platform, can be replaced with
`pgrep -P pid`.
---
 tests/support/test.tcl | 1 -
 tests/support/util.tcl | 9 +++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/support/test.tcl b/tests/support/test.tcl
index f5b4c8bef..54d323fa2 100644
--- a/tests/support/test.tcl
+++ b/tests/support/test.tcl
@@ -190,5 +190,4 @@ proc test {name code {okpattern undefined} {options undefined}} {
             send_data_packet $::test_server_fd err "Detected a memory leak in test '$name': $output"
         }
     }
-    unset ::cur_test
 }
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index ed4ff713a..b9a65358f 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -508,8 +508,13 @@ proc populate {num prefix size} {
 
 proc get_child_pid {idx} {
     set pid [srv $idx pid]
-    set fd [open "|ps --ppid $pid -o pid" "r"]
-    set child_pid [string trim [lindex [split [read $fd] \n] 1]]
+    if {[string match {*Darwin*} [exec uname -a]]} {
+        set fd [open "|pgrep -P $pid" "r"]
+        set child_pid [string trim [lindex [split [read $fd] \n] 0]]
+    } else {
+        set fd [open "|ps --ppid $pid -o pid" "r"]
+        set child_pid [string trim [lindex [split [read $fd] \n] 1]]
+    }
     close $fd
 
     return $child_pid

From 5496b4a7cd9f04fb28ab5c6d903f1b59a9c0842c Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 8 Sep 2020 10:59:25 +0300
Subject: [PATCH 0636/1098] Add daily CI for MacOS (#7759)

---
 .github/workflows/daily.yml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 5b395351b..07cd55c87 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -168,3 +168,21 @@ jobs:
         ./runtest-cluster --tls
         ./runtest-cluster
 
+  test-macos-latest:
+    runs-on: macos-latest
+    if: github.repository == 'redis/redis'
+    timeout-minutes: 14400
+    steps:
+    - uses: actions/checkout@v2
+    - name: make
+      run: make
+    - name: test
+      run: |
+        ./runtest --accurate --verbose
+    - name: module api test
+      run: ./runtest-moduleapi --verbose
+    - name: sentinel tests
+      run: ./runtest-sentinel
+    - name: cluster tests
+      run: ./runtest-cluster
+

From 0a1e7341935dbca4bae582de1a4a26d5ed4c652d Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 8 Sep 2020 14:12:03 +0300
Subject: [PATCH 0637/1098] handle cur_test for nested tests

if there are nested tests and nested servers, we need to restore the
previous value of cur_test when a test exist.

example:
```
test{test 1} {
	start_server {
		test{test 1.1 - master only} {
		}
		start_server {
		    test{test 1.2 - with replication} {
            }
		}
	}
}
```
when `test 1.1 - master only exists`, we're still inside `test 1`
---
 tests/support/test.tcl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/support/test.tcl b/tests/support/test.tcl
index 54d323fa2..55937b8f4 100644
--- a/tests/support/test.tcl
+++ b/tests/support/test.tcl
@@ -4,6 +4,7 @@ set ::num_failed 0
 set ::num_skipped 0
 set ::num_aborted 0
 set ::tests_failed {}
+set ::cur_test ""
 
 proc fail {msg} {
     error "assertion:$msg"
@@ -136,6 +137,7 @@ proc test {name code {okpattern undefined} {options undefined}} {
 
     # set a cur_test global to be logged into new servers that are spown
     # and log the test name in all existing servers
+    set prev_test $::cur_test
     set ::cur_test "$name in $::curfile"
     if {!$::external} {
         foreach srv $::servers {
@@ -190,4 +192,5 @@ proc test {name code {okpattern undefined} {options undefined}} {
             send_data_packet $::test_server_fd err "Detected a memory leak in test '$name': $output"
         }
     }
+    set ::cur_test $prev_test
 }

From 750acf3a457053ad9f57a263baa84ec68e0ae145 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 8 Sep 2020 16:00:20 +0300
Subject: [PATCH 0638/1098] Fix CONFIG REWRITE of oom-score-adj-values. (#7761)

---
 src/config.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/config.c b/src/config.c
index 0e1aa01e9..b10e94566 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1422,7 +1422,8 @@ void rewriteConfigOOMScoreAdjValuesOption(struct rewriteConfigState *state) {
     char *option = "oom-score-adj-values";
     sds line;
 
-    line = sdsempty();
+    line = sdsnew(option);
+    line = sdscatlen(line, " ", 1);
     for (j = 0; j < CONFIG_OOM_COUNT; j++) {
         if (server.oom_score_adj_values[j] != configOOMScoreAdjValuesDefaults[j])
             force = 1;

From b120366d48d9e488a406965773e64f29ba2946f7 Mon Sep 17 00:00:00 2001
From: Eran Liberty <eran.liberty@gmail.com>
Date: Wed, 9 Sep 2020 09:35:42 +0300
Subject: [PATCH 0639/1098] Allow exec with read commands on readonly replica
 in cluster (#7766)

There was a bug. Although cluster replicas would allow read commands,
they would not allow a MULTI-EXEC that's composed solely of read commands.
Adds tests for coverage.

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: Eran Liberty <eranl@amazon.com>
---
 src/cluster.c                                 |  4 +-
 .../tests/16-transactions-on-replica.tcl      | 48 +++++++++++++++++++
 tests/instances.tcl                           | 10 +++-
 3 files changed, 59 insertions(+), 3 deletions(-)
 create mode 100644 tests/cluster/tests/16-transactions-on-replica.tcl

diff --git a/src/cluster.c b/src/cluster.c
index ba0ada04a..b9a0c8661 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -5769,8 +5769,10 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
     /* Handle the read-only client case reading from a slave: if this
      * node is a slave and the request is about an hash slot our master
      * is serving, we can reply without redirection. */
+    int is_readonly_command = (c->cmd->flags & CMD_READONLY) ||
+                              (c->cmd->proc == execCommand && !(c->mstate.cmd_inv_flags & CMD_READONLY));
     if (c->flags & CLIENT_READONLY &&
-        (cmd->flags & CMD_READONLY || cmd->proc == evalCommand ||
+        (is_readonly_command || cmd->proc == evalCommand ||
          cmd->proc == evalShaCommand) &&
         nodeIsSlave(myself) &&
         myself->slaveof == n)
diff --git a/tests/cluster/tests/16-transactions-on-replica.tcl b/tests/cluster/tests/16-transactions-on-replica.tcl
new file mode 100644
index 000000000..da9dff1ca
--- /dev/null
+++ b/tests/cluster/tests/16-transactions-on-replica.tcl
@@ -0,0 +1,48 @@
+# Check basic transactions on a replica.
+
+source "../tests/includes/init-tests.tcl"
+
+test "Create a primary with a replica" {
+    create_cluster 1 1
+}
+
+test "Cluster should start ok" {
+    assert_cluster_state ok
+}
+
+set primary [Rn 0]
+set replica [Rn 1]
+
+test "Cant read from replica without READONLY" {
+    $primary SET a 1
+    catch {$replica GET a} err
+    assert {[string range $err 0 4] eq {MOVED}}
+}
+
+test "Can read from replica after READONLY" {
+    $replica READONLY
+    assert {[$replica GET a] eq {1}}
+}
+
+test "Can preform HSET primary and HGET from replica" {
+    $primary HSET h a 1
+    $primary HSET h b 2
+    $primary HSET h c 3
+    assert {[$replica HGET h a] eq {1}}
+    assert {[$replica HGET h b] eq {2}}
+    assert {[$replica HGET h c] eq {3}}
+}
+
+test "Can MULTI-EXEC transaction of HGET operations from replica" {
+    $replica MULTI
+    assert {[$replica HGET h a] eq {QUEUED}}
+    assert {[$replica HGET h b] eq {QUEUED}}
+    assert {[$replica HGET h c] eq {QUEUED}}
+    assert {[$replica EXEC] eq {1 2 3}}
+}
+
+test "MULTI-EXEC with write operations is MOVED" {
+    $replica MULTI
+    catch {$replica HSET h b 4} err
+    assert {[string range $err 0 4] eq {MOVED}}
+}
diff --git a/tests/instances.tcl b/tests/instances.tcl
index 82c35854b..2199cfcd4 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -422,10 +422,16 @@ proc S {n args} {
     [dict get $s link] {*}$args
 }
 
+# Returns a Redis instance by index.
+# Example:
+#     [Rn 0] info
+proc Rn {n} {
+    return [dict get [lindex $::redis_instances $n] link]
+}
+
 # Like R but to chat with Redis instances.
 proc R {n args} {
-    set r [lindex $::redis_instances $n]
-    [dict get $r link] {*}$args
+    [Rn $n] {*}$args
 }
 
 proc get_info_field {info field} {

From 918abd7276afcb994f2d3f8a86a0708993420e37 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 9 Sep 2020 12:30:43 +0300
Subject: [PATCH 0640/1098] Tests: clean up stale .cli files. (#7768)

---
 tests/integration/redis-cli.tcl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/integration/redis-cli.tcl b/tests/integration/redis-cli.tcl
index 2d4145ff0..1e346a9a5 100644
--- a/tests/integration/redis-cli.tcl
+++ b/tests/integration/redis-cli.tcl
@@ -182,6 +182,7 @@ start_server {tags {"cli"}} {
         set tmpfile [write_tmpfile "from file"]
         assert_equal "OK" [run_cli_with_input_file $tmpfile set key]
         assert_equal "from file" [r get key]
+        file delete $tmpfile
     }
 
     test_nontty_cli "Status reply" {
@@ -215,6 +216,7 @@ start_server {tags {"cli"}} {
         set tmpfile [write_tmpfile "from file"]
         assert_equal "OK" [run_cli_with_input_file $tmpfile set key]
         assert_equal "from file" [r get key]
+        file delete $tmpfile
     }
 
     proc test_redis_cli_rdb_dump {} {

From b2419c31c166bd2d73f7af3d089859795c0e3506 Mon Sep 17 00:00:00 2001
From: Jan-Erik Rediger <janerik@fnordig.de>
Date: Wed, 9 Sep 2020 14:06:04 +0200
Subject: [PATCH 0641/1098] Check that THP is not set to always (madvise is ok)
 (#4001)

THP can also be set to madvise, in which case it shouldn't cause
problems for Redis since redis (or the allocator) doesn't use madvise
to activate it.
---
 src/latency.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/latency.c b/src/latency.c
index dfdc6668c..b5ccc7cc6 100644
--- a/src/latency.c
+++ b/src/latency.c
@@ -71,7 +71,7 @@ int THPIsEnabled(void) {
         return 0;
     }
     fclose(fp);
-    return (strstr(buf,"[never]") == NULL) ? 1 : 0;
+    return (strstr(buf,"[always]") != NULL) ? 1 : 0;
 }
 #endif
 

From ce15620dc17ebad94cd03cb4ee779fdd4e99b212 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Wed, 9 Sep 2020 15:09:41 +0300
Subject: [PATCH 0642/1098] Documents RM_Call's fmt (#5448)

Improve RM_Call inline documentation about the fmt argument
so that we don't completely depend on the web docs.

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/module.c | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/src/module.c b/src/module.c
index de66af506..921e55da2 100644
--- a/src/module.c
+++ b/src/module.c
@@ -3303,6 +3303,23 @@ fmterr:
 }
 
 /* Exported API to call any Redis command from modules.
+ *
+ * * **cmdname**: The Redis command to call.
+ * * **fmt**: A format specifier string for the command's arguments. Each
+ *   of the arguments should be specified by a valid type specification:
+ *   b    The argument is a buffer and is immediately followed by another
+ *        argument that is the buffer's length.
+ *   c    The argument is a pointer to a plain C string (null-terminated).
+ *   l    The argument is long long integer.
+ *   s    The argument is a RedisModuleString.
+ *   v    The argument(s) is a vector of RedisModuleString.
+ *
+ *   The format specifier can also include modifiers:
+ *   !    Sends the Redis command and its arguments to replicas and AOF.
+ *   A    Suppress AOF propagation, send only to replicas (requires `!`).
+ *   R    Suppress replicas propagation, send only to AOF (requires `!`).
+ * * **...**: The actual arguments to the Redis command.
+ *
  * On success a RedisModuleCallReply object is returned, otherwise
  * NULL is returned and errno is set to the following values:
  *
@@ -3314,6 +3331,14 @@ fmterr:
  *         in a readonly state.
  * ENETDOWN: operation in Cluster instance when cluster is down.
  *
+ * Example code fragment:
+ * 
+ *      reply = RedisModule_Call(ctx,"INCRBY","sc",argv[1],"10");
+ *      if (RedisModule_CallReplyType(reply) == REDISMODULE_REPLY_INTEGER) {
+ *        long long myval = RedisModule_CallReplyInteger(reply);
+ *        // Do something with myval.
+ *      }
+ *
  * This API is documented here: https://redis.io/topics/modules-intro
  */
 RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) {

From 818a746e32de934c6c6e828ccf39df361361037e Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 9 Sep 2020 15:12:57 +0300
Subject: [PATCH 0643/1098] Fix default/explicit "save" parameter loading.
 (#7767)

Save parameters should either be default or whatever specified in the
config file. This fixes an issue introduced in #7092 which causes
configuration file settings to be applied on top of the defaults.
---
 src/config.c                 |  9 +++++++++
 tests/assets/minimal.conf    |  5 +++++
 tests/unit/introspection.tcl | 13 +++++++++++++
 3 files changed, 27 insertions(+)
 create mode 100644 tests/assets/minimal.conf

diff --git a/src/config.c b/src/config.c
index b10e94566..3e0984208 100644
--- a/src/config.c
+++ b/src/config.c
@@ -353,6 +353,7 @@ void loadServerConfigFromString(char *config) {
     int linenum = 0, totlines, i;
     int slaveof_linenum = 0;
     sds *lines;
+    int save_loaded = 0;
 
     lines = sdssplitlen(config,strlen(config),"\n",1,&totlines);
 
@@ -425,6 +426,14 @@ void loadServerConfigFromString(char *config) {
                 err = "Invalid socket file permissions"; goto loaderr;
             }
         } else if (!strcasecmp(argv[0],"save")) {
+            /* We don't reset save params before loading, because if they're not part
+             * of the file the defaults should be used.
+             */
+            if (!save_loaded) {
+                save_loaded = 1;
+                resetServerSaveParams();
+            }
+
             if (argc == 3) {
                 int seconds = atoi(argv[1]);
                 int changes = atoi(argv[2]);
diff --git a/tests/assets/minimal.conf b/tests/assets/minimal.conf
new file mode 100644
index 000000000..ae14ae87b
--- /dev/null
+++ b/tests/assets/minimal.conf
@@ -0,0 +1,5 @@
+# Minimal configuration for testing.
+always-show-logo yes
+daemonize no
+pidfile /var/run/redis.pid
+loglevel verbose
diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl
index d681e06d5..869f34732 100644
--- a/tests/unit/introspection.tcl
+++ b/tests/unit/introspection.tcl
@@ -58,6 +58,19 @@ start_server {tags {"introspection"}} {
         }
     }
 
+    test {CONFIG save params special case handled properly} {
+        # No "save" keyword - defaults should apply
+        start_server {config "minimal.conf"} {
+            assert_match [r config get save] {save {3600 1 300 100 60 10000}}
+        }
+
+        # First "save" keyword overrides defaults
+        start_server {config "minimal.conf" overrides {save {100 100}}} {
+            # Defaults
+            assert_match [r config get save] {save {100 100}}
+        }
+    }
+
     test {CONFIG sanity} {
         # Do CONFIG GET, CONFIG SET and then CONFIG GET again
         # Skip immutable configs, one with no get, and other complicated configs

From 63730d9dd0a9fea63e460938407ed7bee9303ee0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=A4=A9=E6=B2=B3?= <w1s2j3229@163.com>
Date: Wed, 9 Sep 2020 20:28:38 +0800
Subject: [PATCH 0644/1098] Fix comments of _quicklistSplitNode function.
 (#4341)

Comments about the behavior of the function where wrong (off by one)
Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/quicklist.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/quicklist.c b/src/quicklist.c
index 52e3988f5..0916d1f1b 100644
--- a/src/quicklist.c
+++ b/src/quicklist.c
@@ -797,16 +797,16 @@ REDIS_STATIC void _quicklistMergeNodes(quicklist *quicklist,
  * The 'after' argument controls which quicklistNode gets returned.
  * If 'after'==1, returned node has elements after 'offset'.
  *                input node keeps elements up to 'offset', including 'offset'.
- * If 'after'==0, returned node has elements up to 'offset', including 'offset'.
- *                input node keeps elements after 'offset'.
+ * If 'after'==0, returned node has elements up to 'offset'.
+ *                input node keeps elements after 'offset', including 'offset'.
  *
- * If 'after'==1, returned node will have elements _after_ 'offset'.
+ * Or in other words:
+ * If 'after'==1, returned node will have elements after 'offset'.
  *                The returned node will have elements [OFFSET+1, END].
  *                The input node keeps elements [0, OFFSET].
- *
- * If 'after'==0, returned node will keep elements up to and including 'offset'.
- *                The returned node will have elements [0, OFFSET].
- *                The input node keeps elements [OFFSET+1, END].
+ * If 'after'==0, returned node will keep elements up to but not including 'offset'.
+ *                The returned node will have elements [0, OFFSET-1].
+ *                The input node keeps elements [OFFSET, END].
  *
  * The input node keeps all elements not taken by the returned node.
  *
@@ -821,7 +821,7 @@ REDIS_STATIC quicklistNode *_quicklistSplitNode(quicklistNode *node, int offset,
     /* Copy original ziplist so we can split it */
     memcpy(new_node->zl, node->zl, zl_sz);
 
-    /* -1 here means "continue deleting until the list ends" */
+    /* Ranges to be trimmed: -1 here means "continue deleting until the list ends" */
     int orig_start = after ? offset + 1 : 0;
     int orig_extent = after ? -1 : offset;
     int new_start = after ? 0 : offset;

From 1461f02deb65585bb47c4d50d68ef733edfba6f9 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 9 Sep 2020 15:39:57 +0300
Subject: [PATCH 0645/1098] Change THP warning to use madvise rather than never
 (#7771)

completes b2419c31c166bd2d73f7af3d089859795c0e3506
---
 src/server.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index aa3e6b21b..8a443fd79 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4800,7 +4800,7 @@ void linuxMemoryWarnings(void) {
         serverLog(LL_WARNING,"WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect.");
     }
     if (THPIsEnabled()) {
-        serverLog(LL_WARNING,"WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled.");
+        serverLog(LL_WARNING,"WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo madvise > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled (set to 'madvise' or 'never').");
     }
 }
 #endif /* __linux__ */

From a8b7268911803f4ac827ab22e8e5b10134664832 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 9 Sep 2020 15:43:11 +0300
Subject: [PATCH 0646/1098] Tests: validate CONFIG REWRITE for all params.
 (#7764)

This is a catch-all test to confirm that that rewrite produces a valid
output for all parameters and that this process does not introduce
undesired configuration changes.
---
 src/config.c                 | 12 +++++++++---
 src/debug.c                  |  7 +++++++
 src/sentinel.c               |  2 +-
 src/server.c                 |  2 +-
 src/server.h                 |  2 +-
 tests/unit/introspection.tcl | 24 ++++++++++++++++++++++++
 6 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/src/config.c b/src/config.c
index 3e0984208..237dc82ad 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1064,6 +1064,8 @@ struct rewriteConfigState {
     sds *lines;           /* Current lines as an array of sds strings */
     int has_tail;         /* True if we already added directives that were
                              not present in the original config file. */
+    int force_all;        /* True if we want all keywords to be force
+                             written. Currently only used for testing. */
 };
 
 /* Append the new line to the current configuration state. */
@@ -1110,6 +1112,7 @@ struct rewriteConfigState *rewriteConfigReadOldFile(char *path) {
     state->numlines = 0;
     state->lines = NULL;
     state->has_tail = 0;
+    state->force_all = 0;
     if (fp == NULL) return state;
 
     /* Read the old file line by line, populate the state. */
@@ -1188,7 +1191,7 @@ void rewriteConfigRewriteLine(struct rewriteConfigState *state, const char *opti
 
     rewriteConfigMarkAsProcessed(state,option);
 
-    if (!l && !force) {
+    if (!l && !force && !state->force_all) {
         /* Option not used previously, and we are not forced to use it. */
         sdsfree(line);
         sdsfree(o);
@@ -1612,15 +1615,18 @@ cleanup:
  *
  * Configuration parameters that are at their default value, unless already
  * explicitly included in the old configuration file, are not rewritten.
+ * The force_all flag overrides this behavior and forces everything to be
+ * written. This is currently only used for testing purposes.
  *
  * On error -1 is returned and errno is set accordingly, otherwise 0. */
-int rewriteConfig(char *path) {
+int rewriteConfig(char *path, int force_all) {
     struct rewriteConfigState *state;
     sds newcontent;
     int retval;
 
     /* Step 1: read the old config into our rewrite state. */
     if ((state = rewriteConfigReadOldFile(path)) == NULL) return -1;
+    if (force_all) state->force_all = 1;
 
     /* Step 2: rewrite every single option, replacing or appending it inside
      * the rewrite state. */
@@ -2427,7 +2433,7 @@ NULL
             addReplyError(c,"The server is running without a config file");
             return;
         }
-        if (rewriteConfig(server.configfile) == -1) {
+        if (rewriteConfig(server.configfile, 0) == -1) {
             serverLog(LL_WARNING,"CONFIG REWRITE failed: %s", strerror(errno));
             addReplyErrorFormat(c,"Rewriting config file: %s", strerror(errno));
         } else {
diff --git a/src/debug.c b/src/debug.c
index 5dd9ec849..82b34e08d 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -408,6 +408,7 @@ void debugCommand(client *c) {
 "STRUCTSIZE -- Return the size of different Redis core C structures.",
 "ZIPLIST <key> -- Show low level info about the ziplist encoding.",
 "STRINGMATCH-TEST -- Run a fuzz tester against the stringmatchlen() function.",
+"CONFIG-REWRITE-FORCE-ALL -- Like CONFIG REWRITE but writes all configuration options, including keywords not listed in original configuration file or default values.",
 #ifdef USE_JEMALLOC
 "MALLCTL <key> [<val>] -- Get or set a malloc tunning integer.",
 "MALLCTL-STR <key> [<val>] -- Get or set a malloc tunning string.",
@@ -805,6 +806,12 @@ NULL
     {
         stringmatchlen_fuzz_test();
         addReplyStatus(c,"Apparently Redis did not crash: test passed");
+    } else if (!strcasecmp(c->argv[1]->ptr,"config-rewrite-force-all") && c->argc == 2)
+    {
+        if (rewriteConfig(server.configfile, 1) == -1)
+            addReplyError(c, "CONFIG-REWRITE-FORCE-ALL failed");
+        else
+            addReply(c, shared.ok);
 #ifdef USE_JEMALLOC
     } else if(!strcasecmp(c->argv[1]->ptr,"mallctl") && c->argc >= 3) {
         mallctl_int(c, c->argv+2, c->argc-2);
diff --git a/src/sentinel.c b/src/sentinel.c
index d93b26131..3cf496731 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -1954,7 +1954,7 @@ void sentinelFlushConfig(void) {
     int rewrite_status;
 
     server.hz = CONFIG_DEFAULT_HZ;
-    rewrite_status = rewriteConfig(server.configfile);
+    rewrite_status = rewriteConfig(server.configfile, 0);
     server.hz = saved_hz;
 
     if (rewrite_status == -1) goto werr;
diff --git a/src/server.c b/src/server.c
index 8a443fd79..b1d7614f5 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2552,7 +2552,7 @@ int restartServer(int flags, mstime_t delay) {
     /* Config rewriting. */
     if (flags & RESTART_SERVER_CONFIG_REWRITE &&
         server.configfile &&
-        rewriteConfig(server.configfile) == -1)
+        rewriteConfig(server.configfile, 0) == -1)
     {
         serverLog(LL_WARNING,"Can't restart: configuration rewrite process "
                              "failed");
diff --git a/src/server.h b/src/server.h
index 05b7d1dd7..45f716390 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2101,7 +2101,7 @@ void appendServerSaveParams(time_t seconds, int changes);
 void resetServerSaveParams(void);
 struct rewriteConfigState; /* Forward declaration to export API. */
 void rewriteConfigRewriteLine(struct rewriteConfigState *state, const char *option, sds line, int force);
-int rewriteConfig(char *path);
+int rewriteConfig(char *path, int force_all);
 void initConfigValues();
 
 /* db.c -- Keyspace access API */
diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl
index 869f34732..c8bc976cf 100644
--- a/tests/unit/introspection.tcl
+++ b/tests/unit/introspection.tcl
@@ -147,4 +147,28 @@ start_server {tags {"introspection"}} {
 
         }
     }
+
+    # Do a force-all config rewrite and make sure we're able to parse
+    # it.
+    test {CONFIG REWRITE sanity} {
+        # Capture state of config before
+        set configs {}
+        foreach {k v} [r config get *] {
+            dict set configs $k $v
+        }
+
+        # Rewrite entire configuration, restart and confirm the
+        # server is able to parse it and start.
+        assert_equal [r debug config-rewrite-force-all] "OK"
+        restart_server 0 0
+        assert_equal [r ping] "PONG"
+
+        # Verify no changes were introduced
+        dict for {k v} $configs {
+            assert_equal $v [lindex [r config get $k] 1]
+        }
+    }
+
+    # Config file at this point is at a wierd state, and includes all
+    # known keywords. Might be a good idea to avoid adding tests here.
 }

From 042189fd8707544139337b3ddcf38b5c5fea1bf0 Mon Sep 17 00:00:00 2001
From: Roi Lipman <swilly22@users.noreply.github.com>
Date: Wed, 9 Sep 2020 16:01:16 +0300
Subject: [PATCH 0647/1098] RM_ThreadSafeContextTryLock a non-blocking method
 for acquiring GIL (#7738)

Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
---
 runtest-moduleapi                      |  1 +
 src/module.c                           | 22 +++++++
 src/redismodule.h                      |  2 +
 src/server.h                           |  1 +
 tests/modules/Makefile                 |  4 +-
 tests/modules/blockedclient.c          | 82 ++++++++++++++++++++++++++
 tests/unit/moduleapi/blockedclient.tcl | 11 ++++
 7 files changed, 122 insertions(+), 1 deletion(-)
 create mode 100644 tests/modules/blockedclient.c
 create mode 100644 tests/unit/moduleapi/blockedclient.tcl

diff --git a/runtest-moduleapi b/runtest-moduleapi
index 85386b44a..e32147d5d 100755
--- a/runtest-moduleapi
+++ b/runtest-moduleapi
@@ -26,4 +26,5 @@ $TCLSH tests/test_helper.tcl \
 --single unit/moduleapi/datatype \
 --single unit/moduleapi/auth \
 --single unit/moduleapi/keyspace_events \
+--single unit/moduleapi/blockedclient \
 "${@}"
diff --git a/src/module.c b/src/module.c
index 921e55da2..17178e6a6 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4906,6 +4906,23 @@ void RM_ThreadSafeContextLock(RedisModuleCtx *ctx) {
     moduleAcquireGIL();
 }
 
+/* Similar to RM_ThreadSafeContextLock but this function
+ * would not block if the server lock is already acquired.
+ *
+ * If successful (lock acquired) REDISMODULE_OK is returned,
+ * otherwise REDISMODULE_ERR is returned and errno is set
+ * accordingly. */
+int RM_ThreadSafeContextTryLock(RedisModuleCtx *ctx) {
+    UNUSED(ctx);
+
+    int res = moduleTryAcquireGIL();
+    if(res != 0) {
+        errno = res;
+        return REDISMODULE_ERR;
+    }
+    return REDISMODULE_OK;
+}
+
 /* Release the server lock after a thread safe API call was executed. */
 void RM_ThreadSafeContextUnlock(RedisModuleCtx *ctx) {
     UNUSED(ctx);
@@ -4916,6 +4933,10 @@ void moduleAcquireGIL(void) {
     pthread_mutex_lock(&moduleGIL);
 }
 
+int moduleTryAcquireGIL(void) {
+    return pthread_mutex_trylock(&moduleGIL);
+}
+
 void moduleReleaseGIL(void) {
     pthread_mutex_unlock(&moduleGIL);
 }
@@ -7936,6 +7957,7 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(GetThreadSafeContext);
     REGISTER_API(FreeThreadSafeContext);
     REGISTER_API(ThreadSafeContextLock);
+    REGISTER_API(ThreadSafeContextTryLock);
     REGISTER_API(ThreadSafeContextUnlock);
     REGISTER_API(DigestAddStringBuffer);
     REGISTER_API(DigestAddLongLong);
diff --git a/src/redismodule.h b/src/redismodule.h
index 460fdd480..4bfc14cc7 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -666,6 +666,7 @@ REDISMODULE_API int (*RedisModule_AbortBlock)(RedisModuleBlockedClient *bc) REDI
 REDISMODULE_API RedisModuleCtx * (*RedisModule_GetThreadSafeContext)(RedisModuleBlockedClient *bc) REDISMODULE_ATTR;
 REDISMODULE_API void (*RedisModule_FreeThreadSafeContext)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
 REDISMODULE_API void (*RedisModule_ThreadSafeContextLock)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_ThreadSafeContextTryLock)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
 REDISMODULE_API void (*RedisModule_ThreadSafeContextUnlock)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_SubscribeToKeyspaceEvents)(RedisModuleCtx *ctx, int types, RedisModuleNotificationFunc cb) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_NotifyKeyspaceEvent)(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key) REDISMODULE_ATTR;
@@ -899,6 +900,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(GetThreadSafeContext);
     REDISMODULE_GET_API(FreeThreadSafeContext);
     REDISMODULE_GET_API(ThreadSafeContextLock);
+    REDISMODULE_GET_API(ThreadSafeContextTryLock);
     REDISMODULE_GET_API(ThreadSafeContextUnlock);
     REDISMODULE_GET_API(BlockClient);
     REDISMODULE_GET_API(UnblockClient);
diff --git a/src/server.h b/src/server.h
index 45f716390..dae891a96 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1601,6 +1601,7 @@ void moduleBlockedClientTimedOut(client *c);
 void moduleBlockedClientPipeReadable(aeEventLoop *el, int fd, void *privdata, int mask);
 size_t moduleCount(void);
 void moduleAcquireGIL(void);
+int moduleTryAcquireGIL(void);
 void moduleReleaseGIL(void);
 void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid);
 void moduleCallCommandFilters(client *c);
diff --git a/tests/modules/Makefile b/tests/modules/Makefile
index de7407a84..fad6e55d8 100644
--- a/tests/modules/Makefile
+++ b/tests/modules/Makefile
@@ -23,7 +23,9 @@ TEST_MODULES = \
     scan.so \
     datatype.so \
     auth.so \
-    keyspace_events.so
+    keyspace_events.so \
+    blockedclient.so
+
 
 .PHONY: all
 
diff --git a/tests/modules/blockedclient.c b/tests/modules/blockedclient.c
new file mode 100644
index 000000000..9d59114a1
--- /dev/null
+++ b/tests/modules/blockedclient.c
@@ -0,0 +1,82 @@
+#define REDISMODULE_EXPERIMENTAL_API
+#include "redismodule.h"
+#include <assert.h>
+#include <stdio.h>
+#include <pthread.h>
+
+#define UNUSED(V) ((void) V)
+
+void *sub_worker(void *arg) {
+    // Get Redis module context
+    RedisModuleCtx *ctx = (RedisModuleCtx *)arg;
+
+    // Try acquiring GIL
+    int res = RedisModule_ThreadSafeContextTryLock(ctx);
+
+    // GIL is already taken by the calling thread expecting to fail.
+    assert(res != REDISMODULE_OK);
+
+    return NULL;
+}
+
+void *worker(void *arg) {
+    // Retrieve blocked client
+    RedisModuleBlockedClient *bc = (RedisModuleBlockedClient *)arg;
+
+    // Get Redis module context
+    RedisModuleCtx *ctx = RedisModule_GetThreadSafeContext(bc);
+
+    // Acquire GIL
+    RedisModule_ThreadSafeContextLock(ctx);
+
+    // Create another thread which will try to acquire the GIL
+    pthread_t tid;
+    int res = pthread_create(&tid, NULL, sub_worker, ctx);
+    assert(res == 0);
+
+    // Wait for thread
+    pthread_join(tid, NULL);
+
+    // Release GIL
+    RedisModule_ThreadSafeContextUnlock(ctx);
+
+    // Reply to client
+    RedisModule_ReplyWithSimpleString(ctx, "OK");
+
+    // Unblock client
+    RedisModule_UnblockClient(bc, NULL);
+
+    return NULL;
+}
+
+int acquire_gil(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    UNUSED(argv);
+    UNUSED(argc);
+
+    /* This command handler tries to acquire the GIL twice
+     * once in the worker thread using "RedisModule_ThreadSafeContextLock"
+     * second in the sub-worker thread
+     * using "RedisModule_ThreadSafeContextTryLock"
+     * as the GIL is already locked. */
+    RedisModuleBlockedClient *bc = RedisModule_BlockClient(ctx, NULL, NULL, NULL, 0);
+
+    pthread_t tid;
+    int res = pthread_create(&tid, NULL, worker, bc);
+    assert(res == 0);
+
+    return REDISMODULE_OK;
+}
+
+int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+
+    if (RedisModule_Init(ctx, "blockedclient", 1, REDISMODULE_APIVER_1)== REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx, "acquire_gil", acquire_gil, "", 0, 0, 0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    return REDISMODULE_OK;
+}
diff --git a/tests/unit/moduleapi/blockedclient.tcl b/tests/unit/moduleapi/blockedclient.tcl
new file mode 100644
index 000000000..d093a0297
--- /dev/null
+++ b/tests/unit/moduleapi/blockedclient.tcl
@@ -0,0 +1,11 @@
+# source tests/support/util.tcl
+
+set testmodule [file normalize tests/modules/blockedclient.so]
+
+start_server {tags {"modules"}} {
+    r module load $testmodule
+
+    test {Locked GIL acquisition} {
+        assert_match "OK" [r acquire_gil]
+    }
+}

From 0666267d2771b1a46cdf36eef27d8a7a393c0c7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9D=A8=E5=8D=9A=E4=B8=9C?= <bodong.ybd@alibaba-inc.com>
Date: Wed, 9 Sep 2020 22:13:35 +0800
Subject: [PATCH 0648/1098] Tests: Add aclfile load and save tests (#7765)

improves test coverage
---
 tests/assets/user.acl |  2 ++
 tests/unit/acl.tcl    | 39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 tests/assets/user.acl

diff --git a/tests/assets/user.acl b/tests/assets/user.acl
new file mode 100644
index 000000000..2f065dab6
--- /dev/null
+++ b/tests/assets/user.acl
@@ -0,0 +1,2 @@
+user alice on allcommands allkeys >alice
+user bob on -@all +@set +acl ~set* >bob
\ No newline at end of file
diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl
index e81280995..381f2f95f 100644
--- a/tests/unit/acl.tcl
+++ b/tests/unit/acl.tcl
@@ -261,3 +261,42 @@ start_server {tags {"acl"}} {
         assert_match "*Unknown subcommand or wrong number of arguments*" $e
     }
 }
+
+set server_path [tmpdir "server.acl"]
+exec cp -f tests/assets/user.acl $server_path
+start_server [list overrides [list "dir" $server_path "aclfile" "user.acl"]] {
+    # user alice on allcommands allkeys >alice
+    # user bob on -@all +@set +acl ~set* >bob
+
+    test "Alice: can excute all command" {
+        r AUTH alice alice
+        assert_equal "alice" [r acl whoami]
+        r SET key value
+    }
+
+    test "Bob: just excute @set and acl command" {
+        r AUTH bob bob
+        assert_equal "bob" [r acl whoami]
+        assert_equal "3" [r sadd set 1 2 3]
+        catch {r SET key value} e
+        set e
+    } {*NOPERM*}
+
+    test "ACL load and save" {
+        r ACL setuser eve +get allkeys >eve on
+        r ACL save
+
+        # ACL load will free user and kill clients
+        r ACL load
+        catch {r ACL LIST} e
+        assert_match {*I/O error*} $e
+
+        reconnect
+        r AUTH alice alice
+        r SET key value
+        r AUTH eve eve
+        r GET key
+        catch {r SET key value} e
+        set e
+    } {*NOPERM*}
+}

From b2a73c404bf277bac287c72494a4c4cd2ba02f8c Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 9 Sep 2020 18:58:06 +0300
Subject: [PATCH 0649/1098] Tests: fix oom-score-adj false positives. (#7772)

The key save delay is too short and on certain systems the child process
is gone before we have a chance to inspect it.
---
 tests/unit/oom-score-adj.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/oom-score-adj.tcl b/tests/unit/oom-score-adj.tcl
index 8eb09a993..cf671fe6a 100644
--- a/tests/unit/oom-score-adj.tcl
+++ b/tests/unit/oom-score-adj.tcl
@@ -35,7 +35,7 @@ if {$system_name eq {linux}} {
 
             # Check child process
             r set key-a value-a
-            r config set rdb-key-save-delay 100000
+            r config set rdb-key-save-delay 1000000
             r bgsave
 
             set child_pid [get_child_pid 0]

From de8d32023089333d6b05ac361b9c2f2ecc386868 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 10 Sep 2020 10:09:13 +0300
Subject: [PATCH 0650/1098] Fix RESP3 response for HKEYS/HVALS on non-existing
 key (#7781)

---
 src/t_hash.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/t_hash.c b/src/t_hash.c
index e6f7b6be9..8421223cd 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -766,7 +766,9 @@ void genericHgetallCommand(client *c, int flags) {
     hashTypeIterator *hi;
     int length, count = 0;
 
-    if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptymap[c->resp]))
+    robj *emptyResp = (flags & OBJ_HASH_KEY && flags & OBJ_HASH_VALUE) ?
+        shared.emptymap[c->resp] : shared.emptyarray;
+    if ((o = lookupKeyReadOrReply(c,c->argv[1],emptyResp))
         == NULL || checkType(c,o,OBJ_HASH)) return;
 
     /* We return a map if the user requested keys and values, like in the

From 8ea131fc805315d79482496fddaa56ec202f0f6d Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 10 Sep 2020 10:22:16 +0300
Subject: [PATCH 0651/1098] Fix leak in new blockedclient module API test
 (#7780)

---
 tests/modules/blockedclient.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/modules/blockedclient.c b/tests/modules/blockedclient.c
index 9d59114a1..ca98281a4 100644
--- a/tests/modules/blockedclient.c
+++ b/tests/modules/blockedclient.c
@@ -46,6 +46,9 @@ void *worker(void *arg) {
     // Unblock client
     RedisModule_UnblockClient(bc, NULL);
 
+    // Free the Redis module context
+    RedisModule_FreeThreadSafeContext(ctx);
+
     return NULL;
 }
 

From 1c71038540f8877adfd5eb2b6a6013a1a761bc6c Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 10 Sep 2020 13:43:38 +0300
Subject: [PATCH 0652/1098] Squash merging 125 typo/grammar/comment/doc PRs
 (#7773)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

List of squashed commits or PRs
===============================

commit 66801ea
Author: hwware <wen.hui.ware@gmail.com>
Date:   Mon Jan 13 00:54:31 2020 -0500

    typo fix in acl.c

commit 46f55db
Author: Itamar Haber <itamar@redislabs.com>
Date:   Sun Sep 6 18:24:11 2020 +0300

    Updates a couple of comments

    Specifically:

    * RM_AutoMemory completed instead of pointing to docs
    * Updated link to custom type doc

commit 61a2aa0
Author: xindoo <xindoo@qq.com>
Date:   Tue Sep 1 19:24:59 2020 +0800

    Correct errors in code comments

commit a5871d1
Author: yz1509 <pro-756@qq.com>
Date:   Tue Sep 1 18:36:06 2020 +0800

    fix typos in module.c

commit 41eede7
Author: bookug <bookug@qq.com>
Date:   Sat Aug 15 01:11:33 2020 +0800

    docs: fix typos in comments

commit c303c84
Author: lazy-snail <ws.niu@outlook.com>
Date:   Fri Aug 7 11:15:44 2020 +0800

    fix spelling in redis.conf

commit 1eb76bf
Author: zhujian <zhujianxyz@gmail.com>
Date:   Thu Aug 6 15:22:10 2020 +0800

    add a missing 'n' in comment

commit 1530ec2
Author: Daniel Dai <764122422@qq.com>
Date:   Mon Jul 27 00:46:35 2020 -0400

    fix spelling in tracking.c

commit e517b31
Author: Hunter-Chen <huntcool001@gmail.com>
Date:   Fri Jul 17 22:33:32 2020 +0800

    Update redis.conf

    Co-authored-by: Itamar Haber <itamar@redislabs.com>

commit c300eff
Author: Hunter-Chen <huntcool001@gmail.com>
Date:   Fri Jul 17 22:33:23 2020 +0800

    Update redis.conf

    Co-authored-by: Itamar Haber <itamar@redislabs.com>

commit 4c058a8
Author: 陈浩鹏 <chenhaopeng@heytea.com>
Date:   Thu Jun 25 19:00:56 2020 +0800

    Grammar fix and clarification

commit 5fcaa81
Author: bodong.ybd <bodong.ybd@alibaba-inc.com>
Date:   Fri Jun 19 10:09:00 2020 +0800

    Fix typos

commit 4caca9a
Author: Pruthvi P <pruthvi@ixigo.com>
Date:   Fri May 22 00:33:22 2020 +0530

    Fix typo eviciton => eviction

commit b2a25f6
Author: Brad Dunbar <dunbarb2@gmail.com>
Date:   Sun May 17 12:39:59 2020 -0400

    Fix a typo.

commit 12842ae
Author: hwware <wen.hui.ware@gmail.com>
Date:   Sun May 3 17:16:59 2020 -0400

    fix spelling in redis conf

commit ddba07c
Author: Chris Lamb <chris@chris-lamb.co.uk>
Date:   Sat May 2 23:25:34 2020 +0100

    Correct a "conflicts" spelling error.

commit 8fc7bf2
Author: Nao YONASHIRO <yonashiro@r.recruit.co.jp>
Date:   Thu Apr 30 10:25:27 2020 +0900

    docs: fix EXPIRE_FAST_CYCLE_DURATION to ACTIVE_EXPIRE_CYCLE_FAST_DURATION

commit 9b2b67a
Author: Brad Dunbar <dunbarb2@gmail.com>
Date:   Fri Apr 24 11:46:22 2020 -0400

    Fix a typo.

commit 0746f10
Author: devilinrust <63737265+devilinrust@users.noreply.github.com>
Date:   Thu Apr 16 00:17:53 2020 +0200

    Fix typos in server.c

commit 92b588d
Author: benjessop12 <56115861+benjessop12@users.noreply.github.com>
Date:   Mon Apr 13 13:43:55 2020 +0100

    Fix spelling mistake in lazyfree.c

commit 1da37aa
Merge: 2d4ba28 af347a8
Author: hwware <wen.hui.ware@gmail.com>
Date:   Thu Mar 5 22:41:31 2020 -0500

    Merge remote-tracking branch 'upstream/unstable' into expiretypofix

commit 2d4ba28
Author: hwware <wen.hui.ware@gmail.com>
Date:   Mon Mar 2 00:09:40 2020 -0500

    fix typo in expire.c

commit 1a746f7
Author: SennoYuki <minakami1yuki@gmail.com>
Date:   Thu Feb 27 16:54:32 2020 +0800

    fix typo

commit 8599b1a
Author: dongheejeong <donghee950403@gmail.com>
Date:   Sun Feb 16 20:31:43 2020 +0000

    Fix typo in server.c

commit f38d4e8
Author: hwware <wen.hui.ware@gmail.com>
Date:   Sun Feb 2 22:58:38 2020 -0500

    fix typo in evict.c

commit fe143fc
Author: Leo Murillo <leonardo.murillo@gmail.com>
Date:   Sun Feb 2 01:57:22 2020 -0600

    Fix a few typos in redis.conf

commit 1ab4d21
Author: viraja1 <anchan.viraj@gmail.com>
Date:   Fri Dec 27 17:15:58 2019 +0530

    Fix typo in Latency API docstring

commit ca1f70e
Author: gosth <danxuedexing@qq.com>
Date:   Wed Dec 18 15:18:02 2019 +0800

    fix typo in sort.c

commit a57c06b
Author: ZYunH <zyunhjob@163.com>
Date:   Mon Dec 16 22:28:46 2019 +0800

    fix-zset-typo

commit b8c92b5
Author: git-hulk <hulk.website@gmail.com>
Date:   Mon Dec 16 15:51:42 2019 +0800

    FIX: typo in cluster.c, onformation->information

commit 9dd981c
Author: wujm2007 <jim.wujm@gmail.com>
Date:   Mon Dec 16 09:37:52 2019 +0800

    Fix typo

commit e132d7a
Author: Sebastien Williams-Wynn <s.williamswynn.mail@gmail.com>
Date:   Fri Nov 15 00:14:07 2019 +0000

    Minor typo change

commit 47f44d5
Author: happynote3966 <01ssrmikururudevice01@gmail.com>
Date:   Mon Nov 11 22:08:48 2019 +0900

    fix comment typo in redis-cli.c

commit b8bdb0d
Author: fulei <fulei@kuaishou.com>
Date:   Wed Oct 16 18:00:17 2019 +0800

    Fix a spelling mistake of comments  in defragDictBucketCallback

commit 0def46a
Author: fulei <fulei@kuaishou.com>
Date:   Wed Oct 16 13:09:27 2019 +0800

    fix some spelling mistakes of comments in defrag.c

commit f3596fd
Author: Phil Rajchgot <tophil@outlook.com>
Date:   Sun Oct 13 02:02:32 2019 -0400

    Typo and grammar fixes

    Redis and its documentation are great -- just wanted to submit a few corrections in the spirit of Hacktoberfest. Thanks for all your work on this project. I use it all the time and it works beautifully.

commit 2b928cd
Author: KangZhiDong <worldkzd@gmail.com>
Date:   Sun Sep 1 07:03:11 2019 +0800

    fix typos

commit 33aea14
Author: Axlgrep <axlgrep@gmail.com>
Date:   Tue Aug 27 11:02:18 2019 +0800

    Fixed eviction spelling issues

commit e282a80
Author: Simen Flatby <simen@oms.no>
Date:   Tue Aug 20 15:25:51 2019 +0200

    Update comments to reflect prop name

    In the comments the prop is referenced as replica-validity-factor,
    but it is really named cluster-replica-validity-factor.

commit 74d1f9a
Author: Jim Green <jimgreen2013@qq.com>
Date:   Tue Aug 20 20:00:31 2019 +0800

    fix comment error, the code is ok

commit eea1407
Author: Liao Tonglang <liaotonglang@gmail.com>
Date:   Fri May 31 10:16:18 2019 +0800

    typo fix

    fix cna't to can't

commit 0da553c
Author: KAWACHI Takashi <tkawachi@gmail.com>
Date:   Wed Jul 17 00:38:16 2019 +0900

    Fix typo

commit 7fc8fb6
Author: Michael Prokop <mika@grml.org>
Date:   Tue May 28 17:58:42 2019 +0200

    Typo fixes

    s/familar/familiar/
    s/compatiblity/compatibility/
    s/ ot / to /
    s/itsef/itself/

commit 5f46c9d
Author: zhumoing <34539422+zhumoing@users.noreply.github.com>
Date:   Tue May 21 21:16:50 2019 +0800

    typo-fixes

    typo-fixes

commit 321dfe1
Author: wxisme <850885154@qq.com>
Date:   Sat Mar 16 15:10:55 2019 +0800

    typo fix

commit b4fb131
Merge: 267e0e6 3df1eb8
Author: Nikitas Bastas <nikitasbst@gmail.com>
Date:   Fri Feb 8 22:55:45 2019 +0200

    Merge branch 'unstable' of antirez/redis into unstable

commit 267e0e6
Author: Nikitas Bastas <nikitasbst@gmail.com>
Date:   Wed Jan 30 21:26:04 2019 +0200

    Minor typo fix

commit 30544e7
Author: inshal96 <39904558+inshal96@users.noreply.github.com>
Date:   Fri Jan 4 16:54:50 2019 +0500

    remove an extra 'a' in the comments

commit 337969d
Author: BrotherGao <yangdongheng11@gmail.com>
Date:   Sat Dec 29 12:37:29 2018 +0800

    fix typo in redis.conf

commit 9f4b121
Merge: 423a030 e504583
Author: BrotherGao <yangdongheng@xiaomi.com>
Date:   Sat Dec 29 11:41:12 2018 +0800

    Merge branch 'unstable' of antirez/redis into unstable

commit 423a030
Merge: 42b02b7 46a51cd
Author: 杨东衡 <yangdongheng@xiaomi.com>
Date:   Tue Dec 4 23:56:11 2018 +0800

    Merge branch 'unstable' of antirez/redis into unstable

commit 42b02b7
Merge: 68c0e6e b8febe6
Author: Dongheng Yang <yangdongheng11@gmail.com>
Date:   Sun Oct 28 15:54:23 2018 +0800

    Merge pull request #1 from antirez/unstable

    update local data

commit 714b589
Author: Christian <crifei93@gmail.com>
Date:   Fri Dec 28 01:17:26 2018 +0100

    fix typo "resulution"

commit e23259d
Author: garenchan <1412950785@qq.com>
Date:   Wed Dec 26 09:58:35 2018 +0800

    fix typo: segfauls -> segfault

commit a9359f8
Author: xjp <jianping_xie@aliyun.com>
Date:   Tue Dec 18 17:31:44 2018 +0800

    Fixed REDISMODULE_H spell bug

commit a12c3e4
Author: jdiaz <jrd.palacios@gmail.com>
Date:   Sat Dec 15 23:39:52 2018 -0600

    Fixes hyperloglog hash function comment block description

commit 770eb11
Author: 林上耀 <1210tom@163.com>
Date:   Sun Nov 25 17:16:10 2018 +0800

    fix typo

commit fd97fbb
Author: Chris Lamb <chris@chris-lamb.co.uk>
Date:   Fri Nov 23 17:14:01 2018 +0100

    Correct "unsupported" typo.

commit a85522d
Author: Jungnam Lee <jungnam.lee@oracle.com>
Date:   Thu Nov 8 23:01:29 2018 +0900

    fix typo in test comments

commit ade8007
Author: Arun Kumar <palerdot@users.noreply.github.com>
Date:   Tue Oct 23 16:56:35 2018 +0530

    Fixed grammatical typo

    Fixed typo for word 'dictionary'

commit 869ee39
Author: Hamid Alaei <hamid.a85@gmail.com>
Date:   Sun Aug 12 16:40:02 2018 +0430

    fix documentations: (ThreadSafeContextStart/Stop -> ThreadSafeContextLock/Unlock), minor typo

commit f89d158
Author: Mayank Jain <mayankjain255@gmail.com>
Date:   Tue Jul 31 23:01:21 2018 +0530

    Updated README.md with some spelling corrections.

    Made correction in spelling of some misspelled words.

commit 892198e
Author: dsomeshwar <someshwar.dhayalan@gmail.com>
Date:   Sat Jul 21 23:23:04 2018 +0530

    typo fix

commit 8a4d780
Author: Itamar Haber <itamar@redislabs.com>
Date:   Mon Apr 30 02:06:52 2018 +0300

    Fixes some typos

commit e3acef6
Author: Noah Rosamilia <ivoahivoah@gmail.com>
Date:   Sat Mar 3 23:41:21 2018 -0500

    Fix typo in /deps/README.md

commit 04442fb
Author: WuYunlong <xzsyeb@126.com>
Date:   Sat Mar 3 10:32:42 2018 +0800

    Fix typo in readSyncBulkPayload() comment.

commit 9f36880
Author: WuYunlong <xzsyeb@126.com>
Date:   Sat Mar 3 10:20:37 2018 +0800

    replication.c comment: run_id -> replid.

commit f866b4a
Author: Francesco 'makevoid' Canessa <makevoid@gmail.com>
Date:   Thu Feb 22 22:01:56 2018 +0000

    fix comment typo in server.c

commit 0ebc69b
Author: 줍 <jubee0124@gmail.com>
Date:   Mon Feb 12 16:38:48 2018 +0900

    Fix typo in redis.conf

    Fix `five behaviors` to `eight behaviors` in [this sentence ](antirez/redis@unstable/redis.conf#L564)

commit b50a620
Author: martinbroadhurst <martinbroadhurst@users.noreply.github.com>
Date:   Thu Dec 28 12:07:30 2017 +0000

    Fix typo in valgrind.sup

commit 7d8f349
Author: Peter Boughton <peter@sorcerersisle.com>
Date:   Mon Nov 27 19:52:19 2017 +0000

    Update CONTRIBUTING; refer doc updates to redis-doc repo.

commit 02dec7e
Author: Klauswk <klauswk1@hotmail.com>
Date:   Tue Oct 24 16:18:38 2017 -0200

    Fix typo in comment

commit e1efbc8
Author: chenshi <baiwfg2@gmail.com>
Date:   Tue Oct 3 18:26:30 2017 +0800

    Correct two spelling errors of comments

commit 93327d8
Author: spacewander <spacewanderlzx@gmail.com>
Date:   Wed Sep 13 16:47:24 2017 +0800

    Update the comment for OBJ_ENCODING_EMBSTR_SIZE_LIMIT's value

    The value of OBJ_ENCODING_EMBSTR_SIZE_LIMIT is 44 now instead of 39.

commit 63d361f
Author: spacewander <spacewanderlzx@gmail.com>
Date:   Tue Sep 12 15:06:42 2017 +0800

    Fix <prevlen> related doc in ziplist.c

    According to the definition of ZIP_BIG_PREVLEN and other related code,
    the guard of single byte <prevlen> should be 254 instead of 255.

commit ebe228d
Author: hanael80 <hanael80@gmail.com>
Date:   Tue Aug 15 09:09:40 2017 +0900

    Fix typo

commit 6b696e6
Author: Matt Robenolt <matt@ydekproductions.com>
Date:   Mon Aug 14 14:50:47 2017 -0700

    Fix typo in LATENCY DOCTOR output

commit a2ec6ae
Author: caosiyang <caosiyang@qiyi.com>
Date:   Tue Aug 15 14:15:16 2017 +0800

    Fix a typo: form => from

commit 3ab7699
Author: caosiyang <caosiyang@qiyi.com>
Date:   Thu Aug 10 18:40:33 2017 +0800

    Fix a typo: replicationFeedSlavesFromMaster() => replicationFeedSlavesFromMasterStream()

commit 72d43ef
Author: caosiyang <caosiyang@qiyi.com>
Date:   Tue Aug 8 15:57:25 2017 +0800

    fix a typo: servewr => server

commit 707c958
Author: Bo Cai <charpty@gmail.com>
Date:   Wed Jul 26 21:49:42 2017 +0800

    redis-cli.c typo: conut -> count.

    Signed-off-by: Bo Cai <charpty@gmail.com>

commit b9385b2
Author: JackDrogon <jack.xsuperman@gmail.com>
Date:   Fri Jun 30 14:22:31 2017 +0800

    Fix some spell problems

commit 20d9230
Author: akosel <aaronjkosel@gmail.com>
Date:   Sun Jun 4 19:35:13 2017 -0500

    Fix typo

commit b167bfc
Author: Krzysiek Witkowicz <krzysiekwitkowicz@gmail.com>
Date:   Mon May 22 21:32:27 2017 +0100

    Fix #4008 small typo in comment

commit 2b78ac8
Author: Jake Clarkson <jacobwclarkson@gmail.com>
Date:   Wed Apr 26 15:49:50 2017 +0100

    Correct typo in tests/unit/hyperloglog.tcl

commit b0f1cdb
Author: Qi Luo <qiluo-msft@users.noreply.github.com>
Date:   Wed Apr 19 14:25:18 2017 -0700

    Fix typo

commit a90b0f9
Author: charsyam <charsyam@naver.com>
Date:   Thu Mar 16 18:19:53 2017 +0900

    fix typos

    fix typos

    fix typos

commit 8430a79
Author: Richard Hart <richardhart92@gmail.com>
Date:   Mon Mar 13 22:17:41 2017 -0400

    Fixed log message typo in listenToPort.

commit 481a1c2
Author: Vinod Kumar <kumar003vinod@gmail.com>
Date:   Sun Jan 15 23:04:51 2017 +0530

    src/db.c: Correct "save" -> "safe" typo

commit 586b4d3
Author: wangshaonan <wshn13@gmail.com>
Date:   Wed Dec 21 20:28:27 2016 +0800

    Fix typo they->the in helloworld.c

commit c1c4b5e
Author: Jenner <hypxm@qq.com>
Date:   Mon Dec 19 16:39:46 2016 +0800

    typo error

commit 1ee1a3f
Author: tielei <43289893@qq.com>
Date:   Mon Jul 18 13:52:25 2016 +0800

    fix some comments

commit 11a41fb
Author: Otto Kekäläinen <otto@seravo.fi>
Date:   Sun Jul 3 10:23:55 2016 +0100

    Fix spelling in documentation and comments

commit 5fb5d82
Author: francischan <f1ancis621@gmail.com>
Date:   Tue Jun 28 00:19:33 2016 +0800

    Fix outdated comments about redis.c file.
    It should now refer to server.c file.

commit 6b254bc
Author: lmatt-bit <lmatt123n@gmail.com>
Date:   Thu Apr 21 21:45:58 2016 +0800

    Refine the comment of dictRehashMilliseconds func

SLAVECONF->REPLCONF in comment - by andyli029

commit ee9869f
Author: clark.kang <charsyam@naver.com>
Date:   Tue Mar 22 11:09:51 2016 +0900

    fix typos

commit f7b3b11
Author: Harisankar H <harisankarh@gmail.com>
Date:   Wed Mar 9 11:49:42 2016 +0530

    Typo correction: "faield" --> "failed"

    Typo correction: "faield" --> "failed"

commit 3fd40fc
Author: Itamar Haber <itamar@redislabs.com>
Date:   Thu Feb 25 10:31:51 2016 +0200

    Fixes a typo in comments

commit 621c160
Author: Prayag Verma <prayag.verma@gmail.com>
Date:   Mon Feb 1 12:36:20 2016 +0530

    Fix typo in Readme.md

    Spelling mistakes -
    `eviciton` > `eviction`
    `familar` > `familiar`

commit d7d07d6
Author: WonCheol Lee <toctoc21c@gmail.com>
Date:   Wed Dec 30 15:11:34 2015 +0900

    Typo fixed

commit a4dade7
Author: Felix Bünemann <buenemann@louis.info>
Date:   Mon Dec 28 11:02:55 2015 +0100

    [ci skip] Improve supervised upstart config docs

    This mentions that "expect stop" is required for supervised upstart
    to work correctly. See http://upstart.ubuntu.com/cookbook/#expect-stop
    for an explanation.

commit d9caba9
Author: daurnimator <quae@daurnimator.com>
Date:   Mon Dec 21 18:30:03 2015 +1100

    README: Remove trailing whitespace

commit 72d42e5
Author: daurnimator <quae@daurnimator.com>
Date:   Mon Dec 21 18:29:32 2015 +1100

    README: Fix typo. th => the

commit dd6e957
Author: daurnimator <quae@daurnimator.com>
Date:   Mon Dec 21 18:29:20 2015 +1100

    README: Fix typo. familar => familiar

commit 3a12b23
Author: daurnimator <quae@daurnimator.com>
Date:   Mon Dec 21 18:28:54 2015 +1100

    README: Fix typo. eviciton => eviction

commit 2d1d03b
Author: daurnimator <quae@daurnimator.com>
Date:   Mon Dec 21 18:21:45 2015 +1100

    README: Fix typo. sever => server

commit 3973b06
Author: Itamar Haber <itamar@garantiadata.com>
Date:   Sat Dec 19 17:01:20 2015 +0200

    Typo fix

commit 4f2e460
Author: Steve Gao <fu@2token.com>
Date:   Fri Dec 4 10:22:05 2015 +0800

    Update README - fix typos

commit b21667c
Author: binyan <binbin.yan@nokia.com>
Date:   Wed Dec 2 22:48:37 2015 +0800

    delete redundancy color judge in sdscatcolor

commit 88894c7
Author: binyan <binbin.yan@nokia.com>
Date:   Wed Dec 2 22:14:42 2015 +0800

    the example output shoule be HelloWorld

commit 2763470
Author: binyan <binbin.yan@nokia.com>
Date:   Wed Dec 2 17:41:39 2015 +0800

    modify error word keyevente

    Signed-off-by: binyan <binbin.yan@nokia.com>

commit 0847b3d
Author: Bruno Martins <bscmartins@gmail.com>
Date:   Wed Nov 4 11:37:01 2015 +0000

    typo

commit bbb9e9e
Author: dawedawe <dawedawe@gmx.de>
Date:   Fri Mar 27 00:46:41 2015 +0100

    typo: zimap -> zipmap

commit 5ed297e
Author: Axel Advento <badwolf.bloodseeker.rev@gmail.com>
Date:   Tue Mar 3 15:58:29 2015 +0800

    Fix 'salve' typos to 'slave'

commit edec9d6
Author: LudwikJaniuk <ludvig.janiuk@gmail.com>
Date:   Wed Jun 12 14:12:47 2019 +0200

    Update README.md

    Co-Authored-By: Qix <Qix-@users.noreply.github.com>

commit 692a7af
Author: LudwikJaniuk <ludvig.janiuk@gmail.com>
Date:   Tue May 28 14:32:04 2019 +0200

    grammar

commit d962b0a
Author: Nick Frost <nickfrostatx@gmail.com>
Date:   Wed Jul 20 15:17:12 2016 -0700

    Minor grammar fix

commit 24fff01aaccaf5956973ada8c50ceb1462e211c6 (typos)
Author: Chad Miller <chadm@squareup.com>
Date:   Tue Sep 8 13:46:11 2020 -0400

    Fix faulty comment about operation of unlink()

commit 3cd5c1f3326c52aa552ada7ec797c6bb16452355
Author: Kevin <kevin.xgr@gmail.com>
Date:   Wed Nov 20 00:13:50 2019 +0800

    Fix typo in server.c.

From a83af59 Mon Sep 17 00:00:00 2001
From: wuwo <wuwo@wacai.com>
Date: Fri, 17 Mar 2017 20:37:45 +0800
Subject: [PATCH] falure to failure

From c961896 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B7=A6=E6=87=B6?= <veficos@gmail.com>
Date: Sat, 27 May 2017 15:33:04 +0800
Subject: [PATCH] fix typo

From e600ef2 Mon Sep 17 00:00:00 2001
From: "rui.zou" <rui.zou@yunify.com>
Date: Sat, 30 Sep 2017 12:38:15 +0800
Subject: [PATCH] fix a typo

From c7d07fa Mon Sep 17 00:00:00 2001
From: Alexandre Perrin <alex@kaworu.ch>
Date: Thu, 16 Aug 2018 10:35:31 +0200
Subject: [PATCH] deps README.md typo

From b25cb67 Mon Sep 17 00:00:00 2001
From: Guy Korland <gkorland@gmail.com>
Date: Wed, 26 Sep 2018 10:55:37 +0300
Subject: [PATCH 1/2] fix typos in header

From ad28ca6 Mon Sep 17 00:00:00 2001
From: Guy Korland <gkorland@gmail.com>
Date: Wed, 26 Sep 2018 11:02:36 +0300
Subject: [PATCH 2/2] fix typos

commit 34924cdedd8552466fc22c1168d49236cb7ee915
Author: Adrian Lynch <adi_ady_ade@hotmail.com>
Date:   Sat Apr 4 21:59:15 2015 +0100

    Typos fixed

commit fd2a1e7
Author: Jan <jsteemann@users.noreply.github.com>
Date:   Sat Oct 27 19:13:01 2018 +0200

    Fix typos

    Fix typos

commit e14e47c1a234b53b0e103c5f6a1c61481cbcbb02
Author: Andy Lester <andy@petdance.com>
Date:   Fri Aug 2 22:30:07 2019 -0500

    Fix multiple misspellings of "following"

commit 79b948ce2dac6b453fe80995abbcaac04c213d5a
Author: Andy Lester <andy@petdance.com>
Date:   Fri Aug 2 22:24:28 2019 -0500

    Fix misspelling of create-cluster

commit 1fffde52666dc99ab35efbd31071a4c008cb5a71
Author: Andy Lester <andy@petdance.com>
Date:   Wed Jul 31 17:57:56 2019 -0500

    Fix typos

commit 204c9ba9651e9e05fd73936b452b9a30be456cfe
Author: Xiaobo Zhu <xiaobo.zhu@shopee.com>
Date:   Tue Aug 13 22:19:25 2019 +0800

    fix typos

Squashed commit of the following:

commit 1d9aaf8
Author: danmedani <danmedani@gmail.com>
Date:   Sun Aug 2 11:40:26 2015 -0700

README typo fix.

Squashed commit of the following:

commit 32bfa7c
Author: Erik Dubbelboer <erik@dubbelboer.com>
Date:   Mon Jul 6 21:15:08 2015 +0200

Fixed grammer

Squashed commit of the following:

commit b24f69c
Author: Sisir Koppaka <sisir.koppaka@gmail.com>
Date:   Mon Mar 2 22:38:45 2015 -0500

utils/hashtable/rehashing.c: Fix typos

Squashed commit of the following:

commit 4e04082
Author: Erik Dubbelboer <erik@dubbelboer.com>
Date:   Mon Mar 23 08:22:21 2015 +0000

Small config file documentation improvements

Squashed commit of the following:

commit acb8773
Author: ctd1500 <ctd1500@gmail.com>
Date:   Fri May 8 01:52:48 2015 -0700

Typo and grammar fixes in readme

commit 2eb75b6
Author: ctd1500 <ctd1500@gmail.com>
Date:   Fri May 8 01:36:18 2015 -0700

fixed redis.conf comment

Squashed commit of the following:

commit a8249a2
Author: Masahiko Sawada <sawada.mshk@gmail.com>
Date:   Fri Dec 11 11:39:52 2015 +0530

Revise correction of typos.

Squashed commit of the following:

commit 3c02028
Author: zhaojun11 <zhaojun11@jd.com>
Date:   Wed Jan 17 19:05:28 2018 +0800

Fix typos include two code typos in cluster.c and latency.c

Squashed commit of the following:

commit 9dba47c
Author: q191201771 <191201771@qq.com>
Date:   Sat Jan 4 11:31:04 2020 +0800

fix function listCreate comment in adlist.c

Update src/server.c

commit 2c7c2cb536e78dd211b1ac6f7bda00f0f54faaeb
Author: charpty <charpty@gmail.com>
Date:   Tue May 1 23:16:59 2018 +0800

    server.c typo: modules system dictionary type comment

    Signed-off-by: charpty <charpty@gmail.com>

commit a8395323fb63cb59cb3591cb0f0c8edb7c29a680
Author: Itamar Haber <itamar@redislabs.com>
Date:   Sun May 6 00:25:18 2018 +0300

    Updates test_helper.tcl's help with undocumented options

    Specifically:

    * Host
    * Port
    * Client

commit bde6f9ced15755cd6407b4af7d601b030f36d60b
Author: wxisme <850885154@qq.com>
Date:   Wed Aug 8 15:19:19 2018 +0800

    fix comments in deps files

commit 3172474ba991532ab799ee1873439f3402412331
Author: wxisme <850885154@qq.com>
Date:   Wed Aug 8 14:33:49 2018 +0800

    fix some comments

commit 01b6f2b6858b5cf2ce4ad5092d2c746e755f53f0
Author: Thor Juhasz <thor@juhasz.pro>
Date:   Sun Nov 18 14:37:41 2018 +0100

    Minor fixes to comments

    Found some parts a little unclear on a first read, which prompted me to have a better look at the file and fix some minor things I noticed.
    Fixing minor typos and grammar. There are no changes to configuration options.
    These changes are only meant to help the user better understand the explanations to the various configuration options
---
 CONTRIBUTING                          |   4 +
 README.md                             |  66 +++++++-------
 deps/README.md                        |   8 +-
 deps/linenoise/linenoise.c            |   4 +-
 redis.conf                            | 124 +++++++++++++-------------
 sentinel.conf                         |   2 +-
 src/acl.c                             |   2 +-
 src/adlist.c                          |   9 +-
 src/ae.c                              |   4 +-
 src/ae_evport.c                       |   2 +-
 src/aof.c                             |   4 +-
 src/atomicvar.h                       |   2 +-
 src/bitops.c                          |   6 +-
 src/blocked.c                         |   4 +-
 src/cluster.c                         |  66 +++++++-------
 src/cluster.h                         |   8 +-
 src/config.c                          |   6 +-
 src/config.h                          |   2 +-
 src/connection.h                      |   4 +-
 src/db.c                              |   8 +-
 src/debug.c                           |   6 +-
 src/defrag.c                          |  30 +++----
 src/dict.c                            |   8 +-
 src/endianconv.c                      |   2 +-
 src/evict.c                           |   8 +-
 src/expire.c                          |   8 +-
 src/geo.c                             |   6 +-
 src/geohash_helper.c                  |   4 +-
 src/hyperloglog.c                     |  16 ++--
 src/latency.c                         |  12 +--
 src/lazyfree.c                        |   6 +-
 src/listpack.c                        |   2 +-
 src/lolwut.c                          |   4 +-
 src/lolwut5.c                         |   2 +-
 src/lzfP.h                            |   2 +-
 src/module.c                          |  93 ++++++++++---------
 src/modules/hellodict.c               |   2 +-
 src/modules/helloworld.c              |   4 +-
 src/multi.c                           |   2 +-
 src/networking.c                      |  12 +--
 src/notify.c                          |   2 +-
 src/object.c                          |   4 +-
 src/quicklist.h                       |   4 +-
 src/rax.c                             |  16 ++--
 src/rax.h                             |   4 +-
 src/rdb.c                             |   4 +-
 src/redis-check-rdb.c                 |   2 +-
 src/redis-cli.c                       |  12 +--
 src/redismodule.h                     |   2 +-
 src/replication.c                     |  40 ++++-----
 src/scripting.c                       |  26 +++---
 src/sds.c                             |   6 +-
 src/sentinel.c                        |  26 +++---
 src/server.c                          |  43 +++++----
 src/server.h                          |  12 +--
 src/siphash.c                         |   6 +-
 src/slowlog.c                         |   2 +-
 src/sort.c                            |   2 +-
 src/sparkline.c                       |   2 +-
 src/stream.h                          |   2 +-
 src/t_hash.c                          |   2 +-
 src/t_list.c                          |   6 +-
 src/t_set.c                           |   6 +-
 src/t_stream.c                        |   6 +-
 src/t_string.c                        |   2 +-
 src/t_zset.c                          |   2 +-
 src/tracking.c                        |   2 +-
 src/valgrind.sup                      |   6 +-
 src/ziplist.c                         |  16 ++--
 src/zipmap.c                          |   2 +-
 tests/cluster/tests/04-resharding.tcl |   6 +-
 tests/instances.tcl                   |   2 +-
 tests/integration/replication-4.tcl   |   2 +-
 tests/support/test.tcl                |   2 +-
 tests/test_helper.tcl                 |   7 +-
 tests/unit/expire.tcl                 |   2 +-
 tests/unit/hyperloglog.tcl            |   2 +-
 tests/unit/scripting.tcl              |   2 +-
 utils/create-cluster/README           |   2 +-
 utils/hashtable/README                |   2 +-
 80 files changed, 440 insertions(+), 420 deletions(-)

diff --git a/CONTRIBUTING b/CONTRIBUTING
index 394fe02ad..50d567861 100644
--- a/CONTRIBUTING
+++ b/CONTRIBUTING
@@ -40,6 +40,10 @@ Redis follows a responsible disclosure process:
 4. We push a fix release and your bug can be posted publicly with credit in
    release notes and the version history (and our thanks!)
 
+  Issues and pull requests for documentation belong on the redis-doc repo:
+
+      https://github.com/redis/redis-doc
+
 # How to provide a patch for a new feature
 
 1. If it is a major feature or a semantical change, please don't start coding
diff --git a/README.md b/README.md
index e80b81c52..eec7e4bdd 100644
--- a/README.md
+++ b/README.md
@@ -3,22 +3,22 @@ This README is just a fast *quick start* document. You can find more detailed do
 What is Redis?
 --------------
 
-Redis is often referred as a *data structures* server. What this means is that Redis provides access to mutable data structures via a set of commands, which are sent using a *server-client* model with TCP sockets and a simple protocol. So different processes can query and modify the same data structures in a shared way.
+Redis is often referred to as a *data structures* server. What this means is that Redis provides access to mutable data structures via a set of commands, which are sent using a *server-client* model with TCP sockets and a simple protocol. So different processes can query and modify the same data structures in a shared way.
 
 Data structures implemented into Redis have a few special properties:
 
-* Redis cares to store them on disk, even if they are always served and modified into the server memory. This means that Redis is fast, but that is also non-volatile.
-* Implementation of data structures stress on memory efficiency, so data structures inside Redis will likely use less memory compared to the same data structure modeled using an high level programming language.
-* Redis offers a number of features that are natural to find in a database, like replication, tunable levels of durability, cluster, high availability.
+* Redis cares to store them on disk, even if they are always served and modified into the server memory. This means that Redis is fast, but that it is also non-volatile.
+* The implementation of data structures emphasizes memory efficiency, so data structures inside Redis will likely use less memory compared to the same data structure modelled using a high-level programming language.
+* Redis offers a number of features that are natural to find in a database, like replication, tunable levels of durability, clustering, and high availability.
 
-Another good example is to think of Redis as a more complex version of memcached, where the operations are not just SETs and GETs, but operations to work with complex data types like Lists, Sets, ordered data structures, and so forth.
+Another good example is to think of Redis as a more complex version of memcached, where the operations are not just SETs and GETs, but operations that work with complex data types like Lists, Sets, ordered data structures, and so forth.
 
 If you want to know more, this is a list of selected starting points:
 
 * Introduction to Redis data types. http://redis.io/topics/data-types-intro
 * Try Redis directly inside your browser. http://try.redis.io
 * The full list of Redis commands. http://redis.io/commands
-* There is much more inside the Redis official documentation. http://redis.io/documentation
+* There is much more inside the official Redis documentation. http://redis.io/documentation
 
 Building Redis
 --------------
@@ -29,7 +29,7 @@ and 64 bit systems.
 
 It may compile on Solaris derived systems (for instance SmartOS) but our
 support for this platform is *best effort* and Redis is not guaranteed to
-work as well as in Linux, OSX, and \*BSD there.
+work as well as in Linux, OSX, and \*BSD.
 
 It is as simple as:
 
@@ -63,7 +63,7 @@ installed):
 Fixing build problems with dependencies or cached build options
 ---------
 
-Redis has some dependencies which are included into the `deps` directory.
+Redis has some dependencies which are included in the `deps` directory.
 `make` does not automatically rebuild dependencies even if something in
 the source code of dependencies changes.
 
@@ -90,7 +90,7 @@ with a 64 bit target, or the other way around, you need to perform a
 In case of build errors when trying to build a 32 bit binary of Redis, try
 the following steps:
 
-* Install the packages libc6-dev-i386 (also try g++-multilib).
+* Install the package libc6-dev-i386 (also try g++-multilib).
 * Try using the following command line instead of `make 32bit`:
   `make CFLAGS="-m32 -march=native" LDFLAGS="-m32"`
 
@@ -126,15 +126,15 @@ To build with support for the processor's internal instruction clock, use:
 Verbose build
 -------------
 
-Redis will build with a user friendly colorized output by default.
-If you want to see a more verbose output use the following:
+Redis will build with a user-friendly colorized output by default.
+If you want to see a more verbose output, use the following:
 
     % make V=1
 
 Running Redis
 -------------
 
-To run Redis with the default configuration just type:
+To run Redis with the default configuration, just type:
 
     % cd src
     % ./redis-server
@@ -185,7 +185,7 @@ You can find the list of all the available commands at http://redis.io/commands.
 Installing Redis
 -----------------
 
-In order to install Redis binaries into /usr/local/bin just use:
+In order to install Redis binaries into /usr/local/bin, just use:
 
     % make install
 
@@ -194,8 +194,8 @@ different destination.
 
 Make install will just install binaries in your system, but will not configure
 init scripts and configuration files in the appropriate place. This is not
-needed if you want just to play a bit with Redis, but if you are installing
-it the proper way for a production system, we have a script doing this
+needed if you just want to play a bit with Redis, but if you are installing
+it the proper way for a production system, we have a script that does this
 for Ubuntu and Debian systems:
 
     % cd utils
@@ -213,7 +213,7 @@ You'll be able to stop and start Redis using the script named
 Code contributions
 -----------------
 
-Note: by contributing code to the Redis project in any form, including sending
+Note: By contributing code to the Redis project in any form, including sending
 a pull request via Github, a code fragment or patch via private email or
 public discussion groups, you agree to release your code under the terms
 of the BSD license that you can find in the [COPYING][1] file included in the Redis
@@ -263,7 +263,7 @@ of complexity incrementally.
 
 Note: lately Redis was refactored quite a bit. Function names and file
 names have been changed, so you may find that this documentation reflects the
-`unstable` branch more closely. For instance in Redis 3.0 the `server.c`
+`unstable` branch more closely. For instance, in Redis 3.0 the `server.c`
 and `server.h` files were named `redis.c` and `redis.h`. However the overall
 structure is the same. Keep in mind that all the new developments and pull
 requests should be performed against the `unstable` branch.
@@ -308,7 +308,7 @@ The client structure defines a *connected client*:
 * The `fd` field is the client socket file descriptor.
 * `argc` and `argv` are populated with the command the client is executing, so that functions implementing a given Redis command can read the arguments.
 * `querybuf` accumulates the requests from the client, which are parsed by the Redis server according to the Redis protocol and executed by calling the implementations of the commands the client is executing.
-* `reply` and `buf` are dynamic and static buffers that accumulate the replies the server sends to the client. These buffers are incrementally written to the socket as soon as the file descriptor is writable.
+* `reply` and `buf` are dynamic and static buffers that accumulate the replies the server sends to the client. These buffers are incrementally written to the socket as soon as the file descriptor is writeable.
 
 As you can see in the client structure above, arguments in a command
 are described as `robj` structures. The following is the full `robj`
@@ -341,13 +341,13 @@ This is the entry point of the Redis server, where the `main()` function
 is defined. The following are the most important steps in order to startup
 the Redis server.
 
-* `initServerConfig()` setups the default values of the `server` structure.
+* `initServerConfig()` sets up the default values of the `server` structure.
 * `initServer()` allocates the data structures needed to operate, setup the listening socket, and so forth.
 * `aeMain()` starts the event loop which listens for new connections.
 
 There are two special functions called periodically by the event loop:
 
-1. `serverCron()` is called periodically (according to `server.hz` frequency), and performs tasks that must be performed from time to time, like checking for timedout clients.
+1. `serverCron()` is called periodically (according to `server.hz` frequency), and performs tasks that must be performed from time to time, like checking for timed out clients.
 2. `beforeSleep()` is called every time the event loop fired, Redis served a few requests, and is returning back into the event loop.
 
 Inside server.c you can find code that handles other vital things of the Redis server:
@@ -364,16 +364,16 @@ This file defines all the I/O functions with clients, masters and replicas
 (which in Redis are just special clients):
 
 * `createClient()` allocates and initializes a new client.
-* the `addReply*()` family of functions are used by commands implementations in order to append data to the client structure, that will be transmitted to the client as a reply for a given command executed.
+* the `addReply*()` family of functions are used by command implementations in order to append data to the client structure, that will be transmitted to the client as a reply for a given command executed.
 * `writeToClient()` transmits the data pending in the output buffers to the client and is called by the *writable event handler* `sendReplyToClient()`.
-* `readQueryFromClient()` is the *readable event handler* and accumulates data from read from the client into the query buffer.
+* `readQueryFromClient()` is the *readable event handler* and accumulates data read from the client into the query buffer.
 * `processInputBuffer()` is the entry point in order to parse the client query buffer according to the Redis protocol. Once commands are ready to be processed, it calls `processCommand()` which is defined inside `server.c` in order to actually execute the command.
 * `freeClient()` deallocates, disconnects and removes a client.
 
 aof.c and rdb.c
 ---
 
-As you can guess from the names these files implement the RDB and AOF
+As you can guess from the names, these files implement the RDB and AOF
 persistence for Redis. Redis uses a persistence model based on the `fork()`
 system call in order to create a thread with the same (shared) memory
 content of the main Redis thread. This secondary thread dumps the content
@@ -385,13 +385,13 @@ The implementation inside `aof.c` has additional functions in order to
 implement an API that allows commands to append new commands into the AOF
 file as clients execute them.
 
-The `call()` function defined inside `server.c` is responsible to call
+The `call()` function defined inside `server.c` is responsible for calling
 the functions that in turn will write the commands into the AOF.
 
 db.c
 ---
 
-Certain Redis commands operate on specific data types, others are general.
+Certain Redis commands operate on specific data types; others are general.
 Examples of generic commands are `DEL` and `EXPIRE`. They operate on keys
 and not on their values specifically. All those generic commands are
 defined inside `db.c`.
@@ -399,7 +399,7 @@ defined inside `db.c`.
 Moreover `db.c` implements an API in order to perform certain operations
 on the Redis dataset without directly accessing the internal data structures.
 
-The most important functions inside `db.c` which are used in many commands
+The most important functions inside `db.c` which are used in many command
 implementations are the following:
 
 * `lookupKeyRead()` and `lookupKeyWrite()` are used in order to get a pointer to the value associated to a given key, or `NULL` if the key does not exist.
@@ -417,7 +417,7 @@ The `robj` structure defining Redis objects was already described. Inside
 a basic level, like functions to allocate new objects, handle the reference
 counting and so forth. Notable functions inside this file:
 
-* `incrRefcount()` and `decrRefCount()` are used in order to increment or decrement an object reference count. When it drops to 0 the object is finally freed.
+* `incrRefCount()` and `decrRefCount()` are used in order to increment or decrement an object reference count. When it drops to 0 the object is finally freed.
 * `createObject()` allocates a new object. There are also specialized functions to allocate string objects having a specific content, like `createStringObjectFromLongLong()` and similar functions.
 
 This file also implements the `OBJECT` command.
@@ -441,12 +441,12 @@ replicas, or to continue the replication after a disconnection.
 Other C files
 ---
 
-* `t_hash.c`, `t_list.c`, `t_set.c`, `t_string.c`, `t_zset.c` and `t_stream.c` contains the implementation of the Redis data types. They implement both an API to access a given data type, and the client commands implementations for these data types.
+* `t_hash.c`, `t_list.c`, `t_set.c`, `t_string.c`, `t_zset.c` and `t_stream.c` contains the implementation of the Redis data types. They implement both an API to access a given data type, and the client command implementations for these data types.
 * `ae.c` implements the Redis event loop, it's a self contained library which is simple to read and understand.
 * `sds.c` is the Redis string library, check http://github.com/antirez/sds for more information.
 * `anet.c` is a library to use POSIX networking in a simpler way compared to the raw interface exposed by the kernel.
 * `dict.c` is an implementation of a non-blocking hash table which rehashes incrementally.
-* `scripting.c` implements Lua scripting. It is completely self contained from the rest of the Redis implementation and is simple enough to understand if you are familiar with the Lua API.
+* `scripting.c` implements Lua scripting. It is completely self-contained and isolated from the rest of the Redis implementation and is simple enough to understand if you are familiar with the Lua API.
 * `cluster.c` implements the Redis Cluster. Probably a good read only after being very familiar with the rest of the Redis code base. If you want to read `cluster.c` make sure to read the [Redis Cluster specification][3].
 
 [3]: http://redis.io/topics/cluster-spec
@@ -472,12 +472,12 @@ top comment inside `server.c`.
 After the command operates in some way, it returns a reply to the client,
 usually using `addReply()` or a similar function defined inside `networking.c`.
 
-There are tons of commands implementations inside the Redis source code
-that can serve as examples of actual commands implementations. To write
-a few toy commands can be a good exercise to familiarize with the code base.
+There are tons of command implementations inside the Redis source code
+that can serve as examples of actual commands implementations. Writing
+a few toy commands can be a good exercise to get familiar with the code base.
 
 There are also many other files not described here, but it is useless to
-cover everything. We want to just help you with the first steps.
+cover everything. We just want to help you with the first steps.
 Eventually you'll find your way inside the Redis code base :-)
 
 Enjoy!
diff --git a/deps/README.md b/deps/README.md
index a7f6e1834..c3ec424b6 100644
--- a/deps/README.md
+++ b/deps/README.md
@@ -21,7 +21,7 @@ just following these steps:
 
 1. Remove the jemalloc directory.
 2. Substitute it with the new jemalloc source tree.
-3. Edit the Makefile localted in the same directory as the README you are
+3. Edit the Makefile located in the same directory as the README you are
    reading, and change the --with-version in the Jemalloc configure script
    options with the version you are using. This is required because otherwise
    Jemalloc configuration script is broken and will not work nested in another
@@ -33,7 +33,7 @@ If you want to upgrade Jemalloc while also providing support for
 active defragmentation, in addition to the above steps you need to perform
 the following additional steps:
 
-5. In Jemalloc three, file `include/jemalloc/jemalloc_macros.h.in`, make sure
+5. In Jemalloc tree, file `include/jemalloc/jemalloc_macros.h.in`, make sure
    to add `#define JEMALLOC_FRAG_HINT`.
 6. Implement the function `je_get_defrag_hint()` inside `src/jemalloc.c`. You
    can see how it is implemented in the current Jemalloc source tree shipped
@@ -49,7 +49,7 @@ Hiredis uses the SDS string library, that must be the same version used inside R
 1. Check with diff if hiredis API changed and what impact it could have in Redis.
 2. Make sure that the SDS library inside Hiredis and inside Redis are compatible.
 3. After the upgrade, run the Redis Sentinel test.
-4. Check manually that redis-cli and redis-benchmark behave as expecteed, since we have no tests for CLI utilities currently.
+4. Check manually that redis-cli and redis-benchmark behave as expected, since we have no tests for CLI utilities currently.
 
 Linenoise
 ---
@@ -77,6 +77,6 @@ and our version:
 
 1. Makefile is modified to allow a different compiler than GCC.
 2. We have the implementation source code, and directly link to the following external libraries: `lua_cjson.o`, `lua_struct.o`, `lua_cmsgpack.o` and `lua_bit.o`.
-3. There is a security fix in `ldo.c`, line 498: The check for `LUA_SIGNATURE[0]` is removed in order toa void direct bytecode execution.
+3. There is a security fix in `ldo.c`, line 498: The check for `LUA_SIGNATURE[0]` is removed in order to avoid direct bytecode execution.
 
 
diff --git a/deps/linenoise/linenoise.c b/deps/linenoise/linenoise.c
index cfe51e768..ccf5c5548 100644
--- a/deps/linenoise/linenoise.c
+++ b/deps/linenoise/linenoise.c
@@ -625,7 +625,7 @@ static void refreshMultiLine(struct linenoiseState *l) {
     rpos2 = (plen+l->pos+l->cols)/l->cols; /* current cursor relative row. */
     lndebug("rpos2 %d", rpos2);
 
-    /* Go up till we reach the expected positon. */
+    /* Go up till we reach the expected position. */
     if (rows-rpos2 > 0) {
         lndebug("go-up %d", rows-rpos2);
         snprintf(seq,64,"\x1b[%dA", rows-rpos2);
@@ -767,7 +767,7 @@ void linenoiseEditBackspace(struct linenoiseState *l) {
     }
 }
 
-/* Delete the previosu word, maintaining the cursor at the start of the
+/* Delete the previous word, maintaining the cursor at the start of the
  * current word. */
 void linenoiseEditDeletePrevWord(struct linenoiseState *l) {
     size_t old_pos = l->pos;
diff --git a/redis.conf b/redis.conf
index 8f26a8f3e..51f71484c 100644
--- a/redis.conf
+++ b/redis.conf
@@ -24,7 +24,7 @@
 # to customize a few per-server settings.  Include files can include
 # other files, so use this wisely.
 #
-# Notice option "include" won't be rewritten by command "CONFIG REWRITE"
+# Note that option "include" won't be rewritten by command "CONFIG REWRITE"
 # from admin or Redis Sentinel. Since Redis always uses the last processed
 # line as value of a configuration directive, you'd better put includes
 # at the beginning of this file to avoid overwriting config change at runtime.
@@ -46,7 +46,7 @@
 ################################## NETWORK #####################################
 
 # By default, if no "bind" configuration directive is specified, Redis listens
-# for connections from all the network interfaces available on the server.
+# for connections from all available network interfaces on the host machine.
 # It is possible to listen to just one or multiple selected interfaces using
 # the "bind" configuration directive, followed by one or more IP addresses.
 #
@@ -58,13 +58,12 @@
 # ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the
 # internet, binding to all the interfaces is dangerous and will expose the
 # instance to everybody on the internet. So by default we uncomment the
-# following bind directive, that will force Redis to listen only into
-# the IPv4 loopback interface address (this means Redis will be able to
-# accept connections only from clients running into the same computer it
-# is running).
+# following bind directive, that will force Redis to listen only on the
+# IPv4 loopback interface address (this means Redis will only be able to
+# accept client connections from the same host that it is running on).
 #
 # IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES
-# JUST COMMENT THE FOLLOWING LINE.
+# JUST COMMENT OUT THE FOLLOWING LINE.
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 bind 127.0.0.1
 
@@ -93,8 +92,8 @@ port 6379
 
 # TCP listen() backlog.
 #
-# In high requests-per-second environments you need an high backlog in order
-# to avoid slow clients connections issues. Note that the Linux kernel
+# In high requests-per-second environments you need a high backlog in order
+# to avoid slow clients connection issues. Note that the Linux kernel
 # will silently truncate it to the value of /proc/sys/net/core/somaxconn so
 # make sure to raise both the value of somaxconn and tcp_max_syn_backlog
 # in order to get the desired effect.
@@ -118,8 +117,8 @@ timeout 0
 # of communication. This is useful for two reasons:
 #
 # 1) Detect dead peers.
-# 2) Take the connection alive from the point of view of network
-#    equipment in the middle.
+# 2) Force network equipment in the middle to consider the connection to be
+#    alive.
 #
 # On Linux, the specified value (in seconds) is the period used to send ACKs.
 # Note that to close the connection the double of the time is needed.
@@ -228,11 +227,12 @@ daemonize no
 # supervision tree. Options:
 #   supervised no      - no supervision interaction
 #   supervised upstart - signal upstart by putting Redis into SIGSTOP mode
+#                        requires "expect stop" in your upstart job config
 #   supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET
 #   supervised auto    - detect upstart or systemd method based on
 #                        UPSTART_JOB or NOTIFY_SOCKET environment variables
 # Note: these supervision methods only signal "process is ready."
-#       They do not enable continuous liveness pings back to your supervisor.
+#       They do not enable continuous pings back to your supervisor.
 supervised no
 
 # If a pid file is specified, Redis writes it where specified at startup
@@ -302,7 +302,7 @@ always-show-logo no
 #   Will save the DB if both the given number of seconds and the given
 #   number of write operations against the DB occurred.
 #
-#   In the example below the behaviour will be to save:
+#   In the example below the behavior will be to save:
 #   after 900 sec (15 min) if at least 1 key changed
 #   after 300 sec (5 min) if at least 10 keys changed
 #   after 60 sec if at least 10000 keys changed
@@ -335,7 +335,7 @@ save 60 10000
 stop-writes-on-bgsave-error yes
 
 # Compress string objects using LZF when dump .rdb databases?
-# For default that's set to 'yes' as it's almost always a win.
+# By default compression is enabled as it's almost always a win.
 # If you want to save some CPU in the saving child set it to 'no' but
 # the dataset will likely be bigger if you have compressible values or keys.
 rdbcompression yes
@@ -423,11 +423,11 @@ dir ./
 #    still reply to client requests, possibly with out of date data, or the
 #    data set may just be empty if this is the first synchronization.
 #
-# 2) if replica-serve-stale-data is set to 'no' the replica will reply with
-#    an error "SYNC with master in progress" to all the kind of commands
-#    but to INFO, replicaOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG,
-#    SUBSCRIBE, UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB,
-#    COMMAND, POST, HOST: and LATENCY.
+# 2) If replica-serve-stale-data is set to 'no' the replica will reply with
+#    an error "SYNC with master in progress" to all commands except:
+#    INFO, REPLICAOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, SUBSCRIBE,
+#    UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, COMMAND, POST,
+#    HOST and LATENCY.
 #
 replica-serve-stale-data yes
 
@@ -498,7 +498,7 @@ repl-diskless-sync-delay 5
 #
 # Replica can load the RDB it reads from the replication link directly from the
 # socket, or store the RDB to a file and read that file after it was completely
-# recived from the master.
+# received from the master.
 #
 # In many cases the disk is slower than the network, and storing and loading
 # the RDB file may increase replication time (and even increase the master's
@@ -528,7 +528,8 @@ repl-diskless-load disabled
 #
 # It is important to make sure that this value is greater than the value
 # specified for repl-ping-replica-period otherwise a timeout will be detected
-# every time there is low traffic between the master and the replica.
+# every time there is low traffic between the master and the replica. The default
+# value is 60 seconds.
 #
 # repl-timeout 60
 
@@ -553,21 +554,21 @@ repl-disable-tcp-nodelay no
 # partial resync is enough, just passing the portion of data the replica
 # missed while disconnected.
 #
-# The bigger the replication backlog, the longer the time the replica can be
-# disconnected and later be able to perform a partial resynchronization.
+# The bigger the replication backlog, the longer the replica can endure the
+# disconnect and later be able to perform a partial resynchronization.
 #
-# The backlog is only allocated once there is at least a replica connected.
+# The backlog is only allocated if there is at least one replica connected.
 #
 # repl-backlog-size 1mb
 
-# After a master has no longer connected replicas for some time, the backlog
-# will be freed. The following option configures the amount of seconds that
-# need to elapse, starting from the time the last replica disconnected, for
-# the backlog buffer to be freed.
+# After a master has no connected replicas for some time, the backlog will be
+# freed. The following option configures the amount of seconds that need to
+# elapse, starting from the time the last replica disconnected, for the backlog
+# buffer to be freed.
 #
 # Note that replicas never free the backlog for timeout, since they may be
 # promoted to masters later, and should be able to correctly "partially
-# resynchronize" with the replicas: hence they should always accumulate backlog.
+# resynchronize" with other replicas: hence they should always accumulate backlog.
 #
 # A value of 0 means to never release the backlog.
 #
@@ -617,8 +618,8 @@ replica-priority 100
 # Another place where this info is available is in the output of the
 # "ROLE" command of a master.
 #
-# The listed IP and address normally reported by a replica is obtained
-# in the following way:
+# The listed IP address and port normally reported by a replica is
+# obtained in the following way:
 #
 #   IP: The address is auto detected by checking the peer address
 #   of the socket used by the replica to connect with the master.
@@ -628,7 +629,7 @@ replica-priority 100
 #   listen for connections.
 #
 # However when port forwarding or Network Address Translation (NAT) is
-# used, the replica may be actually reachable via different IP and port
+# used, the replica may actually be reachable via different IP and port
 # pairs. The following two options can be used by a replica in order to
 # report to its master a specific set of IP and port, so that both INFO
 # and ROLE will report those values.
@@ -645,7 +646,7 @@ replica-priority 100
 # This is implemented using an invalidation table that remembers, using
 # 16 millions of slots, what clients may have certain subsets of keys. In turn
 # this is used in order to send invalidation messages to clients. Please
-# to understand more about the feature check this page:
+# check this page to understand more about the feature:
 #
 #   https://redis.io/topics/client-side-caching
 #
@@ -677,7 +678,7 @@ replica-priority 100
 
 ################################## SECURITY ###################################
 
-# Warning: since Redis is pretty fast an outside user can try up to
+# Warning: since Redis is pretty fast, an outside user can try up to
 # 1 million passwords per second against a modern box. This means that you
 # should use very strong passwords, otherwise they will be very easy to break.
 # Note that because the password is really a shared secret between the client
@@ -701,7 +702,7 @@ replica-priority 100
 # AUTH (or the HELLO command AUTH option) in order to be authenticated and
 # start to work.
 #
-# The ACL rules that describe what an user can do are the following:
+# The ACL rules that describe what a user can do are the following:
 #
 #  on           Enable the user: it is possible to authenticate as this user.
 #  off          Disable the user: it's no longer possible to authenticate
@@ -729,7 +730,7 @@ replica-priority 100
 #               It is possible to specify multiple patterns.
 #  allkeys      Alias for ~*
 #  resetkeys    Flush the list of allowed keys patterns.
-#  ><password>  Add this passowrd to the list of valid password for the user.
+#  ><password>  Add this password to the list of valid password for the user.
 #               For example >mypass will add "mypass" to the list.
 #               This directive clears the "nopass" flag (see later).
 #  <<password>  Remove this password from the list of valid passwords.
@@ -783,7 +784,7 @@ acllog-max-len 128
 #
 # Instead of configuring users here in this file, it is possible to use
 # a stand-alone file just listing users. The two methods cannot be mixed:
-# if you configure users here and at the same time you activate the exteranl
+# if you configure users here and at the same time you activate the external
 # ACL file, the server will refuse to start.
 #
 # The format of the external ACL user file is exactly the same as the
@@ -791,7 +792,7 @@ acllog-max-len 128
 #
 # aclfile /etc/redis/users.acl
 
-# IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatiblity
+# IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatibility
 # layer on top of the new ACL system. The option effect will be just setting
 # the password for the default user. Clients will still authenticate using
 # AUTH <password> as usually, or more explicitly with AUTH default <password>
@@ -902,8 +903,8 @@ acllog-max-len 128
 
 # LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated
 # algorithms (in order to save memory), so you can tune it for speed or
-# accuracy. For default Redis will check five keys and pick the one that was
-# used less recently, you can change the sample size using the following
+# accuracy. By default Redis will check five keys and pick the one that was
+# used least recently, you can change the sample size using the following
 # configuration directive.
 #
 # The default of 5 produces good enough results. 10 Approximates very closely
@@ -943,8 +944,8 @@ acllog-max-len 128
 # it is possible to increase the expire "effort" that is normally set to
 # "1", to a greater value, up to the value "10". At its maximum value the
 # system will use more CPU, longer cycles (and technically may introduce
-# more latency), and will tollerate less already expired keys still present
-# in the system. It's a tradeoff betweeen memory, CPU and latecy.
+# more latency), and will tolerate less already expired keys still present
+# in the system. It's a tradeoff between memory, CPU and latency.
 #
 # active-expire-effort 1
 
@@ -1012,7 +1013,7 @@ lazyfree-lazy-user-del no
 #
 # Now it is also possible to handle Redis clients socket reads and writes
 # in different I/O threads. Since especially writing is so slow, normally
-# Redis users use pipelining in order to speedup the Redis performances per
+# Redis users use pipelining in order to speed up the Redis performances per
 # core, and spawn multiple instances in order to scale more. Using I/O
 # threads it is possible to easily speedup two times Redis without resorting
 # to pipelining nor sharding of the instance.
@@ -1030,7 +1031,7 @@ lazyfree-lazy-user-del no
 #
 # io-threads 4
 #
-# Setting io-threads to 1 will just use the main thread as usually.
+# Setting io-threads to 1 will just use the main thread as usual.
 # When I/O threads are enabled, we only use threads for writes, that is
 # to thread the write(2) syscall and transfer the client buffers to the
 # socket. However it is also possible to enable threading of reads and
@@ -1047,7 +1048,7 @@ lazyfree-lazy-user-del no
 #
 # NOTE 2: If you want to test the Redis speedup using redis-benchmark, make
 # sure you also run the benchmark itself in threaded mode, using the
-# --threads option to match the number of Redis theads, otherwise you'll not
+# --threads option to match the number of Redis threads, otherwise you'll not
 # be able to notice the improvements.
 
 ############################ KERNEL OOM CONTROL ##############################
@@ -1200,8 +1201,8 @@ aof-load-truncated yes
 #
 #   [RDB file][AOF tail]
 #
-# When loading Redis recognizes that the AOF file starts with the "REDIS"
-# string and loads the prefixed RDB file, and continues loading the AOF
+# When loading, Redis recognizes that the AOF file starts with the "REDIS"
+# string and loads the prefixed RDB file, then continues loading the AOF
 # tail.
 aof-use-rdb-preamble yes
 
@@ -1215,7 +1216,7 @@ aof-use-rdb-preamble yes
 #
 # When a long running script exceeds the maximum execution time only the
 # SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be
-# used to stop a script that did not yet called write commands. The second
+# used to stop a script that did not yet call any write commands. The second
 # is the only way to shut down the server in the case a write command was
 # already issued by the script but the user doesn't want to wait for the natural
 # termination of the script.
@@ -1241,7 +1242,7 @@ lua-time-limit 5000
 
 # Cluster node timeout is the amount of milliseconds a node must be unreachable
 # for it to be considered in failure state.
-# Most other internal time limits are multiple of the node timeout.
+# Most other internal time limits are a multiple of the node timeout.
 #
 # cluster-node-timeout 15000
 
@@ -1268,18 +1269,18 @@ lua-time-limit 5000
 # the failover if, since the last interaction with the master, the time
 # elapsed is greater than:
 #
-#   (node-timeout * replica-validity-factor) + repl-ping-replica-period
+#   (node-timeout * cluster-replica-validity-factor) + repl-ping-replica-period
 #
-# So for example if node-timeout is 30 seconds, and the replica-validity-factor
+# So for example if node-timeout is 30 seconds, and the cluster-replica-validity-factor
 # is 10, and assuming a default repl-ping-replica-period of 10 seconds, the
 # replica will not try to failover if it was not able to talk with the master
 # for longer than 310 seconds.
 #
-# A large replica-validity-factor may allow replicas with too old data to failover
+# A large cluster-replica-validity-factor may allow replicas with too old data to failover
 # a master, while a too small value may prevent the cluster from being able to
 # elect a replica at all.
 #
-# For maximum availability, it is possible to set the replica-validity-factor
+# For maximum availability, it is possible to set the cluster-replica-validity-factor
 # to a value of 0, which means, that replicas will always try to failover the
 # master regardless of the last time they interacted with the master.
 # (However they'll always try to apply a delay proportional to their
@@ -1310,7 +1311,7 @@ lua-time-limit 5000
 # cluster-migration-barrier 1
 
 # By default Redis Cluster nodes stop accepting queries if they detect there
-# is at least an hash slot uncovered (no available node is serving it).
+# is at least a hash slot uncovered (no available node is serving it).
 # This way if the cluster is partially down (for example a range of hash slots
 # are no longer covered) all the cluster becomes, eventually, unavailable.
 # It automatically returns available as soon as all the slots are covered again.
@@ -1365,7 +1366,7 @@ lua-time-limit 5000
 # * cluster-announce-port
 # * cluster-announce-bus-port
 #
-# Each instruct the node about its address, client port, and cluster message
+# Each instructs the node about its address, client port, and cluster message
 # bus port. The information is then published in the header of the bus packets
 # so that other nodes will be able to correctly map the address of the node
 # publishing the information.
@@ -1376,7 +1377,7 @@ lua-time-limit 5000
 # Note that when remapped, the bus port may not be at the fixed offset of
 # clients port + 10000, so you can specify any port and bus-port depending
 # on how they get remapped. If the bus-port is not set, a fixed offset of
-# 10000 will be used as usually.
+# 10000 will be used as usual.
 #
 # Example:
 #
@@ -1505,7 +1506,7 @@ notify-keyspace-events ""
 # two kind of inline requests that were anyway illegal: an empty request
 # or any request that starts with "/" (there are no Redis commands starting
 # with such a slash). Normal RESP2/RESP3 requests are completely out of the
-# path of the Gopher protocol implementation and are served as usually as well.
+# path of the Gopher protocol implementation and are served as usual as well.
 #
 # If you open a connection to Redis when Gopher is enabled and send it
 # a string like "/foo", if there is a key named "/foo" it is served via the
@@ -1677,7 +1678,7 @@ client-output-buffer-limit pubsub 32mb 8mb 60
 # client-query-buffer-limit 1gb
 
 # In the Redis protocol, bulk requests, that are, elements representing single
-# strings, are normally limited ot 512 mb. However you can change this limit
+# strings, are normally limited to 512 mb. However you can change this limit
 # here, but must be 1mb or greater
 #
 # proto-max-bulk-len 512mb
@@ -1706,7 +1707,7 @@ hz 10
 #
 # Since the default HZ value by default is conservatively set to 10, Redis
 # offers, and enables by default, the ability to use an adaptive HZ value
-# which will temporary raise when there are many connected clients.
+# which will temporarily raise when there are many connected clients.
 #
 # When dynamic HZ is enabled, the actual configured HZ will be used
 # as a baseline, but multiples of the configured HZ value will be actually
@@ -1773,7 +1774,7 @@ rdb-save-incremental-fsync yes
 # for the key counter to be divided by two (or decremented if it has a value
 # less <= 10).
 #
-# The default value for the lfu-decay-time is 1. A Special value of 0 means to
+# The default value for the lfu-decay-time is 1. A special value of 0 means to
 # decay the counter every time it happens to be scanned.
 #
 # lfu-log-factor 10
@@ -1793,7 +1794,7 @@ rdb-save-incremental-fsync yes
 # restart is needed in order to lower the fragmentation, or at least to flush
 # away all the data and create it again. However thanks to this feature
 # implemented by Oran Agra for Redis 4.0 this process can happen at runtime
-# in an "hot" way, while the server is running.
+# in a "hot" way, while the server is running.
 #
 # Basically when the fragmentation is over a certain level (see the
 # configuration options below) Redis will start to create new copies of the
@@ -1870,3 +1871,4 @@ jemalloc-bg-thread yes
 #
 # Set bgsave child process to cpu affinity 1,10,11
 # bgsave_cpulist 1,10-11
+
diff --git a/sentinel.conf b/sentinel.conf
index 4ca5e5f8f..b6ff05f25 100644
--- a/sentinel.conf
+++ b/sentinel.conf
@@ -259,6 +259,6 @@ sentinel deny-scripts-reconfig yes
 # SENTINEL SET can also be used in order to perform this configuration at runtime.
 #
 # In order to set a command back to its original name (undo the renaming), it
-# is possible to just rename a command to itsef:
+# is possible to just rename a command to itself:
 #
 # SENTINEL rename-command mymaster CONFIG CONFIG
diff --git a/src/acl.c b/src/acl.c
index 987427899..7af3d6242 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -289,7 +289,7 @@ void ACLFreeUserAndKillClients(user *u) {
     while ((ln = listNext(&li)) != NULL) {
         client *c = listNodeValue(ln);
         if (c->user == u) {
-            /* We'll free the conenction asynchronously, so
+            /* We'll free the connection asynchronously, so
              * in theory to set a different user is not needed.
              * However if there are bugs in Redis, soon or later
              * this may result in some security hole: it's much
diff --git a/src/adlist.c b/src/adlist.c
index 0fedc0729..bc06ffc8f 100644
--- a/src/adlist.c
+++ b/src/adlist.c
@@ -34,8 +34,9 @@
 #include "zmalloc.h"
 
 /* Create a new list. The created list can be freed with
- * AlFreeList(), but private value of every node need to be freed
- * by the user before to call AlFreeList().
+ * listRelease(), but private value of every node need to be freed
+ * by the user before to call listRelease(), or by setting a free method using
+ * listSetFreeMethod.
  *
  * On error, NULL is returned. Otherwise the pointer to the new list. */
 list *listCreate(void)
@@ -217,8 +218,8 @@ void listRewindTail(list *list, listIter *li) {
  * listDelNode(), but not to remove other elements.
  *
  * The function returns a pointer to the next element of the list,
- * or NULL if there are no more elements, so the classical usage patter
- * is:
+ * or NULL if there are no more elements, so the classical usage
+ * pattern is:
  *
  * iter = listGetIterator(list,<direction>);
  * while ((node = listNext(iter)) != NULL) {
diff --git a/src/ae.c b/src/ae.c
index 94459acff..e6f8df877 100644
--- a/src/ae.c
+++ b/src/ae.c
@@ -405,7 +405,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
             int fired = 0; /* Number of events fired for current fd. */
 
             /* Normally we execute the readable event first, and the writable
-             * event laster. This is useful as sometimes we may be able
+             * event later. This is useful as sometimes we may be able
              * to serve the reply of a query immediately after processing the
              * query.
              *
@@ -413,7 +413,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
              * asking us to do the reverse: never fire the writable event
              * after the readable. In such a case, we invert the calls.
              * This is useful when, for instance, we want to do things
-             * in the beforeSleep() hook, like fsynching a file to disk,
+             * in the beforeSleep() hook, like fsyncing a file to disk,
              * before replying to a client. */
             int invert = fe->mask & AE_BARRIER;
 
diff --git a/src/ae_evport.c b/src/ae_evport.c
index 5c317becb..b79ed9bc7 100644
--- a/src/ae_evport.c
+++ b/src/ae_evport.c
@@ -232,7 +232,7 @@ static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
         /*
          * ENOMEM is a potentially transient condition, but the kernel won't
          * generally return it unless things are really bad.  EAGAIN indicates
-         * we've reached an resource limit, for which it doesn't make sense to
+         * we've reached a resource limit, for which it doesn't make sense to
          * retry (counter-intuitively).  All other errors indicate a bug.  In any
          * of these cases, the best we can do is to abort.
          */
diff --git a/src/aof.c b/src/aof.c
index 07f9f8d08..e01a11342 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -544,7 +544,7 @@ sds catAppendOnlyGenericCommand(sds dst, int argc, robj **argv) {
     return dst;
 }
 
-/* Create the sds representation of an PEXPIREAT command, using
+/* Create the sds representation of a PEXPIREAT command, using
  * 'seconds' as time to live and 'cmd' to understand what command
  * we are translating into a PEXPIREAT.
  *
@@ -1818,7 +1818,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
             "Background AOF rewrite terminated with error");
     } else {
         /* SIGUSR1 is whitelisted, so we have a way to kill a child without
-         * tirggering an error condition. */
+         * triggering an error condition. */
         if (bysignal != SIGUSR1)
             server.aof_lastbgrewrite_status = C_ERR;
 
diff --git a/src/atomicvar.h b/src/atomicvar.h
index 160056cd7..ecd26ad70 100644
--- a/src/atomicvar.h
+++ b/src/atomicvar.h
@@ -21,7 +21,7 @@
  *
  * Never use return value from the macros, instead use the AtomicGetIncr()
  * if you need to get the current value and increment it atomically, like
- * in the followign example:
+ * in the following example:
  *
  *  long oldvalue;
  *  atomicGetIncr(myvar,oldvalue,1);
diff --git a/src/bitops.c b/src/bitops.c
index 37fb13f0c..0f45aa00d 100644
--- a/src/bitops.c
+++ b/src/bitops.c
@@ -36,7 +36,7 @@
 
 /* Count number of bits set in the binary array pointed by 's' and long
  * 'count' bytes. The implementation of this function is required to
- * work with a input string length up to 512 MB. */
+ * work with an input string length up to 512 MB. */
 size_t redisPopcount(void *s, long count) {
     size_t bits = 0;
     unsigned char *p = s;
@@ -107,7 +107,7 @@ long redisBitpos(void *s, unsigned long count, int bit) {
     int found;
 
     /* Process whole words first, seeking for first word that is not
-     * all ones or all zeros respectively if we are lookig for zeros
+     * all ones or all zeros respectively if we are looking for zeros
      * or ones. This is much faster with large strings having contiguous
      * blocks of 1 or 0 bits compared to the vanilla bit per bit processing.
      *
@@ -496,7 +496,7 @@ robj *lookupStringForBitCommand(client *c, size_t maxbit) {
  * in 'len'. The user is required to pass (likely stack allocated) buffer
  * 'llbuf' of at least LONG_STR_SIZE bytes. Such a buffer is used in the case
  * the object is integer encoded in order to provide the representation
- * without usign heap allocation.
+ * without using heap allocation.
  *
  * The function returns the pointer to the object array of bytes representing
  * the string it contains, that may be a pointer to 'llbuf' or to the
diff --git a/src/blocked.c b/src/blocked.c
index c3884fbdf..08de63485 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -53,7 +53,7 @@
  * to 0, no timeout is processed).
  * It usually just needs to send a reply to the client.
  *
- * When implementing a new type of blocking opeation, the implementation
+ * When implementing a new type of blocking operation, the implementation
  * should modify unblockClient() and replyToBlockedClientTimedOut() in order
  * to handle the btype-specific behavior of this two functions.
  * If the blocking operation waits for certain keys to change state, the
@@ -118,7 +118,7 @@ void processUnblockedClients(void) {
 
 /* This function will schedule the client for reprocessing at a safe time.
  *
- * This is useful when a client was blocked for some reason (blocking opeation,
+ * This is useful when a client was blocked for some reason (blocking operation,
  * CLIENT PAUSE, or whatever), because it may end with some accumulated query
  * buffer that needs to be processed ASAP:
  *
diff --git a/src/cluster.c b/src/cluster.c
index b9a0c8661..03235d58e 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -377,7 +377,7 @@ void clusterSaveConfigOrDie(int do_fsync) {
     }
 }
 
-/* Lock the cluster config using flock(), and leaks the file descritor used to
+/* Lock the cluster config using flock(), and leaks the file descriptor used to
  * acquire the lock so that the file will be locked forever.
  *
  * This works because we always update nodes.conf with a new version
@@ -544,13 +544,13 @@ void clusterInit(void) {
 
 /* Reset a node performing a soft or hard reset:
  *
- * 1) All other nodes are forget.
+ * 1) All other nodes are forgotten.
  * 2) All the assigned / open slots are released.
  * 3) If the node is a slave, it turns into a master.
- * 5) Only for hard reset: a new Node ID is generated.
- * 6) Only for hard reset: currentEpoch and configEpoch are set to 0.
- * 7) The new configuration is saved and the cluster state updated.
- * 8) If the node was a slave, the whole data set is flushed away. */
+ * 4) Only for hard reset: a new Node ID is generated.
+ * 5) Only for hard reset: currentEpoch and configEpoch are set to 0.
+ * 6) The new configuration is saved and the cluster state updated.
+ * 7) If the node was a slave, the whole data set is flushed away. */
 void clusterReset(int hard) {
     dictIterator *di;
     dictEntry *de;
@@ -646,7 +646,7 @@ static void clusterConnAcceptHandler(connection *conn) {
 
     /* Create a link object we use to handle the connection.
      * It gets passed to the readable handler when data is available.
-     * Initiallly the link->node pointer is set to NULL as we don't know
+     * Initially the link->node pointer is set to NULL as we don't know
      * which node is, but the right node is references once we know the
      * node identity. */
     link = createClusterLink(NULL);
@@ -1060,7 +1060,7 @@ uint64_t clusterGetMaxEpoch(void) {
  * 3) Persist the configuration on disk before sending packets with the
  *    new configuration.
  *
- * If the new config epoch is generated and assigend, C_OK is returned,
+ * If the new config epoch is generated and assigned, C_OK is returned,
  * otherwise C_ERR is returned (since the node has already the greatest
  * configuration around) and no operation is performed.
  *
@@ -1133,7 +1133,7 @@ int clusterBumpConfigEpochWithoutConsensus(void) {
  *
  * In general we want a system that eventually always ends with different
  * masters having different configuration epochs whatever happened, since
- * nothign is worse than a split-brain condition in a distributed system.
+ * nothing is worse than a split-brain condition in a distributed system.
  *
  * BEHAVIOR
  *
@@ -1192,7 +1192,7 @@ void clusterHandleConfigEpochCollision(clusterNode *sender) {
  * entries from the black list. This is an O(N) operation but it is not a
  * problem since add / exists operations are called very infrequently and
  * the hash table is supposed to contain very little elements at max.
- * However without the cleanup during long uptimes and with some automated
+ * However without the cleanup during long uptime and with some automated
  * node add/removal procedures, entries could accumulate. */
 void clusterBlacklistCleanup(void) {
     dictIterator *di;
@@ -1346,12 +1346,12 @@ int clusterHandshakeInProgress(char *ip, int port, int cport) {
     return de != NULL;
 }
 
-/* Start an handshake with the specified address if there is not one
+/* Start a handshake with the specified address if there is not one
  * already in progress. Returns non-zero if the handshake was actually
  * started. On error zero is returned and errno is set to one of the
  * following values:
  *
- * EAGAIN - There is already an handshake in progress for this address.
+ * EAGAIN - There is already a handshake in progress for this address.
  * EINVAL - IP or port are not valid. */
 int clusterStartHandshake(char *ip, int port, int cport) {
     clusterNode *n;
@@ -1793,7 +1793,7 @@ int clusterProcessPacket(clusterLink *link) {
     if (sender) sender->data_received = now;
 
     if (sender && !nodeInHandshake(sender)) {
-        /* Update our curretEpoch if we see a newer epoch in the cluster. */
+        /* Update our currentEpoch if we see a newer epoch in the cluster. */
         senderCurrentEpoch = ntohu64(hdr->currentEpoch);
         senderConfigEpoch = ntohu64(hdr->configEpoch);
         if (senderCurrentEpoch > server.cluster->currentEpoch)
@@ -2480,7 +2480,7 @@ void clusterSetGossipEntry(clusterMsg *hdr, int i, clusterNode *n) {
 }
 
 /* Send a PING or PONG packet to the specified node, making sure to add enough
- * gossip informations. */
+ * gossip information. */
 void clusterSendPing(clusterLink *link, int type) {
     unsigned char *buf;
     clusterMsg *hdr;
@@ -2500,7 +2500,7 @@ void clusterSendPing(clusterLink *link, int type) {
      * node_timeout we exchange with each other node at least 4 packets
      * (we ping in the worst case in node_timeout/2 time, and we also
      * receive two pings from the host), we have a total of 8 packets
-     * in the node_timeout*2 falure reports validity time. So we have
+     * in the node_timeout*2 failure reports validity time. So we have
      * that, for a single PFAIL node, we can expect to receive the following
      * number of failure reports (in the specified window of time):
      *
@@ -2527,7 +2527,7 @@ void clusterSendPing(clusterLink *link, int type) {
      * faster to propagate to go from PFAIL to FAIL state. */
     int pfail_wanted = server.cluster->stats_pfail_nodes;
 
-    /* Compute the maxium totlen to allocate our buffer. We'll fix the totlen
+    /* Compute the maximum totlen to allocate our buffer. We'll fix the totlen
      * later according to the number of gossip sections we really were able
      * to put inside the packet. */
     totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData);
@@ -2564,7 +2564,7 @@ void clusterSendPing(clusterLink *link, int type) {
         if (this->flags & (CLUSTER_NODE_HANDSHAKE|CLUSTER_NODE_NOADDR) ||
             (this->link == NULL && this->numslots == 0))
         {
-            freshnodes--; /* Tecnically not correct, but saves CPU. */
+            freshnodes--; /* Technically not correct, but saves CPU. */
             continue;
         }
 
@@ -3149,7 +3149,7 @@ void clusterHandleSlaveFailover(void) {
         }
     }
 
-    /* If the previous failover attempt timedout and the retry time has
+    /* If the previous failover attempt timeout and the retry time has
      * elapsed, we can setup a new one. */
     if (auth_age > auth_retry_time) {
         server.cluster->failover_auth_time = mstime() +
@@ -3255,7 +3255,7 @@ void clusterHandleSlaveFailover(void) {
  *
  * Slave migration is the process that allows a slave of a master that is
  * already covered by at least another slave, to "migrate" to a master that
- * is orpaned, that is, left with no working slaves.
+ * is orphaned, that is, left with no working slaves.
  * ------------------------------------------------------------------------- */
 
 /* This function is responsible to decide if this replica should be migrated
@@ -3272,7 +3272,7 @@ void clusterHandleSlaveFailover(void) {
  * the nodes anyway, so we spend time into clusterHandleSlaveMigration()
  * if definitely needed.
  *
- * The fuction is called with a pre-computed max_slaves, that is the max
+ * The function is called with a pre-computed max_slaves, that is the max
  * number of working (not in FAIL state) slaves for a single master.
  *
  * Additional conditions for migration are examined inside the function.
@@ -3391,7 +3391,7 @@ void clusterHandleSlaveMigration(int max_slaves) {
  * data loss due to the asynchronous master-slave replication.
  * -------------------------------------------------------------------------- */
 
-/* Reset the manual failover state. This works for both masters and slavesa
+/* Reset the manual failover state. This works for both masters and slaves
  * as all the state about manual failover is cleared.
  *
  * The function can be used both to initialize the manual failover state at
@@ -3683,7 +3683,7 @@ void clusterCron(void) {
         replicationSetMaster(myself->slaveof->ip, myself->slaveof->port);
     }
 
-    /* Abourt a manual failover if the timeout is reached. */
+    /* Abort a manual failover if the timeout is reached. */
     manualFailoverCheckTimeout();
 
     if (nodeIsSlave(myself)) {
@@ -3788,12 +3788,12 @@ int clusterNodeSetSlotBit(clusterNode *n, int slot) {
          * target for replicas migration, if and only if at least one of
          * the other masters has slaves right now.
          *
-         * Normally masters are valid targerts of replica migration if:
+         * Normally masters are valid targets of replica migration if:
          * 1. The used to have slaves (but no longer have).
          * 2. They are slaves failing over a master that used to have slaves.
          *
          * However new masters with slots assigned are considered valid
-         * migration tagets if the rest of the cluster is not a slave-less.
+         * migration targets if the rest of the cluster is not a slave-less.
          *
          * See https://github.com/antirez/redis/issues/3043 for more info. */
         if (n->numslots == 1 && clusterMastersHaveSlaves())
@@ -3977,7 +3977,7 @@ void clusterUpdateState(void) {
  *    A) If no other node is in charge according to the current cluster
  *       configuration, we add these slots to our node.
  *    B) If according to our config other nodes are already in charge for
- *       this lots, we set the slots as IMPORTING from our point of view
+ *       this slots, we set the slots as IMPORTING from our point of view
  *       in order to justify we have those slots, and in order to make
  *       redis-trib aware of the issue, so that it can try to fix it.
  * 2) If we find data in a DB different than DB0 we return C_ERR to
@@ -4507,7 +4507,7 @@ NULL
             }
             /* If this slot is in migrating status but we have no keys
              * for it assigning the slot to another node will clear
-             * the migratig status. */
+             * the migrating status. */
             if (countKeysInSlot(slot) == 0 &&
                 server.cluster->migrating_slots_to[slot])
                 server.cluster->migrating_slots_to[slot] = NULL;
@@ -4856,7 +4856,7 @@ NULL
                 server.cluster->currentEpoch = epoch;
             /* No need to fsync the config here since in the unlucky event
              * of a failure to persist the config, the conflict resolution code
-             * will assign an unique config to this node. */
+             * will assign a unique config to this node. */
             clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|
                                  CLUSTER_TODO_SAVE_CONFIG);
             addReply(c,shared.ok);
@@ -4904,7 +4904,7 @@ void createDumpPayload(rio *payload, robj *o, robj *key) {
     unsigned char buf[2];
     uint64_t crc;
 
-    /* Serialize the object in a RDB-like format. It consist of an object type
+    /* Serialize the object in an RDB-like format. It consist of an object type
      * byte followed by the serialized object. This is understood by RESTORE. */
     rioInitWithBuffer(payload,sdsempty());
     serverAssert(rdbSaveObjectType(payload,o));
@@ -5571,7 +5571,7 @@ void readwriteCommand(client *c) {
  *    resharding in progress).
  *
  * On success the function returns the node that is able to serve the request.
- * If the node is not 'myself' a redirection must be perfomed. The kind of
+ * If the node is not 'myself' a redirection must be performed. The kind of
  * redirection is specified setting the integer passed by reference
  * 'error_code', which will be set to CLUSTER_REDIR_ASK or
  * CLUSTER_REDIR_MOVED.
@@ -5698,7 +5698,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
                 }
             }
 
-            /* Migarting / Improrting slot? Count keys we don't have. */
+            /* Migrating / Importing slot? Count keys we don't have. */
             if ((migrating_slot || importing_slot) &&
                 lookupKeyRead(&server.db[0],thiskey) == NULL)
             {
@@ -5767,7 +5767,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
     }
 
     /* Handle the read-only client case reading from a slave: if this
-     * node is a slave and the request is about an hash slot our master
+     * node is a slave and the request is about a hash slot our master
      * is serving, we can reply without redirection. */
     int is_readonly_command = (c->cmd->flags & CMD_READONLY) ||
                               (c->cmd->proc == execCommand && !(c->mstate.cmd_inv_flags & CMD_READONLY));
@@ -5781,7 +5781,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
     }
 
     /* Base case: just return the right node. However if this node is not
-     * myself, set error_code to MOVED since we need to issue a rediretion. */
+     * myself, set error_code to MOVED since we need to issue a redirection. */
     if (n != myself && error_code) *error_code = CLUSTER_REDIR_MOVED;
     return n;
 }
@@ -5827,7 +5827,7 @@ void clusterRedirectClient(client *c, clusterNode *n, int hashslot, int error_co
  * 3) The client may remain blocked forever (or up to the max timeout time)
  *    waiting for a key change that will never happen.
  *
- * If the client is found to be blocked into an hash slot this node no
+ * If the client is found to be blocked into a hash slot this node no
  * longer handles, the client is sent a redirection error, and the function
  * returns 1. Otherwise 0 is returned and no operation is performed. */
 int clusterRedirectBlockedClientIfNeeded(client *c) {
diff --git a/src/cluster.h b/src/cluster.h
index 596a4629a..48a111764 100644
--- a/src/cluster.h
+++ b/src/cluster.h
@@ -51,8 +51,8 @@ typedef struct clusterLink {
 #define CLUSTER_NODE_HANDSHAKE 32 /* We have still to exchange the first ping */
 #define CLUSTER_NODE_NOADDR   64  /* We don't know the address of this node */
 #define CLUSTER_NODE_MEET 128     /* Send a MEET message to this node */
-#define CLUSTER_NODE_MIGRATE_TO 256 /* Master elegible for replica migration. */
-#define CLUSTER_NODE_NOFAILOVER 512 /* Slave will not try to failver. */
+#define CLUSTER_NODE_MIGRATE_TO 256 /* Master eligible for replica migration. */
+#define CLUSTER_NODE_NOFAILOVER 512 /* Slave will not try to failover. */
 #define CLUSTER_NODE_NULL_NAME "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"
 
 #define nodeIsMaster(n) ((n)->flags & CLUSTER_NODE_MASTER)
@@ -164,10 +164,10 @@ typedef struct clusterState {
     clusterNode *mf_slave;      /* Slave performing the manual failover. */
     /* Manual failover state of slave. */
     long long mf_master_offset; /* Master offset the slave needs to start MF
-                                   or zero if stil not received. */
+                                   or zero if still not received. */
     int mf_can_start;           /* If non-zero signal that the manual failover
                                    can start requesting masters vote. */
-    /* The followign fields are used by masters to take state on elections. */
+    /* The following fields are used by masters to take state on elections. */
     uint64_t lastVoteEpoch;     /* Epoch of the last vote granted. */
     int todo_before_sleep; /* Things to do in clusterBeforeSleep(). */
     /* Messages received and sent by type. */
diff --git a/src/config.c b/src/config.c
index 237dc82ad..64a31fc67 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1288,7 +1288,7 @@ void rewriteConfigNumericalOption(struct rewriteConfigState *state, const char *
     rewriteConfigRewriteLine(state,option,line,force);
 }
 
-/* Rewrite a octal option. */
+/* Rewrite an octal option. */
 void rewriteConfigOctalOption(struct rewriteConfigState *state, char *option, int value, int defvalue) {
     int force = value != defvalue;
     sds line = sdscatprintf(sdsempty(),"%s %o",option,value);
@@ -2106,7 +2106,7 @@ static int isValidAOFfilename(char *val, char **err) {
 static int updateHZ(long long val, long long prev, char **err) {
     UNUSED(prev);
     UNUSED(err);
-    /* Hz is more an hint from the user, so we accept values out of range
+    /* Hz is more a hint from the user, so we accept values out of range
      * but cap them to reasonable values. */
     server.config_hz = val;
     if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ;
@@ -2124,7 +2124,7 @@ static int updateJemallocBgThread(int val, int prev, char **err) {
 
 static int updateReplBacklogSize(long long val, long long prev, char **err) {
     /* resizeReplicationBacklog sets server.repl_backlog_size, and relies on
-     * being able to tell when the size changes, so restore prev becore calling it. */
+     * being able to tell when the size changes, so restore prev before calling it. */
     UNUSED(err);
     server.repl_backlog_size = prev;
     resizeReplicationBacklog(val);
diff --git a/src/config.h b/src/config.h
index e807b9330..f9ec7e44a 100644
--- a/src/config.h
+++ b/src/config.h
@@ -166,7 +166,7 @@ void setproctitle(const char *fmt, ...);
 #endif /* BYTE_ORDER */
 
 /* Sometimes after including an OS-specific header that defines the
- * endianess we end with __BYTE_ORDER but not with BYTE_ORDER that is what
+ * endianness we end with __BYTE_ORDER but not with BYTE_ORDER that is what
  * the Redis code uses. In this case let's define everything without the
  * underscores. */
 #ifndef BYTE_ORDER
diff --git a/src/connection.h b/src/connection.h
index 85585a3d0..e00d2ea17 100644
--- a/src/connection.h
+++ b/src/connection.h
@@ -106,7 +106,7 @@ static inline int connAccept(connection *conn, ConnectionCallbackFunc accept_han
 }
 
 /* Establish a connection.  The connect_handler will be called when the connection
- * is established, or if an error has occured.
+ * is established, or if an error has occurred.
  *
  * The connection handler will be responsible to set up any read/write handlers
  * as needed.
@@ -168,7 +168,7 @@ static inline int connSetReadHandler(connection *conn, ConnectionCallbackFunc fu
 
 /* Set a write handler, and possibly enable a write barrier, this flag is
  * cleared when write handler is changed or removed.
- * With barroer enabled, we never fire the event if the read handler already
+ * With barrier enabled, we never fire the event if the read handler already
  * fired in the same event loop iteration. Useful when you want to persist
  * things to disk before sending replies, and want to do that in a group fashion. */
 static inline int connSetWriteHandlerWithBarrier(connection *conn, ConnectionCallbackFunc func, int barrier) {
diff --git a/src/db.c b/src/db.c
index 70de568fb..2405cc2ab 100644
--- a/src/db.c
+++ b/src/db.c
@@ -116,7 +116,7 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) {
          * However, if the command caller is not the master, and as additional
          * safety measure, the command invoked is a read-only command, we can
          * safely return NULL here, and provide a more consistent behavior
-         * to clients accessign expired values in a read-only fashion, that
+         * to clients accessing expired values in a read-only fashion, that
          * will say the key as non existing.
          *
          * Notably this covers GETs when slaves are used to scale reads. */
@@ -371,7 +371,7 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) {
  *    firing module events.
  * and the function to return ASAP.
  *
- * On success the fuction returns the number of keys removed from the
+ * On success the function returns the number of keys removed from the
  * database(s). Otherwise -1 is returned in the specific case the
  * DB number is out of range, and errno is set to EINVAL. */
 long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(void*)) {
@@ -863,7 +863,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) {
         /* Filter element if it is an expired key. */
         if (!filter && o == NULL && expireIfNeeded(c->db, kobj)) filter = 1;
 
-        /* Remove the element and its associted value if needed. */
+        /* Remove the element and its associated value if needed. */
         if (filter) {
             decrRefCount(kobj);
             listDelNode(keys, node);
@@ -1359,7 +1359,7 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in
 /* Return all the arguments that are keys in the command passed via argc / argv.
  *
  * The command returns the positions of all the key arguments inside the array,
- * so the actual return value is an heap allocated array of integers. The
+ * so the actual return value is a heap allocated array of integers. The
  * length of the array is returned by reference into *numkeys.
  *
  * 'cmd' must be point to the corresponding entry into the redisCommand
diff --git a/src/debug.c b/src/debug.c
index 82b34e08d..a59c46046 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -398,7 +398,7 @@ void debugCommand(client *c) {
 "OOM -- Crash the server simulating an out-of-memory error.",
 "PANIC -- Crash the server simulating a panic.",
 "POPULATE <count> [prefix] [size] -- Create <count> string keys named key:<num>. If a prefix is specified is used instead of the 'key' prefix.",
-"RELOAD [MERGE] [NOFLUSH] [NOSAVE] -- Save the RDB on disk and reload it back in memory. By default it will save the RDB file and load it back. With the NOFLUSH option the current database is not removed before loading the new one, but conficts in keys will kill the server with an exception. When MERGE is used, conflicting keys will be loaded (the key in the loaded RDB file will win). When NOSAVE is used, the server will not save the current dataset in the RDB file before loading. Use DEBUG RELOAD NOSAVE when you want just to load the RDB file you placed in the Redis working directory in order to replace the current dataset in memory. Use DEBUG RELOAD NOSAVE NOFLUSH MERGE when you want to add what is in the current RDB file placed in the Redis current directory, with the current memory content. Use DEBUG RELOAD when you want to verify Redis is able to persist the current dataset in the RDB file, flush the memory content, and load it back.",
+"RELOAD [MERGE] [NOFLUSH] [NOSAVE] -- Save the RDB on disk and reload it back in memory. By default it will save the RDB file and load it back. With the NOFLUSH option the current database is not removed before loading the new one, but conflicts in keys will kill the server with an exception. When MERGE is used, conflicting keys will be loaded (the key in the loaded RDB file will win). When NOSAVE is used, the server will not save the current dataset in the RDB file before loading. Use DEBUG RELOAD NOSAVE when you want just to load the RDB file you placed in the Redis working directory in order to replace the current dataset in memory. Use DEBUG RELOAD NOSAVE NOFLUSH MERGE when you want to add what is in the current RDB file placed in the Redis current directory, with the current memory content. Use DEBUG RELOAD when you want to verify Redis is able to persist the current dataset in the RDB file, flush the memory content, and load it back.",
 "RESTART -- Graceful restart: save config, db, restart.",
 "SDSLEN <key> -- Show low level SDS string info representing key and value.",
 "SEGFAULT -- Crash the server with sigsegv.",
@@ -467,7 +467,7 @@ NULL
             }
         }
 
-        /* The default beahvior is to save the RDB file before loading
+        /* The default behavior is to save the RDB file before loading
          * it back. */
         if (save) {
             rdbSaveInfo rsi, *rsiptr;
@@ -1502,7 +1502,7 @@ void logCurrentClient(void) {
 
 #define MEMTEST_MAX_REGIONS 128
 
-/* A non destructive memory test executed during segfauls. */
+/* A non destructive memory test executed during segfault. */
 int memtest_test_linux_anonymous_maps(void) {
     FILE *fp;
     char line[1024];
diff --git a/src/defrag.c b/src/defrag.c
index 07a16ca6c..04ade30ea 100644
--- a/src/defrag.c
+++ b/src/defrag.c
@@ -47,11 +47,11 @@ int je_get_defrag_hint(void* ptr);
 
 /* forward declarations*/
 void defragDictBucketCallback(void *privdata, dictEntry **bucketref);
-dictEntry* replaceSateliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sds newkey, uint64_t hash, long *defragged);
+dictEntry* replaceSatelliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sds newkey, uint64_t hash, long *defragged);
 
 /* Defrag helper for generic allocations.
  *
- * returns NULL in case the allocatoin wasn't moved.
+ * returns NULL in case the allocation wasn't moved.
  * when it returns a non-null value, the old pointer was already released
  * and should NOT be accessed. */
 void* activeDefragAlloc(void *ptr) {
@@ -74,7 +74,7 @@ void* activeDefragAlloc(void *ptr) {
 
 /*Defrag helper for sds strings
  *
- * returns NULL in case the allocatoin wasn't moved.
+ * returns NULL in case the allocation wasn't moved.
  * when it returns a non-null value, the old pointer was already released
  * and should NOT be accessed. */
 sds activeDefragSds(sds sdsptr) {
@@ -90,7 +90,7 @@ sds activeDefragSds(sds sdsptr) {
 
 /* Defrag helper for robj and/or string objects
  *
- * returns NULL in case the allocatoin wasn't moved.
+ * returns NULL in case the allocation wasn't moved.
  * when it returns a non-null value, the old pointer was already released
  * and should NOT be accessed. */
 robj *activeDefragStringOb(robj* ob, long *defragged) {
@@ -130,11 +130,11 @@ robj *activeDefragStringOb(robj* ob, long *defragged) {
 }
 
 /* Defrag helper for dictEntries to be used during dict iteration (called on
- * each step). Teturns a stat of how many pointers were moved. */
+ * each step). Returns a stat of how many pointers were moved. */
 long dictIterDefragEntry(dictIterator *iter) {
     /* This function is a little bit dirty since it messes with the internals
      * of the dict and it's iterator, but the benefit is that it is very easy
-     * to use, and require no other chagnes in the dict. */
+     * to use, and require no other changes in the dict. */
     long defragged = 0;
     dictht *ht;
     /* Handle the next entry (if there is one), and update the pointer in the
@@ -238,7 +238,7 @@ double *zslDefrag(zskiplist *zsl, double score, sds oldele, sds newele) {
     return NULL;
 }
 
-/* Defrag helpler for sorted set.
+/* Defrag helper for sorted set.
  * Defrag a single dict entry key name, and corresponding skiplist struct */
 long activeDefragZsetEntry(zset *zs, dictEntry *de) {
     sds newsds;
@@ -349,7 +349,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) {
         if ((newsds = activeDefragSds(sdsele))) {
             /* When defragging an sds value, we need to update the dict key */
             uint64_t hash = dictGetHash(d, newsds);
-            replaceSateliteDictKeyPtrAndOrDefragDictEntry(d, sdsele, newsds, hash, &defragged);
+            replaceSatelliteDictKeyPtrAndOrDefragDictEntry(d, sdsele, newsds, hash, &defragged);
             ln->value = newsds;
             defragged++;
         }
@@ -385,7 +385,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) {
  * moved. Return value is the the dictEntry if found, or NULL if not found.
  * NOTE: this is very ugly code, but it let's us avoid the complication of
  * doing a scan on another dict. */
-dictEntry* replaceSateliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sds newkey, uint64_t hash, long *defragged) {
+dictEntry* replaceSatelliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sds newkey, uint64_t hash, long *defragged) {
     dictEntry **deref = dictFindEntryRefByPtrAndHash(d, oldkey, hash);
     if (deref) {
         dictEntry *de = *deref;
@@ -433,7 +433,7 @@ long activeDefragQuickListNodes(quicklist *ql) {
 }
 
 /* when the value has lots of elements, we want to handle it later and not as
- * oart of the main dictionary scan. this is needed in order to prevent latency
+ * part of the main dictionary scan. this is needed in order to prevent latency
  * spikes when handling large items */
 void defragLater(redisDb *db, dictEntry *kde) {
     sds key = sdsdup(dictGetKey(kde));
@@ -814,7 +814,7 @@ long defragKey(redisDb *db, dictEntry *de) {
           * I can't search in db->expires for that key after i already released
           * the pointer it holds it won't be able to do the string compare */
         uint64_t hash = dictGetHash(db->dict, de->key);
-        replaceSateliteDictKeyPtrAndOrDefragDictEntry(db->expires, keysds, newsds, hash, &defragged);
+        replaceSatelliteDictKeyPtrAndOrDefragDictEntry(db->expires, keysds, newsds, hash, &defragged);
     }
 
     /* Try to defrag robj and / or string value. */
@@ -885,7 +885,7 @@ void defragScanCallback(void *privdata, const dictEntry *de) {
     server.stat_active_defrag_scanned++;
 }
 
-/* Defrag scan callback for each hash table bicket,
+/* Defrag scan callback for each hash table bucket,
  * used in order to defrag the dictEntry allocations. */
 void defragDictBucketCallback(void *privdata, dictEntry **bucketref) {
     UNUSED(privdata); /* NOTE: this function is also used by both activeDefragCycle and scanLaterHash, etc. don't use privdata */
@@ -919,7 +919,7 @@ float getAllocatorFragmentation(size_t *out_frag_bytes) {
     return frag_pct;
 }
 
-/* We may need to defrag other globals, one small allcation can hold a full allocator run.
+/* We may need to defrag other globals, one small allocation can hold a full allocator run.
  * so although small, it is still important to defrag these */
 long defragOtherGlobals() {
     long defragged = 0;
@@ -1090,7 +1090,7 @@ void activeDefragCycle(void) {
     if (hasActiveChildProcess())
         return; /* Defragging memory while there's a fork will just do damage. */
 
-    /* Once a second, check if we the fragmentation justfies starting a scan
+    /* Once a second, check if the fragmentation justfies starting a scan
      * or making it more aggressive. */
     run_with_period(1000) {
         computeDefragCycles();
@@ -1160,7 +1160,7 @@ void activeDefragCycle(void) {
              * (if we have a lot of pointers in one hash bucket or rehasing),
              * check if we reached the time limit.
              * But regardless, don't start a new db in this loop, this is because after
-             * the last db we call defragOtherGlobals, which must be done in once cycle */
+             * the last db we call defragOtherGlobals, which must be done in one cycle */
             if (!cursor || (++iterations > 16 ||
                             server.stat_active_defrag_hits - prev_defragged > 512 ||
                             server.stat_active_defrag_scanned - prev_scanned > 64)) {
diff --git a/src/dict.c b/src/dict.c
index 39af94707..4760a5d26 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -237,7 +237,9 @@ long long timeInMilliseconds(void) {
     return (((long long)tv.tv_sec)*1000)+(tv.tv_usec/1000);
 }
 
-/* Rehash for an amount of time between ms milliseconds and ms+1 milliseconds */
+/* Rehash in ms+"delta" milliseconds. The value of "delta" is larger 
+ * than 0, and is smaller than 1 in most cases. The exact upper bound 
+ * depends on the running time of dictRehash(d,100).*/
 int dictRehashMilliseconds(dict *d, int ms) {
     if (d->iterators > 0) return 0;
 
@@ -749,7 +751,7 @@ unsigned int dictGetSomeKeys(dict *d, dictEntry **des, unsigned int count) {
  * this function instead what we do is to consider a "linear" range of the table
  * that may be constituted of N buckets with chains of different lengths
  * appearing one after the other. Then we report a random element in the range.
- * In this way we smooth away the problem of different chain lenghts. */
+ * In this way we smooth away the problem of different chain lengths. */
 #define GETFAIR_NUM_ENTRIES 15
 dictEntry *dictGetFairRandomKey(dict *d) {
     dictEntry *entries[GETFAIR_NUM_ENTRIES];
@@ -1119,7 +1121,7 @@ size_t _dictGetStatsHt(char *buf, size_t bufsize, dictht *ht, int tableid) {
             i, clvector[i], ((float)clvector[i]/ht->size)*100);
     }
 
-    /* Unlike snprintf(), teturn the number of characters actually written. */
+    /* Unlike snprintf(), return the number of characters actually written. */
     if (bufsize) buf[bufsize-1] = '\0';
     return strlen(buf);
 }
diff --git a/src/endianconv.c b/src/endianconv.c
index f3b0b4730..918844e25 100644
--- a/src/endianconv.c
+++ b/src/endianconv.c
@@ -8,7 +8,7 @@
  * to be backward compatible are still in big endian) because most of the
  * production environments are little endian, and we have a lot of conversions
  * in a few places because ziplists, intsets, zipmaps, need to be endian-neutral
- * even in memory, since they are serialied on RDB files directly with a single
+ * even in memory, since they are serialized on RDB files directly with a single
  * write(2) without other additional steps.
  *
  * ----------------------------------------------------------------------------
diff --git a/src/evict.c b/src/evict.c
index 0755acc0e..5d398c6c9 100644
--- a/src/evict.c
+++ b/src/evict.c
@@ -41,7 +41,7 @@
 /* To improve the quality of the LRU approximation we take a set of keys
  * that are good candidate for eviction across freeMemoryIfNeeded() calls.
  *
- * Entries inside the eviciton pool are taken ordered by idle time, putting
+ * Entries inside the eviction pool are taken ordered by idle time, putting
  * greater idle times to the right (ascending order).
  *
  * When an LFU policy is used instead, a reverse frequency indication is used
@@ -242,7 +242,7 @@ void evictionPoolPopulate(int dbid, dict *sampledict, dict *keydict, struct evic
         /* Try to reuse the cached SDS string allocated in the pool entry,
          * because allocating and deallocating this object is costly
          * (according to the profiler, not my fantasy. Remember:
-         * premature optimizbla bla bla bla. */
+         * premature optimization bla bla bla. */
         int klen = sdslen(key);
         if (klen > EVPOOL_CACHED_SDS_SIZE) {
             pool[k].key = sdsdup(key);
@@ -342,7 +342,7 @@ unsigned long LFUDecrAndReturn(robj *o) {
 }
 
 /* ----------------------------------------------------------------------------
- * The external API for eviction: freeMemroyIfNeeded() is called by the
+ * The external API for eviction: freeMemoryIfNeeded() is called by the
  * server when there is data to add in order to make space if needed.
  * --------------------------------------------------------------------------*/
 
@@ -441,7 +441,7 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev
  *
  * The function returns C_OK if we are under the memory limit or if we
  * were over the limit, but the attempt to free memory was successful.
- * Otehrwise if we are over the memory limit, but not enough memory
+ * Otherwise if we are over the memory limit, but not enough memory
  * was freed to return back under the limit, the function returns C_ERR. */
 int freeMemoryIfNeeded(void) {
     int keys_freed = 0;
diff --git a/src/expire.c b/src/expire.c
index 1c4f71df3..85fd59fe2 100644
--- a/src/expire.c
+++ b/src/expire.c
@@ -97,7 +97,7 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) {
  * conditions:
  *
  * If type is ACTIVE_EXPIRE_CYCLE_FAST the function will try to run a
- * "fast" expire cycle that takes no longer than EXPIRE_FAST_CYCLE_DURATION
+ * "fast" expire cycle that takes no longer than ACTIVE_EXPIRE_CYCLE_FAST_DURATION
  * microseconds, and is not repeated again before the same amount of time.
  * The cycle will also refuse to run at all if the latest slow cycle did not
  * terminate because of a time limit condition.
@@ -414,7 +414,7 @@ void expireSlaveKeys(void) {
         else
             dictDelete(slaveKeysWithExpire,keyname);
 
-        /* Stop conditions: found 3 keys we cna't expire in a row or
+        /* Stop conditions: found 3 keys we can't expire in a row or
          * time limit was reached. */
         cycles++;
         if (noexpire > 3) break;
@@ -466,7 +466,7 @@ size_t getSlaveKeyWithExpireCount(void) {
  *
  * Note: technically we should handle the case of a single DB being flushed
  * but it is not worth it since anyway race conditions using the same set
- * of key names in a wriatable slave and in its master will lead to
+ * of key names in a writable slave and in its master will lead to
  * inconsistencies. This is just a best-effort thing we do. */
 void flushSlaveKeysWithExpireList(void) {
     if (slaveKeysWithExpire) {
@@ -490,7 +490,7 @@ int checkAlreadyExpired(long long when) {
  *----------------------------------------------------------------------------*/
 
 /* This is the generic command implementation for EXPIRE, PEXPIRE, EXPIREAT
- * and PEXPIREAT. Because the commad second argument may be relative or absolute
+ * and PEXPIREAT. Because the command second argument may be relative or absolute
  * the "basetime" argument is used to signal what the base time is (either 0
  * for *AT variants of the command, or the current time for relative expires).
  *
diff --git a/src/geo.c b/src/geo.c
index 77f07637a..7594bb2e9 100644
--- a/src/geo.c
+++ b/src/geo.c
@@ -143,8 +143,8 @@ double extractUnitOrReply(client *c, robj *unit) {
 }
 
 /* Input Argument Helper.
- * Extract the dinstance from the specified two arguments starting at 'argv'
- * that shouldbe in the form: <number> <unit> and return the dinstance in the
+ * Extract the distance from the specified two arguments starting at 'argv'
+ * that should be in the form: <number> <unit>, and return the distance in the
  * specified unit on success. *conversions is populated with the coefficient
  * to use in order to convert meters to the unit.
  *
@@ -788,7 +788,7 @@ void geoposCommand(client *c) {
 
 /* GEODIST key ele1 ele2 [unit]
  *
- * Return the distance, in meters by default, otherwise accordig to "unit",
+ * Return the distance, in meters by default, otherwise according to "unit",
  * between points ele1 and ele2. If one or more elements are missing NULL
  * is returned. */
 void geodistCommand(client *c) {
diff --git a/src/geohash_helper.c b/src/geohash_helper.c
index e23f17b4e..01fb2cb88 100644
--- a/src/geohash_helper.c
+++ b/src/geohash_helper.c
@@ -68,7 +68,7 @@ uint8_t geohashEstimateStepsByRadius(double range_meters, double lat) {
     }
     step -= 2; /* Make sure range is included in most of the base cases. */
 
-    /* Wider range torwards the poles... Note: it is possible to do better
+    /* Wider range towards the poles... Note: it is possible to do better
      * than this approximation by computing the distance between meridians
      * at this latitude, but this does the trick for now. */
     if (lat > 66 || lat < -66) {
@@ -84,7 +84,7 @@ uint8_t geohashEstimateStepsByRadius(double range_meters, double lat) {
 
 /* Return the bounding box of the search area centered at latitude,longitude
  * having a radius of radius_meter. bounds[0] - bounds[2] is the minimum
- * and maxium longitude, while bounds[1] - bounds[3] is the minimum and
+ * and maximum longitude, while bounds[1] - bounds[3] is the minimum and
  * maximum latitude.
  *
  * This function does not behave correctly with very large radius values, for
diff --git a/src/hyperloglog.c b/src/hyperloglog.c
index 721f492a1..d018e975e 100644
--- a/src/hyperloglog.c
+++ b/src/hyperloglog.c
@@ -36,9 +36,9 @@
 
 /* The Redis HyperLogLog implementation is based on the following ideas:
  *
- * * The use of a 64 bit hash function as proposed in [1], in order to don't
- *   limited to cardinalities up to 10^9, at the cost of just 1 additional
- *   bit per register.
+ * * The use of a 64 bit hash function as proposed in [1], in order to estimate
+ *   cardinalities larger than 10^9, at the cost of just 1 additional bit per
+ *   register.
  * * The use of 16384 6-bit registers for a great level of accuracy, using
  *   a total of 12k per key.
  * * The use of the Redis string data type. No new type is introduced.
@@ -279,7 +279,7 @@ static char *invalid_hll_err = "-INVALIDOBJ Corrupted HLL object detected\r\n";
  *  So we right shift of 0 bits (no shift in practice) and
  *  left shift the next byte of 8 bits, even if we don't use it,
  *  but this has the effect of clearing the bits so the result
- *  will not be affacted after the OR.
+ *  will not be affected after the OR.
  *
  * -------------------------------------------------------------------------
  *
@@ -297,7 +297,7 @@ static char *invalid_hll_err = "-INVALIDOBJ Corrupted HLL object detected\r\n";
  *   |11000000|  <- Our byte at b0
  *   +--------+
  *
- * To create a AND-mask to clear the bits about this position, we just
+ * To create an AND-mask to clear the bits about this position, we just
  * initialize the mask with the value 63, left shift it of "fs" bits,
  * and finally invert the result.
  *
@@ -766,7 +766,7 @@ int hllSparseSet(robj *o, long index, uint8_t count) {
      * by a ZERO opcode with len > 1, or by an XZERO opcode.
      *
      * In those cases the original opcode must be split into multiple
-     * opcodes. The worst case is an XZERO split in the middle resuling into
+     * opcodes. The worst case is an XZERO split in the middle resulting into
      * XZERO - VAL - XZERO, so the resulting sequence max length is
      * 5 bytes.
      *
@@ -899,7 +899,7 @@ promote: /* Promote to dense representation. */
  * the element belongs to is incremented if needed.
  *
  * This function is actually a wrapper for hllSparseSet(), it only performs
- * the hashshing of the elmenet to obtain the index and zeros run length. */
+ * the hashshing of the element to obtain the index and zeros run length. */
 int hllSparseAdd(robj *o, unsigned char *ele, size_t elesize) {
     long index;
     uint8_t count = hllPatLen(ele,elesize,&index);
@@ -1014,7 +1014,7 @@ uint64_t hllCount(struct hllhdr *hdr, int *invalid) {
     double m = HLL_REGISTERS;
     double E;
     int j;
-    /* Note that reghisto size could be just HLL_Q+2, becuase HLL_Q+1 is
+    /* Note that reghisto size could be just HLL_Q+2, because HLL_Q+1 is
      * the maximum frequency of the "000...1" sequence the hash function is
      * able to return. However it is slow to check for sanity of the
      * input: instead we history array at a safe size: overflows will
diff --git a/src/latency.c b/src/latency.c
index b5ccc7cc6..6148543c8 100644
--- a/src/latency.c
+++ b/src/latency.c
@@ -85,7 +85,7 @@ int THPGetAnonHugePagesSize(void) {
 /* ---------------------------- Latency API --------------------------------- */
 
 /* Latency monitor initialization. We just need to create the dictionary
- * of time series, each time serie is created on demand in order to avoid
+ * of time series, each time series is created on demand in order to avoid
  * having a fixed list to maintain. */
 void latencyMonitorInit(void) {
     server.latency_events = dictCreate(&latencyTimeSeriesDictType,NULL);
@@ -154,7 +154,7 @@ int latencyResetEvent(char *event_to_reset) {
 
 /* Analyze the samples available for a given event and return a structure
  * populate with different metrics, average, MAD, min, max, and so forth.
- * Check latency.h definition of struct latenctStat for more info.
+ * Check latency.h definition of struct latencyStats for more info.
  * If the specified event has no elements the structure is populate with
  * zero values. */
 void analyzeLatencyForEvent(char *event, struct latencyStats *ls) {
@@ -343,7 +343,7 @@ sds createLatencyReport(void) {
         }
 
         if (!strcasecmp(event,"aof-fstat") ||
-            !strcasecmp(event,"rdb-unlik-temp-file")) {
+            !strcasecmp(event,"rdb-unlink-temp-file")) {
             advise_disk_contention = 1;
             advise_local_disk = 1;
             advices += 2;
@@ -396,7 +396,7 @@ sds createLatencyReport(void) {
         /* Better VM. */
         report = sdscat(report,"\nI have a few advices for you:\n\n");
         if (advise_better_vm) {
-            report = sdscat(report,"- If you are using a virtual machine, consider upgrading it with a faster one using an hypervisior that provides less latency during fork() calls. Xen is known to have poor fork() performance. Even in the context of the same VM provider, certain kinds of instances can execute fork faster than others.\n");
+            report = sdscat(report,"- If you are using a virtual machine, consider upgrading it with a faster one using a hypervisior that provides less latency during fork() calls. Xen is known to have poor fork() performance. Even in the context of the same VM provider, certain kinds of instances can execute fork faster than others.\n");
         }
 
         /* Slow log. */
@@ -416,7 +416,7 @@ sds createLatencyReport(void) {
         if (advise_scheduler) {
             report = sdscat(report,"- The system is slow to execute Redis code paths not containing system calls. This usually means the system does not provide Redis CPU time to run for long periods. You should try to:\n"
             "  1) Lower the system load.\n"
-            "  2) Use a computer / VM just for Redis if you are running other softawre in the same system.\n"
+            "  2) Use a computer / VM just for Redis if you are running other software in the same system.\n"
             "  3) Check if you have a \"noisy neighbour\" problem.\n"
             "  4) Check with 'redis-cli --intrinsic-latency 100' what is the intrinsic latency in your system.\n"
             "  5) Check if the problem is allocator-related by recompiling Redis with MALLOC=libc, if you are using Jemalloc. However this may create fragmentation problems.\n");
@@ -432,7 +432,7 @@ sds createLatencyReport(void) {
         }
 
         if (advise_data_writeback) {
-            report = sdscat(report,"- Mounting ext3/4 filesystems with data=writeback can provide a performance boost compared to data=ordered, however this mode of operation provides less guarantees, and sometimes it can happen that after a hard crash the AOF file will have an half-written command at the end and will require to be repaired before Redis restarts.\n");
+            report = sdscat(report,"- Mounting ext3/4 filesystems with data=writeback can provide a performance boost compared to data=ordered, however this mode of operation provides less guarantees, and sometimes it can happen that after a hard crash the AOF file will have a half-written command at the end and will require to be repaired before Redis restarts.\n");
         }
 
         if (advise_disk_contention) {
diff --git a/src/lazyfree.c b/src/lazyfree.c
index cbcc1c240..821dc50df 100644
--- a/src/lazyfree.c
+++ b/src/lazyfree.c
@@ -15,7 +15,7 @@ size_t lazyfreeGetPendingObjectsCount(void) {
 
 /* Return the amount of work needed in order to free an object.
  * The return value is not always the actual number of allocations the
- * object is compoesd of, but a number proportional to it.
+ * object is composed of, but a number proportional to it.
  *
  * For strings the function always returns 1.
  *
@@ -137,7 +137,7 @@ void emptyDbAsync(redisDb *db) {
 }
 
 /* Empty the slots-keys map of Redis CLuster by creating a new empty one
- * and scheduiling the old for lazy freeing. */
+ * and scheduling the old for lazy freeing. */
 void slotToKeyFlushAsync(void) {
     rax *old = server.cluster->slots_to_keys;
 
@@ -156,7 +156,7 @@ void lazyfreeFreeObjectFromBioThread(robj *o) {
 }
 
 /* Release a database from the lazyfree thread. The 'db' pointer is the
- * database which was substitutied with a fresh one in the main thread
+ * database which was substituted with a fresh one in the main thread
  * when the database was logically deleted. 'sl' is a skiplist used by
  * Redis Cluster in order to take the hash slots -> keys mapping. This
  * may be NULL if Redis Cluster is disabled. */
diff --git a/src/listpack.c b/src/listpack.c
index 9e77ab12d..075552ccb 100644
--- a/src/listpack.c
+++ b/src/listpack.c
@@ -405,7 +405,7 @@ unsigned char *lpNext(unsigned char *lp, unsigned char *p) {
 }
 
 /* If 'p' points to an element of the listpack, calling lpPrev() will return
- * the pointer to the preivous element (the one on the left), or NULL if 'p'
+ * the pointer to the previous element (the one on the left), or NULL if 'p'
  * already pointed to the first element of the listpack. */
 unsigned char *lpPrev(unsigned char *lp, unsigned char *p) {
     if (p-lp == LP_HDR_SIZE) return NULL;
diff --git a/src/lolwut.c b/src/lolwut.c
index 0e1552ba0..eebd5da6a 100644
--- a/src/lolwut.c
+++ b/src/lolwut.c
@@ -85,7 +85,7 @@ void lolwutCommand(client *c) {
 }
 
 /* ========================== LOLWUT Canvase ===============================
- * Many LOWUT versions will likely print some computer art to the screen.
+ * Many LOLWUT versions will likely print some computer art to the screen.
  * This is the case with LOLWUT 5 and LOLWUT 6, so here there is a generic
  * canvas implementation that can be reused.  */
 
@@ -106,7 +106,7 @@ void lwFreeCanvas(lwCanvas *canvas) {
 }
 
 /* Set a pixel to the specified color. Color is 0 or 1, where zero means no
- * dot will be displyed, and 1 means dot will be displayed.
+ * dot will be displayed, and 1 means dot will be displayed.
  * Coordinates are arranged so that left-top corner is 0,0. You can write
  * out of the size of the canvas without issues. */
 void lwDrawPixel(lwCanvas *canvas, int x, int y, int color) {
diff --git a/src/lolwut5.c b/src/lolwut5.c
index 5a9348800..d64e0bb27 100644
--- a/src/lolwut5.c
+++ b/src/lolwut5.c
@@ -156,7 +156,7 @@ void lolwut5Command(client *c) {
         return;
 
     /* Limits. We want LOLWUT to be always reasonably fast and cheap to execute
-     * so we have maximum number of columns, rows, and output resulution. */
+     * so we have maximum number of columns, rows, and output resolution. */
     if (cols < 1) cols = 1;
     if (cols > 1000) cols = 1000;
     if (squares_per_row < 1) squares_per_row = 1;
diff --git a/src/lzfP.h b/src/lzfP.h
index 93c27b42d..78c858fad 100644
--- a/src/lzfP.h
+++ b/src/lzfP.h
@@ -127,7 +127,7 @@
 
 /*
  * Whether to store pointers or offsets inside the hash table. On
- * 64 bit architetcures, pointers take up twice as much space,
+ * 64 bit architectures, pointers take up twice as much space,
  * and might also be slower. Default is to autodetect.
  */
 /*#define LZF_USER_OFFSETS autodetect */
diff --git a/src/module.c b/src/module.c
index 17178e6a6..315161ed9 100644
--- a/src/module.c
+++ b/src/module.c
@@ -46,7 +46,7 @@ typedef struct RedisModuleInfoCtx {
     sds info;           /* info string we collected so far */
     int sections;       /* number of sections we collected so far */
     int in_section;     /* indication if we're in an active section or not */
-    int in_dict_field;  /* indication that we're curreintly appending to a dict */
+    int in_dict_field;  /* indication that we're currently appending to a dict */
 } RedisModuleInfoCtx;
 
 typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report);
@@ -906,10 +906,21 @@ int RM_SignalModifiedKey(RedisModuleCtx *ctx, RedisModuleString *keyname) {
  * Automatic memory management for modules
  * -------------------------------------------------------------------------- */
 
-/* Enable automatic memory management. See API.md for more information.
+/* Enable automatic memory management.
  *
  * The function must be called as the first function of a command implementation
- * that wants to use automatic memory. */
+ * that wants to use automatic memory.
+ *
+ * When enabled, automatic memory management tracks and automatically frees
+ * keys, call replies and Redis string objects once the command returns. In most
+ * cases this eliminates the need of calling the following functions:
+ *
+ * 1) RedisModule_CloseKey()
+ * 2) RedisModule_FreeCallReply()
+ * 3) RedisModule_FreeString()
+ *
+ * These functions can still be used with automatic memory management enabled,
+ * to optimize loops that make numerous allocations for example. */
 void RM_AutoMemory(RedisModuleCtx *ctx) {
     ctx->flags |= REDISMODULE_CTX_AUTO_MEMORY;
 }
@@ -1045,7 +1056,7 @@ RedisModuleString *RM_CreateStringFromLongLong(RedisModuleCtx *ctx, long long ll
 }
 
 /* Like RedisModule_CreatString(), but creates a string starting from a double
- * integer instead of taking a buffer and its length.
+ * instead of taking a buffer and its length.
  *
  * The returned string must be released with RedisModule_FreeString() or by
  * enabling automatic memory management. */
@@ -1922,7 +1933,7 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) {
              flags |= REDISMODULE_CTX_FLAGS_LUA;
             if (ctx->client->flags & CLIENT_MULTI)
              flags |= REDISMODULE_CTX_FLAGS_MULTI;
-            /* Module command recieved from MASTER, is replicated. */
+            /* Module command received from MASTER, is replicated. */
             if (ctx->client->flags & CLIENT_MASTER)
              flags |= REDISMODULE_CTX_FLAGS_REPLICATED;
         }
@@ -2921,7 +2932,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) {
 /* Get fields from an hash value. This function is called using a variable
  * number of arguments, alternating a field name (as a StringRedisModule
  * pointer) with a pointer to a StringRedisModule pointer, that is set to the
- * value of the field if the field exist, or NULL if the field did not exist.
+ * value of the field if the field exists, or NULL if the field does not exist.
  * At the end of the field/value-ptr pairs, NULL must be specified as last
  * argument to signal the end of the arguments in the variadic function.
  *
@@ -3040,7 +3051,7 @@ void moduleParseCallReply_SimpleString(RedisModuleCallReply *reply);
 void moduleParseCallReply_Array(RedisModuleCallReply *reply);
 
 /* Do nothing if REDISMODULE_REPLYFLAG_TOPARSE is false, otherwise
- * use the protcol of the reply in reply->proto in order to fill the
+ * use the protocol of the reply in reply->proto in order to fill the
  * reply with parsed data according to the reply type. */
 void moduleParseCallReply(RedisModuleCallReply *reply) {
     if (!(reply->flags & REDISMODULE_REPLYFLAG_TOPARSE)) return;
@@ -3599,7 +3610,7 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) {
 
 /* Register a new data type exported by the module. The parameters are the
  * following. Please for in depth documentation check the modules API
- * documentation, especially the TYPES.md file.
+ * documentation, especially https://redis.io/topics/modules-native-types.
  *
  * * **name**: A 9 characters data type name that MUST be unique in the Redis
  *   Modules ecosystem. Be creative... and there will be no collisions. Use
@@ -3646,7 +3657,7 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) {
  * * **aux_load**: A callback function pointer that loads out of keyspace data from RDB files.
  *   Similar to aux_save, returns REDISMODULE_OK on success, and ERR otherwise.
  *
- * The **digest* and **mem_usage** methods should currently be omitted since
+ * The **digest** and **mem_usage** methods should currently be omitted since
  * they are not yet implemented inside the Redis modules core.
  *
  * Note: the module name "AAAAAAAAA" is reserved and produces an error, it
@@ -3656,7 +3667,7 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) {
  * and if the module name or encver is invalid, NULL is returned.
  * Otherwise the new type is registered into Redis, and a reference of
  * type RedisModuleType is returned: the caller of the function should store
- * this reference into a gobal variable to make future use of it in the
+ * this reference into a global variable to make future use of it in the
  * modules type API, since a single module may register multiple types.
  * Example code fragment:
  *
@@ -3738,7 +3749,7 @@ moduleType *RM_ModuleTypeGetType(RedisModuleKey *key) {
 
 /* Assuming RedisModule_KeyType() returned REDISMODULE_KEYTYPE_MODULE on
  * the key, returns the module type low-level value stored at key, as
- * it was set by the user via RedisModule_ModuleTypeSet().
+ * it was set by the user via RedisModule_ModuleTypeSetValue().
  *
  * If the key is NULL, is not associated with a module type, or is empty,
  * then NULL is returned instead. */
@@ -3795,7 +3806,7 @@ int moduleAllDatatypesHandleErrors() {
 
 /* Returns true if any previous IO API failed.
  * for Load* APIs the REDISMODULE_OPTIONS_HANDLE_IO_ERRORS flag must be set with
- * RediModule_SetModuleOptions first. */
+ * RedisModule_SetModuleOptions first. */
 int RM_IsIOError(RedisModuleIO *io) {
     return io->error;
 }
@@ -3928,7 +3939,7 @@ RedisModuleString *RM_LoadString(RedisModuleIO *io) {
  *
  * The size of the string is stored at '*lenptr' if not NULL.
  * The returned string is not automatically NULL terminated, it is loaded
- * exactly as it was stored inisde the RDB file. */
+ * exactly as it was stored inside the RDB file. */
 char *RM_LoadStringBuffer(RedisModuleIO *io, size_t *lenptr) {
     return moduleLoadString(io,1,lenptr);
 }
@@ -4517,14 +4528,14 @@ int moduleTryServeClientBlockedOnKey(client *c, robj *key) {
  *
  * The callbacks are called in the following contexts:
  *
- *     reply_callback:  called after a successful RedisModule_UnblockClient()
- *                      call in order to reply to the client and unblock it.
+ *     reply_callback:   called after a successful RedisModule_UnblockClient()
+ *                       call in order to reply to the client and unblock it.
  *
- *     reply_timeout:   called when the timeout is reached in order to send an
- *                      error to the client.
+ *     timeout_callback: called when the timeout is reached in order to send an
+ *                       error to the client.
  *
- *     free_privdata:   called in order to free the private data that is passed
- *                      by RedisModule_UnblockClient() call.
+ *     free_privdata:    called in order to free the private data that is passed
+ *                       by RedisModule_UnblockClient() call.
  *
  * Note: RedisModule_UnblockClient should be called for every blocked client,
  *       even if client was killed, timed-out or disconnected. Failing to do so
@@ -4547,13 +4558,13 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc
  * once certain keys become "ready", that is, contain more data.
  *
  * Basically this is similar to what a typical Redis command usually does,
- * like BLPOP or ZPOPMAX: the client blocks if it cannot be served ASAP,
+ * like BLPOP or BZPOPMAX: the client blocks if it cannot be served ASAP,
  * and later when the key receives new data (a list push for instance), the
  * client is unblocked and served.
  *
  * However in the case of this module API, when the client is unblocked?
  *
- * 1. If you block ok a key of a type that has blocking operations associated,
+ * 1. If you block on a key of a type that has blocking operations associated,
  *    like a list, a sorted set, a stream, and so forth, the client may be
  *    unblocked once the relevant key is targeted by an operation that normally
  *    unblocks the native blocking operations for that type. So if we block
@@ -4948,7 +4959,7 @@ void moduleReleaseGIL(void) {
 
 /* Subscribe to keyspace notifications. This is a low-level version of the
  * keyspace-notifications API. A module can register callbacks to be notified
- * when keyspce events occur.
+ * when keyspace events occur.
  *
  * Notification events are filtered by their type (string events, set events,
  * etc), and the subscriber callback receives only events that match a specific
@@ -5659,7 +5670,7 @@ int RM_AuthenticateClientWithACLUser(RedisModuleCtx *ctx, const char *name, size
 /* Deauthenticate and close the client. The client resources will not be
  * be immediately freed, but will be cleaned up in a background job. This is 
  * the recommended way to deauthenicate a client since most clients can't 
- * handle users becomming deauthenticated. Returns REDISMODULE_ERR when the 
+ * handle users becoming deauthenticated. Returns REDISMODULE_ERR when the
  * client doesn't exist and REDISMODULE_OK when the operation was successful. 
  * 
  * The client ID is returned from the RM_AuthenticateClientWithUser and
@@ -5779,14 +5790,14 @@ int RM_DictDel(RedisModuleDict *d, RedisModuleString *key, void *oldval) {
     return RM_DictDelC(d,key->ptr,sdslen(key->ptr),oldval);
 }
 
-/* Return an interator, setup in order to start iterating from the specified
+/* Return an iterator, setup in order to start iterating from the specified
  * key by applying the operator 'op', which is just a string specifying the
  * comparison operator to use in order to seek the first element. The
- * operators avalable are:
+ * operators available are:
  *
  * "^"   -- Seek the first (lexicographically smaller) key.
  * "$"   -- Seek the last  (lexicographically biffer) key.
- * ">"   -- Seek the first element greter than the specified key.
+ * ">"   -- Seek the first element greater than the specified key.
  * ">="  -- Seek the first element greater or equal than the specified key.
  * "<"   -- Seek the first element smaller than the specified key.
  * "<="  -- Seek the first element smaller or equal than the specified key.
@@ -5913,7 +5924,7 @@ RedisModuleString *RM_DictPrev(RedisModuleCtx *ctx, RedisModuleDictIter *di, voi
  * in the loop, as we iterate elements, we can also check if we are still
  * on range.
  *
- * The function returne REDISMODULE_ERR if the iterator reached the
+ * The function return REDISMODULE_ERR if the iterator reached the
  * end of elements condition as well. */
 int RM_DictCompareC(RedisModuleDictIter *di, const char *op, void *key, size_t keylen) {
     if (raxEOF(&di->ri)) return REDISMODULE_ERR;
@@ -6294,7 +6305,7 @@ int RM_ExportSharedAPI(RedisModuleCtx *ctx, const char *apiname, void *func) {
  * command that requires external APIs: if some API cannot be resolved, the
  * command should return an error.
  *
- * Here is an exmaple:
+ * Here is an example:
  *
  *     int ... myCommandImplementation() {
  *        if (getExternalAPIs() == 0) {
@@ -6680,7 +6691,7 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) {
  *      RedisModule_ScanCursorDestroy(c);
  *
  * It is also possible to use this API from another thread while the lock
- * is acquired durring the actuall call to RM_Scan:
+ * is acquired during the actuall call to RM_Scan:
  *
  *      RedisModuleCursor *c = RedisModule_ScanCursorCreate();
  *      RedisModule_ThreadSafeContextLock(ctx);
@@ -6694,7 +6705,7 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) {
  * The function will return 1 if there are more elements to scan and
  * 0 otherwise, possibly setting errno if the call failed.
  *
- * It is also possible to restart and existing cursor using RM_CursorRestart.
+ * It is also possible to restart an existing cursor using RM_ScanCursorRestart.
  *
  * IMPORTANT: This API is very similar to the Redis SCAN command from the
  * point of view of the guarantees it provides. This means that the API
@@ -6708,7 +6719,7 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) {
  * Moreover playing with the Redis keyspace while iterating may have the
  * effect of returning more duplicates. A safe pattern is to store the keys
  * names you want to modify elsewhere, and perform the actions on the keys
- * later when the iteration is complete. Howerver this can cost a lot of
+ * later when the iteration is complete. However this can cost a lot of
  * memory, so it may make sense to just operate on the current key when
  * possible during the iteration, given that this is safe. */
 int RM_Scan(RedisModuleCtx *ctx, RedisModuleScanCursor *cursor, RedisModuleScanCB fn, void *privdata) {
@@ -6773,8 +6784,8 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) {
  *      RedisModule_CloseKey(key);
  *      RedisModule_ScanCursorDestroy(c);
  *
- * It is also possible to use this API from another thread while the lock is acquired durring
- * the actuall call to RM_Scan, and re-opening the key each time:
+ * It is also possible to use this API from another thread while the lock is acquired during
+ * the actuall call to RM_ScanKey, and re-opening the key each time:
  *      RedisModuleCursor *c = RedisModule_ScanCursorCreate();
  *      RedisModule_ThreadSafeContextLock(ctx);
  *      RedisModuleKey *key = RedisModule_OpenKey(...)
@@ -6790,7 +6801,7 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) {
  *
  * The function will return 1 if there are more elements to scan and 0 otherwise,
  * possibly setting errno if the call failed.
- * It is also possible to restart and existing cursor using RM_CursorRestart.
+ * It is also possible to restart an existing cursor using RM_ScanCursorRestart.
  *
  * NOTE: Certain operations are unsafe while iterating the object. For instance
  * while the API guarantees to return at least one time all the elements that
@@ -6943,7 +6954,7 @@ int TerminateModuleForkChild(int child_pid, int wait) {
 }
 
 /* Can be used to kill the forked child process from the parent process.
- * child_pid whould be the return value of RedisModule_Fork. */
+ * child_pid would be the return value of RedisModule_Fork. */
 int RM_KillForkChild(int child_pid) {
     /* Kill module child, wait for child exit. */
     if (TerminateModuleForkChild(child_pid,1) == C_OK)
@@ -7081,7 +7092,7 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) {
  *              REDISMODULE_SUBEVENT_LOADING_FAILED
  *
  *          Note that AOF loading may start with an RDB data in case of
- *          rdb-preamble, in which case you'll only recieve an AOF_START event.
+ *          rdb-preamble, in which case you'll only receive an AOF_START event.
  *
  *
  *      RedisModuleEvent_ClientChange
@@ -7103,7 +7114,7 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) {
  *          This event is called when the instance (that can be both a
  *          master or a replica) get a new online replica, or lose a
  *          replica since it gets disconnected.
- *          The following sub events are availble:
+ *          The following sub events are available:
  *
  *              REDISMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE
  *              REDISMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE
@@ -7141,7 +7152,7 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) {
  *  RedisModuleEvent_ModuleChange
  *
  *          This event is called when a new module is loaded or one is unloaded.
- *          The following sub events are availble:
+ *          The following sub events are available:
  *
  *              REDISMODULE_SUBEVENT_MODULE_LOADED
  *              REDISMODULE_SUBEVENT_MODULE_UNLOADED
@@ -7168,7 +7179,7 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) {
  *              int32_t progress;  // Approximate progress between 0 and 1024,
  *                                    or -1 if unknown.
  *
- * The function returns REDISMODULE_OK if the module was successfully subscrived
+ * The function returns REDISMODULE_OK if the module was successfully subscribed
  * for the specified event. If the API is called from a wrong context then
  * REDISMODULE_ERR is returned. */
 int RM_SubscribeToServerEvent(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback) {
@@ -7374,7 +7385,7 @@ void moduleInitModulesSystem(void) {
     server.loadmodule_queue = listCreate();
     modules = dictCreate(&modulesDictType,NULL);
 
-    /* Set up the keyspace notification susbscriber list and static client */
+    /* Set up the keyspace notification subscriber list and static client */
     moduleKeyspaceSubscribers = listCreate();
 
     /* Set up filter list */
@@ -7735,7 +7746,7 @@ size_t moduleCount(void) {
     return dictSize(modules);
 }
 
-/* Set the key last access time for LRU based eviction. not relevent if the
+/* Set the key last access time for LRU based eviction. not relevant if the
  * servers's maxmemory policy is LFU based. Value is idle time in milliseconds.
  * returns REDISMODULE_OK if the LRU was updated, REDISMODULE_ERR otherwise. */
 int RM_SetLRU(RedisModuleKey *key, mstime_t lru_idle) {
diff --git a/src/modules/hellodict.c b/src/modules/hellodict.c
index 651615b03..1428a1381 100644
--- a/src/modules/hellodict.c
+++ b/src/modules/hellodict.c
@@ -125,7 +125,7 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
         cmd_KEYRANGE,"readonly",1,1,0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
 
-    /* Create our global dictionray. Here we'll set our keys and values. */
+    /* Create our global dictionary. Here we'll set our keys and values. */
     Keyspace = RedisModule_CreateDict(NULL);
 
     return REDISMODULE_OK;
diff --git a/src/modules/helloworld.c b/src/modules/helloworld.c
index 3b00dea77..043f5be32 100644
--- a/src/modules/helloworld.c
+++ b/src/modules/helloworld.c
@@ -91,7 +91,7 @@ int HelloPushCall_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, in
 }
 
 /* HELLO.PUSH.CALL2
- * This is exaxctly as HELLO.PUSH.CALL, but shows how we can reply to the
+ * This is exactly as HELLO.PUSH.CALL, but shows how we can reply to the
  * client using directly a reply object that Call() returned. */
 int HelloPushCall2_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
 {
@@ -345,7 +345,7 @@ int HelloToggleCase_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv,
 
 /* HELLO.MORE.EXPIRE key milliseconds.
  *
- * If they key has already an associated TTL, extends it by "milliseconds"
+ * If the key has already an associated TTL, extends it by "milliseconds"
  * milliseconds. Otherwise no operation is performed. */
 int HelloMoreExpire_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     RedisModule_AutoMemory(ctx); /* Use automatic memory management. */
diff --git a/src/multi.c b/src/multi.c
index a99c308be..3ce6d60ec 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -87,7 +87,7 @@ void discardTransaction(client *c) {
     unwatchAllKeys(c);
 }
 
-/* Flag the transacation as DIRTY_EXEC so that EXEC will fail.
+/* Flag the transaction as DIRTY_EXEC so that EXEC will fail.
  * Should be called every time there is an error while queueing a command. */
 void flagTransaction(client *c) {
     if (c->flags & CLIENT_MULTI)
diff --git a/src/networking.c b/src/networking.c
index 71e30cfa1..cf40b69e4 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -170,7 +170,7 @@ client *createClient(connection *conn) {
     return c;
 }
 
-/* This funciton puts the client in the queue of clients that should write
+/* This function puts the client in the queue of clients that should write
  * their output buffers to the socket. Note that it does not *yet* install
  * the write handler, to start clients are put in a queue of clients that need
  * to write, so we try to do that before returning in the event loop (see the
@@ -268,7 +268,7 @@ void _addReplyProtoToList(client *c, const char *s, size_t len) {
     listNode *ln = listLast(c->reply);
     clientReplyBlock *tail = ln? listNodeValue(ln): NULL;
 
-    /* Note that 'tail' may be NULL even if we have a tail node, becuase when
+    /* Note that 'tail' may be NULL even if we have a tail node, because when
      * addReplyDeferredLen() is used, it sets a dummy node to NULL just
      * fo fill it later, when the size of the bulk length is set. */
 
@@ -1161,7 +1161,7 @@ void freeClient(client *c) {
         listDelNode(server.clients_to_close,ln);
     }
 
-    /* If it is our master that's beging disconnected we should make sure
+    /* If it is our master that's being disconnected we should make sure
      * to cache the state to try a partial resynchronization later.
      *
      * Note that before doing this we make sure that the client is not in
@@ -1491,7 +1491,7 @@ void resetClient(client *c) {
     }
 }
 
-/* This funciton is used when we want to re-enter the event loop but there
+/* This function is used when we want to re-enter the event loop but there
  * is the risk that the client we are dealing with will be freed in some
  * way. This happens for instance in:
  *
@@ -2050,7 +2050,7 @@ char *getClientPeerId(client *c) {
     return c->peerid;
 }
 
-/* Concatenate a string representing the state of a client in an human
+/* Concatenate a string representing the state of a client in a human
  * readable format, into the sds string 's'. */
 sds catClientInfoString(sds s, client *client) {
     char flags[16], events[3], conninfo[CONN_INFO_LEN], *p;
@@ -3057,7 +3057,7 @@ void stopThreadedIO(void) {
  * we need to handle in parallel, however the I/O threading is disabled
  * globally for reads as well if we have too little pending clients.
  *
- * The function returns 0 if the I/O threading should be used becuase there
+ * The function returns 0 if the I/O threading should be used because there
  * are enough active threads, otherwise 1 is returned and the I/O threads
  * could be possibly stopped (if already active) as a side effect. */
 int stopThreadedIOIfNeeded(void) {
diff --git a/src/notify.c b/src/notify.c
index bb1055724..5c7634bce 100644
--- a/src/notify.c
+++ b/src/notify.c
@@ -62,7 +62,7 @@ int keyspaceEventsStringToFlags(char *classes) {
     return flags;
 }
 
-/* This function does exactly the revese of the function above: it gets
+/* This function does exactly the reverse of the function above: it gets
  * as input an integer with the xored flags and returns a string representing
  * the selected classes. The string returned is an sds string that needs to
  * be released with sdsfree(). */
diff --git a/src/object.c b/src/object.c
index 184e5ffc0..732241e84 100644
--- a/src/object.c
+++ b/src/object.c
@@ -126,7 +126,7 @@ robj *createStringObject(const char *ptr, size_t len) {
 /* Create a string object from a long long value. When possible returns a
  * shared integer object, or at least an integer encoded one.
  *
- * If valueobj is non zero, the function avoids returning a a shared
+ * If valueobj is non zero, the function avoids returning a shared
  * integer, because the object is going to be used as value in the Redis key
  * space (for instance when the INCR command is used), so we want LFU/LRU
  * values specific for each key. */
@@ -1224,7 +1224,7 @@ robj *objectCommandLookupOrReply(client *c, robj *key, robj *reply) {
     return o;
 }
 
-/* Object command allows to inspect the internals of an Redis Object.
+/* Object command allows to inspect the internals of a Redis Object.
  * Usage: OBJECT <refcount|encoding|idletime|freq> <key> */
 void objectCommand(client *c) {
     robj *o;
diff --git a/src/quicklist.h b/src/quicklist.h
index 8b553c119..fd9878af0 100644
--- a/src/quicklist.h
+++ b/src/quicklist.h
@@ -40,7 +40,7 @@
  * count: 16 bits, max 65536 (max zl bytes is 65k, so max count actually < 32k).
  * encoding: 2 bits, RAW=1, LZF=2.
  * container: 2 bits, NONE=1, ZIPLIST=2.
- * recompress: 1 bit, bool, true if node is temporarry decompressed for usage.
+ * recompress: 1 bit, bool, true if node is temporary decompressed for usage.
  * attempted_compress: 1 bit, boolean, used for verifying during testing.
  * extra: 10 bits, free for future use; pads out the remainder of 32 bits */
 typedef struct quicklistNode {
@@ -97,7 +97,7 @@ typedef struct quicklistBookmark {
 /* quicklist is a 40 byte struct (on 64-bit systems) describing a quicklist.
  * 'count' is the number of total entries.
  * 'len' is the number of quicklist nodes.
- * 'compress' is: -1 if compression disabled, otherwise it's the number
+ * 'compress' is: 0 if compression disabled, otherwise it's the number
  *                of quicklistNodes to leave uncompressed at ends of quicklist.
  * 'fill' is the user-requested (or default) fill factor.
  * 'bookmakrs are an optional feature that is used by realloc this struct,
diff --git a/src/rax.c b/src/rax.c
index 0ca915746..89345fe26 100644
--- a/src/rax.c
+++ b/src/rax.c
@@ -628,7 +628,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void **
      *
      * 3b. IF $SPLITPOS != 0:
      *     Trim the compressed node (reallocating it as well) in order to
-     *     contain $splitpos characters. Change chilid pointer in order to link
+     *     contain $splitpos characters. Change child pointer in order to link
      *     to the split node. If new compressed node len is just 1, set
      *     iscompr to 0 (layout is the same). Fix parent's reference.
      *
@@ -1082,7 +1082,7 @@ int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) {
         }
     } else if (h->size == 1) {
         /* If the node had just one child, after the removal of the key
-         * further compression with adjacent nodes is pontentially possible. */
+         * further compression with adjacent nodes is potentially possible. */
         trycompress = 1;
     }
 
@@ -1329,7 +1329,7 @@ int raxIteratorNextStep(raxIterator *it, int noup) {
         if (!noup && children) {
             debugf("GO DEEPER\n");
             /* Seek the lexicographically smaller key in this subtree, which
-             * is the first one found always going torwards the first child
+             * is the first one found always going towards the first child
              * of every successive node. */
             if (!raxStackPush(&it->stack,it->node)) return 0;
             raxNode **cp = raxNodeFirstChildPtr(it->node);
@@ -1348,7 +1348,7 @@ int raxIteratorNextStep(raxIterator *it, int noup) {
                 return 1;
             }
         } else {
-            /* If we finished exporing the previous sub-tree, switch to the
+            /* If we finished exploring the previous sub-tree, switch to the
              * new one: go upper until a node is found where there are
              * children representing keys lexicographically greater than the
              * current key. */
@@ -1510,7 +1510,7 @@ int raxIteratorPrevStep(raxIterator *it, int noup) {
 int raxSeek(raxIterator *it, const char *op, unsigned char *ele, size_t len) {
     int eq = 0, lt = 0, gt = 0, first = 0, last = 0;
 
-    it->stack.items = 0; /* Just resetting. Intialized by raxStart(). */
+    it->stack.items = 0; /* Just resetting. Initialized by raxStart(). */
     it->flags |= RAX_ITER_JUST_SEEKED;
     it->flags &= ~RAX_ITER_EOF;
     it->key_len = 0;
@@ -1731,7 +1731,7 @@ int raxPrev(raxIterator *it) {
  * tree, expect a disappointing distribution. A random walk produces good
  * random elements if the tree is not sparse, however in the case of a radix
  * tree certain keys will be reported much more often than others. At least
- * this function should be able to expore every possible element eventually. */
+ * this function should be able to explore every possible element eventually. */
 int raxRandomWalk(raxIterator *it, size_t steps) {
     if (it->rt->numele == 0) {
         it->flags |= RAX_ITER_EOF;
@@ -1825,7 +1825,7 @@ uint64_t raxSize(rax *rax) {
 /* ----------------------------- Introspection ------------------------------ */
 
 /* This function is mostly used for debugging and learning purposes.
- * It shows an ASCII representation of a tree on standard output, outling
+ * It shows an ASCII representation of a tree on standard output, outline
  * all the nodes and the contained keys.
  *
  * The representation is as follow:
@@ -1835,7 +1835,7 @@ uint64_t raxSize(rax *rax) {
  *  [abc]=0x12345678 (node is a key, pointing to value 0x12345678)
  *  [] (a normal empty node)
  *
- *  Children are represented in new idented lines, each children prefixed by
+ *  Children are represented in new indented lines, each children prefixed by
  *  the "`-(x)" string, where "x" is the edge byte.
  *
  *  [abc]
diff --git a/src/rax.h b/src/rax.h
index f2521d14a..6b1fd4188 100644
--- a/src/rax.h
+++ b/src/rax.h
@@ -58,7 +58,7 @@
  * successive nodes having a single child are "compressed" into the node
  * itself as a string of characters, each representing a next-level child,
  * and only the link to the node representing the last character node is
- * provided inside the representation. So the above representation is turend
+ * provided inside the representation. So the above representation is turned
  * into:
  *
  *                  ["foo"] ""
@@ -123,7 +123,7 @@ typedef struct raxNode {
      * nodes).
      *
      * If the node has an associated key (iskey=1) and is not NULL
-     * (isnull=0), then after the raxNode pointers poiting to the
+     * (isnull=0), then after the raxNode pointers pointing to the
      * children, an additional value pointer is present (as you can see
      * in the representation above as "value-ptr" field).
      */
diff --git a/src/rdb.c b/src/rdb.c
index 827ddc4f2..80fe3c5a6 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2170,7 +2170,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
         } else if (type == RDB_OPCODE_AUX) {
             /* AUX: generic string-string fields. Use to add state to RDB
              * which is backward compatible. Implementations of RDB loading
-             * are requierd to skip AUX fields they don't understand.
+             * are required to skip AUX fields they don't understand.
              *
              * An AUX field is composed of two strings: key and value. */
             robj *auxkey, *auxval;
@@ -2420,7 +2420,7 @@ void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal) {
         latencyEndMonitor(latency);
         latencyAddSampleIfNeeded("rdb-unlink-temp-file",latency);
         /* SIGUSR1 is whitelisted, so we have a way to kill a child without
-         * tirggering an error condition. */
+         * triggering an error condition. */
         if (bysignal != SIGUSR1)
             server.lastbgsave_status = C_ERR;
     }
diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c
index 17ec656ce..592feaf42 100644
--- a/src/redis-check-rdb.c
+++ b/src/redis-check-rdb.c
@@ -331,7 +331,7 @@ err:
     return 1;
 }
 
-/* RDB check main: called form redis.c when Redis is executed with the
+/* RDB check main: called form server.c when Redis is executed with the
  * redis-check-rdb alias, on during RDB loading errors.
  *
  * The function works in two ways: can be called with argc/argv as a
diff --git a/src/redis-cli.c b/src/redis-cli.c
index 944ba44ec..60bdca6de 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -313,7 +313,7 @@ static void cliRefreshPrompt(void) {
 
 /* Return the name of the dotfile for the specified 'dotfilename'.
  * Normally it just concatenates user $HOME to the file specified
- * in 'dotfilename'. However if the environment varialbe 'envoverride'
+ * in 'dotfilename'. However if the environment variable 'envoverride'
  * is set, its value is taken as the path.
  *
  * The function returns NULL (if the file is /dev/null or cannot be
@@ -1794,7 +1794,7 @@ static void usage(void) {
 "  -a <password>      Password to use when connecting to the server.\n"
 "                     You can also use the " REDIS_CLI_AUTH_ENV " environment\n"
 "                     variable to pass this password more safely\n"
-"                     (if both are used, this argument takes predecence).\n"
+"                     (if both are used, this argument takes precedence).\n"
 "  --user <username>  Used to send ACL style 'AUTH username pass'. Needs -a.\n"
 "  --pass <password>  Alias of -a for consistency with the new --user option.\n"
 "  --askpass          Force user to input password with mask from STDIN.\n"
@@ -2225,7 +2225,7 @@ static int evalMode(int argc, char **argv) {
         argv2[2] = sdscatprintf(sdsempty(),"%d",keys);
 
         /* Call it */
-        int eval_ldb = config.eval_ldb; /* Save it, may be reverteed. */
+        int eval_ldb = config.eval_ldb; /* Save it, may be reverted. */
         retval = issueCommand(argc+3-got_comma, argv2);
         if (eval_ldb) {
             if (!config.eval_ldb) {
@@ -6741,13 +6741,13 @@ struct distsamples {
  * samples greater than the previous one, and is also the stop sentinel.
  *
  * "tot' is the total number of samples in the different buckets, so it
- * is the SUM(samples[i].conut) for i to 0 up to the max sample.
+ * is the SUM(samples[i].count) for i to 0 up to the max sample.
  *
  * As a side effect the function sets all the buckets count to 0. */
 void showLatencyDistSamples(struct distsamples *samples, long long tot) {
     int j;
 
-     /* We convert samples into a index inside the palette
+     /* We convert samples into an index inside the palette
      * proportional to the percentage a given bucket represents.
      * This way intensity of the different parts of the spectrum
      * don't change relative to the number of requests, which avoids to
@@ -8054,7 +8054,7 @@ static void LRUTestMode(void) {
  * Intrisic latency mode.
  *
  * Measure max latency of a running process that does not result from
- * syscalls. Basically this software should provide an hint about how much
+ * syscalls. Basically this software should provide a hint about how much
  * time the kernel leaves the process without a chance to run.
  *--------------------------------------------------------------------------- */
 
diff --git a/src/redismodule.h b/src/redismodule.h
index 4bfc14cc7..4a0e5bf15 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -963,4 +963,4 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
 #define RedisModuleString robj
 
 #endif /* REDISMODULE_CORE */
-#endif /* REDISMOUDLE_H */
+#endif /* REDISMODULE_H */
diff --git a/src/replication.c b/src/replication.c
index 32c41361c..3f98b1062 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -83,16 +83,16 @@ char *replicationGetSlaveName(client *c) {
  * the file deletion to the filesystem. This call removes the file in a
  * background thread instead. We actually just do close() in the thread,
  * by using the fact that if there is another instance of the same file open,
- * the foreground unlink() will not really do anything, and deleting the
- * file will only happen once the last reference is lost. */
+ * the foreground unlink() will only remove the fs name, and deleting the
+ * file's storage space will only happen once the last reference is lost. */
 int bg_unlink(const char *filename) {
     int fd = open(filename,O_RDONLY|O_NONBLOCK);
     if (fd == -1) {
         /* Can't open the file? Fall back to unlinking in the main thread. */
         return unlink(filename);
     } else {
-        /* The following unlink() will not do anything since file
-         * is still open. */
+        /* The following unlink() removes the name but doesn't free the
+         * file contents because a process still has it open. */
         int retval = unlink(filename);
         if (retval == -1) {
             /* If we got an unlink error, we just return it, closing the
@@ -204,7 +204,7 @@ void feedReplicationBacklogWithObject(robj *o) {
  * as well. This function is used if the instance is a master: we use
  * the commands received by our clients in order to create the replication
  * stream. Instead if the instance is a slave and has sub-slaves attached,
- * we use replicationFeedSlavesFromMaster() */
+ * we use replicationFeedSlavesFromMasterStream() */
 void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) {
     listNode *ln;
     listIter li;
@@ -535,7 +535,7 @@ int masterTryPartialResynchronization(client *c) {
         (strcasecmp(master_replid, server.replid2) ||
          psync_offset > server.second_replid_offset))
     {
-        /* Run id "?" is used by slaves that want to force a full resync. */
+        /* Replid "?" is used by slaves that want to force a full resync. */
         if (master_replid[0] != '?') {
             if (strcasecmp(master_replid, server.replid) &&
                 strcasecmp(master_replid, server.replid2))
@@ -707,7 +707,7 @@ int startBgsaveForReplication(int mincapa) {
     return retval;
 }
 
-/* SYNC and PSYNC command implemenation. */
+/* SYNC and PSYNC command implementation. */
 void syncCommand(client *c) {
     /* ignore SYNC if already slave or in monitor mode */
     if (c->flags & CLIENT_SLAVE) return;
@@ -1364,7 +1364,7 @@ void replicationEmptyDbCallback(void *privdata) {
     replicationSendNewlineToMaster();
 }
 
-/* Once we have a link with the master and the synchroniziation was
+/* Once we have a link with the master and the synchronization was
  * performed, this function materializes the master client we store
  * at server.master, starting from the specified file descriptor. */
 void replicationCreateMasterClient(connection *conn, int dbid) {
@@ -1441,7 +1441,7 @@ redisDb *disklessLoadMakeBackups(void) {
  * the 'restore' argument (the number of DBs to replace) is non-zero.
  *
  * When instead the loading succeeded we want just to free our old backups,
- * in that case the funciton will do just that when 'restore' is 0. */
+ * in that case the function will do just that when 'restore' is 0. */
 void disklessLoadRestoreBackups(redisDb *backup, int restore, int empty_db_flags)
 {
     if (restore) {
@@ -1475,7 +1475,7 @@ void readSyncBulkPayload(connection *conn) {
     off_t left;
 
     /* Static vars used to hold the EOF mark, and the last bytes received
-     * form the server: when they match, we reached the end of the transfer. */
+     * from the server: when they match, we reached the end of the transfer. */
     static char eofmark[CONFIG_RUN_ID_SIZE];
     static char lastbytes[CONFIG_RUN_ID_SIZE];
     static int usemark = 0;
@@ -1792,7 +1792,7 @@ void readSyncBulkPayload(connection *conn) {
                           REDISMODULE_SUBEVENT_MASTER_LINK_UP,
                           NULL);
 
-    /* After a full resynchroniziation we use the replication ID and
+    /* After a full resynchronization we use the replication ID and
      * offset of the master. The secondary ID / offset are cleared since
      * we are starting a new history. */
     memcpy(server.replid,server.master->replid,sizeof(server.replid));
@@ -1891,7 +1891,7 @@ char *sendSynchronousCommand(int flags, connection *conn, ...) {
 /* Try a partial resynchronization with the master if we are about to reconnect.
  * If there is no cached master structure, at least try to issue a
  * "PSYNC ? -1" command in order to trigger a full resync using the PSYNC
- * command in order to obtain the master run id and the master replication
+ * command in order to obtain the master replid and the master replication
  * global offset.
  *
  * This function is designed to be called from syncWithMaster(), so the
@@ -1919,7 +1919,7 @@ char *sendSynchronousCommand(int flags, connection *conn, ...) {
  *
  * PSYNC_CONTINUE: If the PSYNC command succeeded and we can continue.
  * PSYNC_FULLRESYNC: If PSYNC is supported but a full resync is needed.
- *                   In this case the master run_id and global replication
+ *                   In this case the master replid and global replication
  *                   offset is saved.
  * PSYNC_NOT_SUPPORTED: If the server does not understand PSYNC at all and
  *                      the caller should fall back to SYNC.
@@ -1950,7 +1950,7 @@ int slaveTryPartialResynchronization(connection *conn, int read_reply) {
     /* Writing half */
     if (!read_reply) {
         /* Initially set master_initial_offset to -1 to mark the current
-         * master run_id and offset as not valid. Later if we'll be able to do
+         * master replid and offset as not valid. Later if we'll be able to do
          * a FULL resync using the PSYNC command we'll set the offset at the
          * right value, so that this information will be propagated to the
          * client structure representing the master into server.master. */
@@ -1991,7 +1991,7 @@ int slaveTryPartialResynchronization(connection *conn, int read_reply) {
     if (!strncmp(reply,"+FULLRESYNC",11)) {
         char *replid = NULL, *offset = NULL;
 
-        /* FULL RESYNC, parse the reply in order to extract the run id
+        /* FULL RESYNC, parse the reply in order to extract the replid
          * and the replication offset. */
         replid = strchr(reply,' ');
         if (replid) {
@@ -2283,7 +2283,7 @@ void syncWithMaster(connection *conn) {
 
     /* Try a partial resynchonization. If we don't have a cached master
      * slaveTryPartialResynchronization() will at least try to use PSYNC
-     * to start a full resynchronization so that we get the master run id
+     * to start a full resynchronization so that we get the master replid
      * and the global offset, to try a partial resync at the next
      * reconnection attempt. */
     if (server.repl_state == REPL_STATE_SEND_PSYNC) {
@@ -2446,7 +2446,7 @@ void replicationAbortSyncTransfer(void) {
  * If there was a replication handshake in progress 1 is returned and
  * the replication state (server.repl_state) set to REPL_STATE_CONNECT.
  *
- * Otherwise zero is returned and no operation is perforemd at all. */
+ * Otherwise zero is returned and no operation is performed at all. */
 int cancelReplicationHandshake(int reconnect) {
     if (server.repl_state == REPL_STATE_TRANSFER) {
         replicationAbortSyncTransfer();
@@ -2906,7 +2906,7 @@ void refreshGoodSlavesCount(void) {
  *
  * We don't care about taking a different cache for every different slave
  * since to fill the cache again is not very costly, the goal of this code
- * is to avoid that the same big script is trasmitted a big number of times
+ * is to avoid that the same big script is transmitted a big number of times
  * per second wasting bandwidth and processor speed, but it is not a problem
  * if we need to rebuild the cache from scratch from time to time, every used
  * script will need to be transmitted a single time to reappear in the cache.
@@ -2916,7 +2916,7 @@ void refreshGoodSlavesCount(void) {
  * 1) Every time a new slave connects, we flush the whole script cache.
  * 2) We only send as EVALSHA what was sent to the master as EVALSHA, without
  *    trying to convert EVAL into EVALSHA specifically for slaves.
- * 3) Every time we trasmit a script as EVAL to the slaves, we also add the
+ * 3) Every time we transmit a script as EVAL to the slaves, we also add the
  *    corresponding SHA1 of the script into the cache as we are sure every
  *    slave knows about the script starting from now.
  * 4) On SCRIPT FLUSH command, we replicate the command to all the slaves
@@ -3007,7 +3007,7 @@ int replicationScriptCacheExists(sds sha1) {
 
 /* This just set a flag so that we broadcast a REPLCONF GETACK command
  * to all the slaves in the beforeSleep() function. Note that this way
- * we "group" all the clients that want to wait for synchronouns replication
+ * we "group" all the clients that want to wait for synchronous replication
  * in a given event loop iteration, and send a single GETACK for them all. */
 void replicationRequestAckFromSlaves(void) {
     server.get_ack_from_slaves = 1;
diff --git a/src/scripting.c b/src/scripting.c
index be44ec67f..fdccb5db7 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -69,7 +69,7 @@ struct ldbState {
     list *children; /* All forked debugging sessions pids. */
     int bp[LDB_BREAKPOINTS_MAX]; /* An array of breakpoints line numbers. */
     int bpcount; /* Number of valid entries inside bp. */
-    int step;   /* Stop at next line ragardless of breakpoints. */
+    int step;   /* Stop at next line regardless of breakpoints. */
     int luabp;  /* Stop at next line because redis.breakpoint() was called. */
     sds *src;   /* Lua script source code split by line. */
     int lines;  /* Number of lines in 'src'. */
@@ -886,7 +886,7 @@ int luaRedisReplicateCommandsCommand(lua_State *lua) {
 
 /* redis.breakpoint()
  *
- * Allows to stop execution during a debuggign session from within
+ * Allows to stop execution during a debugging session from within
  * the Lua code implementation, like if a breakpoint was set in the code
  * immediately after the function. */
 int luaRedisBreakpointCommand(lua_State *lua) {
@@ -1500,7 +1500,7 @@ void evalGenericCommand(client *c, int evalsha) {
         /* Hash the code if this is an EVAL call */
         sha1hex(funcname+2,c->argv[1]->ptr,sdslen(c->argv[1]->ptr));
     } else {
-        /* We already have the SHA if it is a EVALSHA */
+        /* We already have the SHA if it is an EVALSHA */
         int j;
         char *sha = c->argv[1]->ptr;
 
@@ -1630,7 +1630,7 @@ void evalGenericCommand(client *c, int evalsha) {
      * To do so we use a cache of SHA1s of scripts that we already propagated
      * as full EVAL, that's called the Replication Script Cache.
      *
-     * For repliation, everytime a new slave attaches to the master, we need to
+     * For replication, everytime a new slave attaches to the master, we need to
      * flush our cache of scripts that can be replicated as EVALSHA, while
      * for AOF we need to do so every time we rewrite the AOF file. */
     if (evalsha && !server.lua_replicate_commands) {
@@ -1803,7 +1803,7 @@ void ldbLog(sds entry) {
 }
 
 /* A version of ldbLog() which prevents producing logs greater than
- * ldb.maxlen. The first time the limit is reached an hint is generated
+ * ldb.maxlen. The first time the limit is reached a hint is generated
  * to inform the user that reply trimming can be disabled using the
  * debugger "maxlen" command. */
 void ldbLogWithMaxLen(sds entry) {
@@ -1844,7 +1844,7 @@ void ldbSendLogs(void) {
 }
 
 /* Start a debugging session before calling EVAL implementation.
- * The techique we use is to capture the client socket file descriptor,
+ * The technique we use is to capture the client socket file descriptor,
  * in order to perform direct I/O with it from within Lua hooks. This
  * way we don't have to re-enter Redis in order to handle I/O.
  *
@@ -1927,7 +1927,7 @@ void ldbEndSession(client *c) {
     connNonBlock(ldb.conn);
     connSendTimeout(ldb.conn,0);
 
-    /* Close the client connectin after sending the final EVAL reply
+    /* Close the client connection after sending the final EVAL reply
      * in order to signal the end of the debugging session. */
     c->flags |= CLIENT_CLOSE_AFTER_REPLY;
 
@@ -2096,7 +2096,7 @@ void ldbLogSourceLine(int lnum) {
 /* Implement the "list" command of the Lua debugger. If around is 0
  * the whole file is listed, otherwise only a small portion of the file
  * around the specified line is shown. When a line number is specified
- * the amonut of context (lines before/after) is specified via the
+ * the amount of context (lines before/after) is specified via the
  * 'context' argument. */
 void ldbList(int around, int context) {
     int j;
@@ -2107,7 +2107,7 @@ void ldbList(int around, int context) {
     }
 }
 
-/* Append an human readable representation of the Lua value at position 'idx'
+/* Append a human readable representation of the Lua value at position 'idx'
  * on the stack of the 'lua' state, to the SDS string passed as argument.
  * The new SDS string with the represented value attached is returned.
  * Used in order to implement ldbLogStackValue().
@@ -2351,7 +2351,7 @@ char *ldbRedisProtocolToHuman_Double(sds *o, char *reply) {
     return p+2;
 }
 
-/* Log a Redis reply as debugger output, in an human readable format.
+/* Log a Redis reply as debugger output, in a human readable format.
  * If the resulting string is longer than 'len' plus a few more chars
  * used as prefix, it gets truncated. */
 void ldbLogRedisReply(char *reply) {
@@ -2535,7 +2535,7 @@ void ldbTrace(lua_State *lua) {
     }
 }
 
-/* Impleemnts the debugger "maxlen" command. It just queries or sets the
+/* Implements the debugger "maxlen" command. It just queries or sets the
  * ldb.maxlen variable. */
 void ldbMaxlen(sds *argv, int argc) {
     if (argc == 2) {
@@ -2608,8 +2608,8 @@ ldbLog(sdsnew("                     mode dataset changes will be retained."));
 ldbLog(sdsnew(""));
 ldbLog(sdsnew("Debugger functions you can call from Lua scripts:"));
 ldbLog(sdsnew("redis.debug()        Produce logs in the debugger console."));
-ldbLog(sdsnew("redis.breakpoint()   Stop execution like if there was a breakpoing."));
-ldbLog(sdsnew("                     in the next line of code."));
+ldbLog(sdsnew("redis.breakpoint()   Stop execution like if there was a breakpoint in the"));
+ldbLog(sdsnew("                     next line of code."));
             ldbSendLogs();
         } else if (!strcasecmp(argv[0],"s") || !strcasecmp(argv[0],"step") ||
                    !strcasecmp(argv[0],"n") || !strcasecmp(argv[0],"next")) {
diff --git a/src/sds.c b/src/sds.c
index b13bf3467..ce9cd7032 100644
--- a/src/sds.c
+++ b/src/sds.c
@@ -405,7 +405,7 @@ sds sdscatlen(sds s, const void *t, size_t len) {
     return s;
 }
 
-/* Append the specified null termianted C string to the sds string 's'.
+/* Append the specified null terminated C string to the sds string 's'.
  *
  * After the call, the passed sds string is no longer valid and all the
  * references must be substituted with the new pointer returned by the call. */
@@ -453,7 +453,7 @@ int sdsll2str(char *s, long long value) {
     size_t l;
 
     /* Generate the string representation, this method produces
-     * an reversed string. */
+     * a reversed string. */
     v = (value < 0) ? -value : value;
     p = s;
     do {
@@ -484,7 +484,7 @@ int sdsull2str(char *s, unsigned long long v) {
     size_t l;
 
     /* Generate the string representation, this method produces
-     * an reversed string. */
+     * a reversed string. */
     p = s;
     do {
         *p++ = '0'+(v%10);
diff --git a/src/sentinel.c b/src/sentinel.c
index 3cf496731..36644a63c 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -131,13 +131,13 @@ typedef struct sentinelAddr {
 /* The link to a sentinelRedisInstance. When we have the same set of Sentinels
  * monitoring many masters, we have different instances representing the
  * same Sentinels, one per master, and we need to share the hiredis connections
- * among them. Oherwise if 5 Sentinels are monitoring 100 masters we create
+ * among them. Otherwise if 5 Sentinels are monitoring 100 masters we create
  * 500 outgoing connections instead of 5.
  *
  * So this structure represents a reference counted link in terms of the two
  * hiredis connections for commands and Pub/Sub, and the fields needed for
  * failure detection, since the ping/pong time are now local to the link: if
- * the link is available, the instance is avaialbe. This way we don't just
+ * the link is available, the instance is available. This way we don't just
  * have 5 connections instead of 500, we also send 5 pings instead of 500.
  *
  * Links are shared only for Sentinels: master and slave instances have
@@ -986,7 +986,7 @@ instanceLink *createInstanceLink(void) {
     return link;
 }
 
-/* Disconnect an hiredis connection in the context of an instance link. */
+/* Disconnect a hiredis connection in the context of an instance link. */
 void instanceLinkCloseConnection(instanceLink *link, redisAsyncContext *c) {
     if (c == NULL) return;
 
@@ -1125,7 +1125,7 @@ int sentinelUpdateSentinelAddressInAllMasters(sentinelRedisInstance *ri) {
     return reconfigured;
 }
 
-/* This function is called when an hiredis connection reported an error.
+/* This function is called when a hiredis connection reported an error.
  * We set it to NULL and mark the link as disconnected so that it will be
  * reconnected again.
  *
@@ -2015,7 +2015,7 @@ void sentinelSendAuthIfNeeded(sentinelRedisInstance *ri, redisAsyncContext *c) {
  * The connection type is "cmd" or "pubsub" as specified by 'type'.
  *
  * This makes it possible to list all the sentinel instances connected
- * to a Redis servewr with CLIENT LIST, grepping for a specific name format. */
+ * to a Redis server with CLIENT LIST, grepping for a specific name format. */
 void sentinelSetClientName(sentinelRedisInstance *ri, redisAsyncContext *c, char *type) {
     char name[64];
 
@@ -2470,7 +2470,7 @@ void sentinelPublishReplyCallback(redisAsyncContext *c, void *reply, void *privd
         ri->last_pub_time = mstime();
 }
 
-/* Process an hello message received via Pub/Sub in master or slave instance,
+/* Process a hello message received via Pub/Sub in master or slave instance,
  * or sent directly to this sentinel via the (fake) PUBLISH command of Sentinel.
  *
  * If the master name specified in the message is not known, the message is
@@ -2607,7 +2607,7 @@ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privd
     sentinelProcessHelloMessage(r->element[2]->str, r->element[2]->len);
 }
 
-/* Send an "Hello" message via Pub/Sub to the specified 'ri' Redis
+/* Send a "Hello" message via Pub/Sub to the specified 'ri' Redis
  * instance in order to broadcast the current configuration for this
  * master, and to advertise the existence of this Sentinel at the same time.
  *
@@ -2661,7 +2661,7 @@ int sentinelSendHello(sentinelRedisInstance *ri) {
 }
 
 /* Reset last_pub_time in all the instances in the specified dictionary
- * in order to force the delivery of an Hello update ASAP. */
+ * in order to force the delivery of a Hello update ASAP. */
 void sentinelForceHelloUpdateDictOfRedisInstances(dict *instances) {
     dictIterator *di;
     dictEntry *de;
@@ -2675,13 +2675,13 @@ void sentinelForceHelloUpdateDictOfRedisInstances(dict *instances) {
     dictReleaseIterator(di);
 }
 
-/* This function forces the delivery of an "Hello" message (see
+/* This function forces the delivery of a "Hello" message (see
  * sentinelSendHello() top comment for further information) to all the Redis
  * and Sentinel instances related to the specified 'master'.
  *
  * It is technically not needed since we send an update to every instance
  * with a period of SENTINEL_PUBLISH_PERIOD milliseconds, however when a
- * Sentinel upgrades a configuration it is a good idea to deliever an update
+ * Sentinel upgrades a configuration it is a good idea to deliver an update
  * to the other Sentinels ASAP. */
 int sentinelForceHelloUpdateForMaster(sentinelRedisInstance *master) {
     if (!(master->flags & SRI_MASTER)) return C_ERR;
@@ -3105,7 +3105,7 @@ NULL
          * ip and port are the ip and port of the master we want to be
          * checked by Sentinel. Note that the command will not check by
          * name but just by master, in theory different Sentinels may monitor
-         * differnet masters with the same name.
+         * different masters with the same name.
          *
          * current-epoch is needed in order to understand if we are allowed
          * to vote for a failover leader or not. Each Sentinel can vote just
@@ -4017,7 +4017,7 @@ int sentinelSendSlaveOf(sentinelRedisInstance *ri, char *host, int port) {
      * the following tasks:
      * 1) Reconfigure the instance according to the specified host/port params.
      * 2) Rewrite the configuration.
-     * 3) Disconnect all clients (but this one sending the commnad) in order
+     * 3) Disconnect all clients (but this one sending the command) in order
      *    to trigger the ask-master-on-reconnection protocol for connected
      *    clients.
      *
@@ -4569,7 +4569,7 @@ void sentinelHandleDictOfRedisInstances(dict *instances) {
  * difference bigger than SENTINEL_TILT_TRIGGER milliseconds if one of the
  * following conditions happen:
  *
- * 1) The Sentiel process for some time is blocked, for every kind of
+ * 1) The Sentinel process for some time is blocked, for every kind of
  * random reason: the load is huge, the computer was frozen for some time
  * in I/O or alike, the process was stopped by a signal. Everything.
  * 2) The system clock was altered significantly.
diff --git a/src/server.c b/src/server.c
index b1d7614f5..ed62dd402 100644
--- a/src/server.c
+++ b/src/server.c
@@ -116,7 +116,7 @@ volatile unsigned long lru_clock; /* Server global current LRU time. */
  * write:       Write command (may modify the key space).
  *
  * read-only:   All the non special commands just reading from keys without
- *              changing the content, or returning other informations like
+ *              changing the content, or returning other information like
  *              the TIME command. Special commands such administrative commands
  *              or transaction related commands (multi, exec, discard, ...)
  *              are not flagged as read-only commands, since they affect the
@@ -1289,7 +1289,7 @@ dictType objectKeyHeapPointerValueDictType = {
     dictVanillaFree            /* val destructor */
 };
 
-/* Set dictionary type. Keys are SDS strings, values are ot used. */
+/* Set dictionary type. Keys are SDS strings, values are not used. */
 dictType setDictType = {
     dictSdsHash,               /* hash function */
     NULL,                      /* key dup */
@@ -1394,9 +1394,8 @@ dictType clusterNodesBlackListDictType = {
     NULL                        /* val destructor */
 };
 
-/* Cluster re-addition blacklist. This maps node IDs to the time
- * we can re-add this node. The goal is to avoid readding a removed
- * node for some time. */
+/* Modules system dictionary type. Keys are module name,
+ * values are pointer to RedisModule struct. */
 dictType modulesDictType = {
     dictSdsCaseHash,            /* hash function */
     NULL,                       /* key dup */
@@ -1449,7 +1448,7 @@ void tryResizeHashTables(int dbid) {
 /* Our hash table implementation performs rehashing incrementally while
  * we write/read from the hash table. Still if the server is idle, the hash
  * table will use two tables for a long time. So we try to use 1 millisecond
- * of CPU time at every call of this function to perform some rehahsing.
+ * of CPU time at every call of this function to perform some rehashing.
  *
  * The function returns 1 if some rehashing was performed, otherwise 0
  * is returned. */
@@ -1471,8 +1470,8 @@ int incrementallyRehash(int dbid) {
  * as we want to avoid resizing the hash tables when there is a child in order
  * to play well with copy-on-write (otherwise when a resize happens lots of
  * memory pages are copied). The goal of this function is to update the ability
- * for dict.c to resize the hash tables accordingly to the fact we have o not
- * running childs. */
+ * for dict.c to resize the hash tables accordingly to the fact we have an
+ * active fork child running. */
 void updateDictResizePolicy(void) {
     if (!hasActiveChildProcess())
         dictEnableResize();
@@ -1623,7 +1622,7 @@ int clientsCronTrackClientsMemUsage(client *c) {
     mem += sdsAllocSize(c->querybuf);
     mem += sizeof(client);
     /* Now that we have the memory used by the client, remove the old
-     * value from the old categoty, and add it back. */
+     * value from the old category, and add it back. */
     server.stat_clients_type_memory[c->client_cron_last_memory_type] -=
         c->client_cron_last_memory_usage;
     server.stat_clients_type_memory[type] += mem;
@@ -1844,18 +1843,18 @@ void cronUpdateMemoryStats() {
          * the frag ratio calculate may be off (ratio of two samples at different times) */
         server.cron_malloc_stats.process_rss = zmalloc_get_rss();
         server.cron_malloc_stats.zmalloc_used = zmalloc_used_memory();
-        /* Sampling the allcator info can be slow too.
-         * The fragmentation ratio it'll show is potentically more accurate
+        /* Sampling the allocator info can be slow too.
+         * The fragmentation ratio it'll show is potentially more accurate
          * it excludes other RSS pages such as: shared libraries, LUA and other non-zmalloc
          * allocations, and allocator reserved pages that can be pursed (all not actual frag) */
         zmalloc_get_allocator_info(&server.cron_malloc_stats.allocator_allocated,
                                    &server.cron_malloc_stats.allocator_active,
                                    &server.cron_malloc_stats.allocator_resident);
         /* in case the allocator isn't providing these stats, fake them so that
-         * fragmention info still shows some (inaccurate metrics) */
+         * fragmentation info still shows some (inaccurate metrics) */
         if (!server.cron_malloc_stats.allocator_resident) {
             /* LUA memory isn't part of zmalloc_used, but it is part of the process RSS,
-             * so we must desuct it in order to be able to calculate correct
+             * so we must deduct it in order to be able to calculate correct
              * "allocator fragmentation" ratio */
             size_t lua_memory = lua_gc(server.lua,LUA_GCCOUNT,0)*1024LL;
             server.cron_malloc_stats.allocator_resident = server.cron_malloc_stats.process_rss - lua_memory;
@@ -2038,7 +2037,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
     /* AOF write errors: in this case we have a buffer to flush as well and
      * clear the AOF error in case of success to make the DB writable again,
      * however to try every second is enough in case of 'hz' is set to
-     * an higher frequency. */
+     * a higher frequency. */
     run_with_period(1000) {
         if (server.aof_last_write_status == C_ERR)
             flushAppendOnlyFile(0);
@@ -2266,7 +2265,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     if (moduleCount()) moduleReleaseGIL();
 }
 
-/* This function is called immadiately after the event loop multiplexing
+/* This function is called immediately after the event loop multiplexing
  * API returned, and the control is going to soon return to Redis by invoking
  * the different events callbacks. */
 void afterSleep(struct aeEventLoop *eventLoop) {
@@ -2488,7 +2487,7 @@ void initServerConfig(void) {
     R_NegInf = -1.0/R_Zero;
     R_Nan = R_Zero/R_Zero;
 
-    /* Command table -- we initiialize it here as it is part of the
+    /* Command table -- we initialize it here as it is part of the
      * initial configuration, since command names may be changed via
      * redis.conf using the rename-command directive. */
     server.commands = dictCreate(&commandTableDictType,NULL);
@@ -3152,7 +3151,7 @@ int populateCommandTableParseFlags(struct redisCommand *c, char *strflags) {
 }
 
 /* Populates the Redis Command Table starting from the hard coded list
- * we have on top of redis.c file. */
+ * we have on top of server.c file. */
 void populateCommandTable(void) {
     int j;
     int numcommands = sizeof(redisCommandTable)/sizeof(struct redisCommand);
@@ -3286,12 +3285,12 @@ void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc,
  *
  * 'cmd' must be a pointer to the Redis command to replicate, dbid is the
  * database ID the command should be propagated into.
- * Arguments of the command to propagte are passed as an array of redis
+ * Arguments of the command to propagate are passed as an array of redis
  * objects pointers of len 'argc', using the 'argv' vector.
  *
  * The function does not take a reference to the passed 'argv' vector,
  * so it is up to the caller to release the passed argv (but it is usually
- * stack allocated).  The function autoamtically increments ref count of
+ * stack allocated).  The function automatically increments ref count of
  * passed objects, so the caller does not need to. */
 void alsoPropagate(struct redisCommand *cmd, int dbid, robj **argv, int argc,
                    int target)
@@ -3451,7 +3450,7 @@ void call(client *c, int flags) {
         if (c->flags & CLIENT_FORCE_AOF) propagate_flags |= PROPAGATE_AOF;
 
         /* However prevent AOF / replication propagation if the command
-         * implementations called preventCommandPropagation() or similar,
+         * implementation called preventCommandPropagation() or similar,
          * or if we don't have the call() flags to do so. */
         if (c->flags & CLIENT_PREVENT_REPL_PROP ||
             !(flags & CMD_CALL_PROPAGATE_REPL))
@@ -3706,7 +3705,7 @@ int processCommand(client *c) {
         }
 
         /* Save out_of_memory result at script start, otherwise if we check OOM
-         * untill first write within script, memory used by lua stack and
+         * until first write within script, memory used by lua stack and
          * arguments might interfere. */
         if (c->cmd->proc == evalCommand || c->cmd->proc == evalShaCommand) {
             server.lua_oom = out_of_memory;
@@ -3944,7 +3943,7 @@ int prepareForShutdown(int flags) {
 
 /*================================== Commands =============================== */
 
-/* Sometimes Redis cannot accept write commands because there is a perstence
+/* Sometimes Redis cannot accept write commands because there is a persistence
  * error with the RDB or AOF file, and Redis is configured in order to stop
  * accepting writes in such situation. This function returns if such a
  * condition is active, and the type of the condition.
diff --git a/src/server.h b/src/server.h
index dae891a96..8d26e6009 100644
--- a/src/server.h
+++ b/src/server.h
@@ -161,7 +161,7 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
 /* Hash table parameters */
 #define HASHTABLE_MIN_FILL        10      /* Minimal hash table fill 10% */
 
-/* Command flags. Please check the command table defined in the redis.c file
+/* Command flags. Please check the command table defined in the server.c file
  * for more information about the meaning of every flag. */
 #define CMD_WRITE (1ULL<<0)            /* "write" flag */
 #define CMD_READONLY (1ULL<<1)         /* "read-only" flag */
@@ -827,7 +827,7 @@ typedef struct client {
                                        copying this slave output buffer
                                        should use. */
     char replid[CONFIG_RUN_ID_SIZE+1]; /* Master replication ID (if master). */
-    int slave_listening_port; /* As configured with: SLAVECONF listening-port */
+    int slave_listening_port; /* As configured with: REPLCONF listening-port */
     char slave_ip[NET_IP_STR_LEN]; /* Optionally given by REPLCONF ip-address */
     int slave_capa;         /* Slave capabilities: SLAVE_CAPA_* bitwise OR. */
     multiState mstate;      /* MULTI/EXEC state */
@@ -939,7 +939,7 @@ typedef struct redisOp {
 } redisOp;
 
 /* Defines an array of Redis operations. There is an API to add to this
- * structure in a easy way.
+ * structure in an easy way.
  *
  * redisOpArrayInit();
  * redisOpArrayAppend();
@@ -1357,7 +1357,7 @@ struct redisServer {
     unsigned int maxclients;            /* Max number of simultaneous clients */
     unsigned long long maxmemory;   /* Max number of memory bytes to use */
     int maxmemory_policy;           /* Policy for key eviction */
-    int maxmemory_samples;          /* Pricision of random sampling */
+    int maxmemory_samples;          /* Precision of random sampling */
     int lfu_log_factor;             /* LFU logarithmic counter factor. */
     int lfu_decay_time;             /* LFU counter decay factor. */
     long long proto_max_bulk_len;   /* Protocol bulk length maximum size. */
@@ -1438,7 +1438,7 @@ struct redisServer {
     int lua_random_dirty; /* True if a random command was called during the
                              execution of the current script. */
     int lua_replicate_commands; /* True if we are doing single commands repl. */
-    int lua_multi_emitted;/* True if we already proagated MULTI. */
+    int lua_multi_emitted;/* True if we already propagated MULTI. */
     int lua_repl;         /* Script replication flags for redis.set_repl(). */
     int lua_timedout;     /* True if we reached the time limit for script
                              execution. */
@@ -1946,7 +1946,7 @@ void addACLLogEntry(client *c, int reason, int keypos, sds username);
 /* Flags only used by the ZADD command but not by zsetAdd() API: */
 #define ZADD_CH (1<<16)      /* Return num of elements added or updated. */
 
-/* Struct to hold a inclusive/exclusive range spec by score comparison. */
+/* Struct to hold an inclusive/exclusive range spec by score comparison. */
 typedef struct {
     double min, max;
     int minex, maxex; /* are min or max exclusive? */
diff --git a/src/siphash.c b/src/siphash.c
index 357741132..30c15c04e 100644
--- a/src/siphash.c
+++ b/src/siphash.c
@@ -22,7 +22,7 @@
    1. We use SipHash 1-2. This is not believed to be as strong as the
       suggested 2-4 variant, but AFAIK there are not trivial attacks
       against this reduced-rounds version, and it runs at the same speed
-      as Murmurhash2 that we used previously, why the 2-4 variant slowed
+      as Murmurhash2 that we used previously, while the 2-4 variant slowed
       down Redis by a 4% figure more or less.
    2. Hard-code rounds in the hope the compiler can optimize it more
       in this raw from. Anyway we always want the standard 2-4 variant.
@@ -36,7 +36,7 @@
       perform a text transformation in some temporary buffer, which is costly.
    5. Remove debugging code.
    6. Modified the original test.c file to be a stand-alone function testing
-      the function in the new form (returing an uint64_t) using just the
+      the function in the new form (returning an uint64_t) using just the
       relevant test vector.
  */
 #include <assert.h>
@@ -46,7 +46,7 @@
 #include <ctype.h>
 
 /* Fast tolower() alike function that does not care about locale
- * but just returns a-z insetad of A-Z. */
+ * but just returns a-z instead of A-Z. */
 int siptlw(int c) {
     if (c >= 'A' && c <= 'Z') {
         return c+('a'-'A');
diff --git a/src/slowlog.c b/src/slowlog.c
index 1d715e39b..408456b14 100644
--- a/src/slowlog.c
+++ b/src/slowlog.c
@@ -75,7 +75,7 @@ slowlogEntry *slowlogCreateEntry(client *c, robj **argv, int argc, long long dur
             } else if (argv[j]->refcount == OBJ_SHARED_REFCOUNT) {
                 se->argv[j] = argv[j];
             } else {
-                /* Here we need to dupliacate the string objects composing the
+                /* Here we need to duplicate the string objects composing the
                  * argument vector of the command, because those may otherwise
                  * end shared with string objects stored into keys. Having
                  * shared objects between any part of Redis, and the data
diff --git a/src/sort.c b/src/sort.c
index f269a7731..aeef53e6a 100644
--- a/src/sort.c
+++ b/src/sort.c
@@ -115,7 +115,7 @@ robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst, int writeflag)
     if (fieldobj) {
         if (o->type != OBJ_HASH) goto noobj;
 
-        /* Retrieve value from hash by the field name. The returend object
+        /* Retrieve value from hash by the field name. The returned object
          * is a new object with refcount already incremented. */
         o = hashTypeGetValueObject(o, fieldobj->ptr);
     } else {
diff --git a/src/sparkline.c b/src/sparkline.c
index 0a986883d..67482c774 100644
--- a/src/sparkline.c
+++ b/src/sparkline.c
@@ -92,7 +92,7 @@ void freeSparklineSequence(struct sequence *seq) {
  * ------------------------------------------------------------------------- */
 
 /* Render part of a sequence, so that render_sequence() call call this function
- * with differnent parts in order to create the full output without overflowing
+ * with different parts in order to create the full output without overflowing
  * the current terminal columns. */
 sds sparklineRenderRange(sds output, struct sequence *seq, int rows, int offset, int len, int flags) {
     int j;
diff --git a/src/stream.h b/src/stream.h
index 0d3bf63fc..e4c5ff78d 100644
--- a/src/stream.h
+++ b/src/stream.h
@@ -74,7 +74,7 @@ typedef struct streamConsumer {
                                    consumer not yet acknowledged. Keys are
                                    big endian message IDs, while values are
                                    the same streamNACK structure referenced
-                                   in the "pel" of the conumser group structure
+                                   in the "pel" of the consumer group structure
                                    itself, so the value is shared. */
 } streamConsumer;
 
diff --git a/src/t_hash.c b/src/t_hash.c
index 8421223cd..89193a41e 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -627,7 +627,7 @@ void hincrbyfloatCommand(client *c) {
     server.dirty++;
 
     /* Always replicate HINCRBYFLOAT as an HSET command with the final value
-     * in order to make sure that differences in float pricision or formatting
+     * in order to make sure that differences in float precision or formatting
      * will not create differences in replicas or after an AOF restart. */
     robj *aux, *newobj;
     aux = createStringObject("HSET",4);
diff --git a/src/t_list.c b/src/t_list.c
index ecf09cd94..4edcc8be0 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -722,7 +722,7 @@ void rpoplpushCommand(client *c) {
  * Blocking POP operations
  *----------------------------------------------------------------------------*/
 
-/* This is a helper function for handleClientsBlockedOnKeys(). It's work
+/* This is a helper function for handleClientsBlockedOnKeys(). Its work
  * is to serve a specific client (receiver) that is blocked on 'key'
  * in the context of the specified 'db', doing the following:
  *
@@ -807,7 +807,7 @@ void blockingPopGenericCommand(client *c, int where) {
                 return;
             } else {
                 if (listTypeLength(o) != 0) {
-                    /* Non empty list, this is like a non normal [LR]POP. */
+                    /* Non empty list, this is like a normal [LR]POP. */
                     char *event = (where == LIST_HEAD) ? "lpop" : "rpop";
                     robj *value = listTypePop(o,where);
                     serverAssert(value != NULL);
@@ -843,7 +843,7 @@ void blockingPopGenericCommand(client *c, int where) {
         return;
     }
 
-    /* If the list is empty or the key does not exists we must block */
+    /* If the keys do not exist we must block */
     blockForKeys(c,BLOCKED_LIST,c->argv + 1,c->argc - 2,timeout,NULL,NULL);
 }
 
diff --git a/src/t_set.c b/src/t_set.c
index 9166e2b9a..b4f308963 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -193,7 +193,7 @@ sds setTypeNextObject(setTypeIterator *si) {
 }
 
 /* Return random element from a non empty set.
- * The returned element can be a int64_t value if the set is encoded
+ * The returned element can be an int64_t value if the set is encoded
  * as an "intset" blob of integers, or an SDS string if the set
  * is a regular set.
  *
@@ -458,7 +458,7 @@ void spopWithCountCommand(client *c) {
         dbDelete(c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id);
 
-        /* Propagate this command as an DEL operation */
+        /* Propagate this command as a DEL operation */
         rewriteClientCommandVector(c,2,shared.del,c->argv[1]);
         signalModifiedKey(c,c->db,c->argv[1]);
         server.dirty++;
@@ -692,7 +692,7 @@ void srandmemberWithCountCommand(client *c) {
      * In this case we create a set from scratch with all the elements, and
      * subtract random elements to reach the requested number of elements.
      *
-     * This is done because if the number of requsted elements is just
+     * This is done because if the number of requested elements is just
      * a bit less than the number of elements in the set, the natural approach
      * used into CASE 3 is highly inefficient. */
     if (count*SRANDMEMBER_SUB_STRATEGY_MUL > size) {
diff --git a/src/t_stream.c b/src/t_stream.c
index 1ab85a3b1..d565dd1f6 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1196,7 +1196,7 @@ void xaddCommand(client *c) {
     int id_given = 0; /* Was an ID different than "*" specified? */
     long long maxlen = -1;  /* If left to -1 no trimming is performed. */
     int approx_maxlen = 0;  /* If 1 only delete whole radix tree nodes, so
-                               the maxium length is not applied verbatim. */
+                               the maximum length is not applied verbatim. */
     int maxlen_arg_idx = 0; /* Index of the count in MAXLEN, for rewriting. */
 
     /* Parse options. */
@@ -1890,7 +1890,7 @@ NULL
     }
 }
 
-/* XSETID <stream> <groupname> <id>
+/* XSETID <stream> <id>
  *
  * Set the internal "last ID" of a stream. */
 void xsetidCommand(client *c) {
@@ -1982,7 +1982,7 @@ cleanup:
  *
  * If start and stop are omitted, the command just outputs information about
  * the amount of pending messages for the key/group pair, together with
- * the minimum and maxium ID of pending messages.
+ * the minimum and maximum ID of pending messages.
  *
  * If start and stop are provided instead, the pending messages are returned
  * with informations about the current owner, number of deliveries and last
diff --git a/src/t_string.c b/src/t_string.c
index 93910e5a9..6c54a6fe9 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -315,7 +315,7 @@ void msetGenericCommand(client *c, int nx) {
     }
 
     /* Handle the NX flag. The MSETNX semantic is to return zero and don't
-     * set anything if at least one key alerady exists. */
+     * set anything if at least one key already exists. */
     if (nx) {
         for (j = 1; j < c->argc; j += 2) {
             if (lookupKeyWrite(c->db,c->argv[j]) != NULL) {
diff --git a/src/t_zset.c b/src/t_zset.c
index 3b716bb99..4b28c24ef 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -245,7 +245,7 @@ int zslDelete(zskiplist *zsl, double score, sds ele, zskiplistNode **node) {
     return 0; /* not found */
 }
 
-/* Update the score of an elmenent inside the sorted set skiplist.
+/* Update the score of an element inside the sorted set skiplist.
  * Note that the element must exist and must match 'score'.
  * This function does not update the score in the hash table side, the
  * caller should take care of it.
diff --git a/src/tracking.c b/src/tracking.c
index 2721de32a..3737f6859 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -134,7 +134,7 @@ void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **pr
                   CLIENT_TRACKING_NOLOOP);
     c->client_tracking_redirection = redirect_to;
 
-    /* This may be the first client we ever enable. Crete the tracking
+    /* This may be the first client we ever enable. Create the tracking
      * table if it does not exist. */
     if (TrackingTable == NULL) {
         TrackingTable = raxNew();
diff --git a/src/valgrind.sup b/src/valgrind.sup
index 3024d63bc..b05843d8c 100644
--- a/src/valgrind.sup
+++ b/src/valgrind.sup
@@ -1,17 +1,17 @@
 {
-   <lzf_unitialized_hash_table>
+   <lzf_uninitialized_hash_table>
    Memcheck:Cond
    fun:lzf_compress
 }
 
 {
-   <lzf_unitialized_hash_table>
+   <lzf_uninitialized_hash_table>
    Memcheck:Value4
    fun:lzf_compress
 }
 
 {
-   <lzf_unitialized_hash_table>
+   <lzf_uninitialized_hash_table>
    Memcheck:Value8
    fun:lzf_compress
 }
diff --git a/src/ziplist.c b/src/ziplist.c
index d4bdb4e78..6bfe34461 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -99,7 +99,7 @@
  *      Integer encoded as 24 bit signed (3 bytes).
  * |11111110| - 2 bytes
  *      Integer encoded as 8 bit signed (1 byte).
- * |1111xxxx| - (with xxxx between 0000 and 1101) immediate 4 bit integer.
+ * |1111xxxx| - (with xxxx between 0001 and 1101) immediate 4 bit integer.
  *      Unsigned integer from 0 to 12. The encoded value is actually from
  *      1 to 13 because 0000 and 1111 can not be used, so 1 should be
  *      subtracted from the encoded 4 bit value to obtain the right value.
@@ -191,10 +191,10 @@
 #include "redisassert.h"
 
 #define ZIP_END 255         /* Special "end of ziplist" entry. */
-#define ZIP_BIG_PREVLEN 254 /* Max number of bytes of the previous entry, for
-                               the "prevlen" field prefixing each entry, to be
-                               represented with just a single byte. Otherwise
-                               it is represented as FE AA BB CC DD, where
+#define ZIP_BIG_PREVLEN 254 /* ZIP_BIG_PREVLEN - 1 is the max number of bytes of
+                               the previous entry, for the "prevlen" field prefixing
+                               each entry, to be represented with just a single byte.
+                               Otherwise it is represented as FE AA BB CC DD, where
                                AA BB CC DD are a 4 bytes unsigned integer
                                representing the previous entry len. */
 
@@ -317,7 +317,7 @@ unsigned int zipIntSize(unsigned char encoding) {
     return 0;
 }
 
-/* Write the encoidng header of the entry in 'p'. If p is NULL it just returns
+/* Write the encoding header of the entry in 'p'. If p is NULL it just returns
  * the amount of bytes required to encode such a length. Arguments:
  *
  * 'encoding' is the encoding we are using for the entry. It could be
@@ -325,7 +325,7 @@ unsigned int zipIntSize(unsigned char encoding) {
  * for single-byte small immediate integers.
  *
  * 'rawlen' is only used for ZIP_STR_* encodings and is the length of the
- * srting that this entry represents.
+ * string that this entry represents.
  *
  * The function returns the number of bytes used by the encoding/length
  * header stored in 'p'. */
@@ -951,7 +951,7 @@ unsigned char *ziplistMerge(unsigned char **first, unsigned char **second) {
     } else {
         /* !append == prepending to target */
         /* Move target *contents* exactly size of (source - [END]),
-         * then copy source into vacataed space (source - [END]):
+         * then copy source into vacated space (source - [END]):
          *   [SOURCE - END, TARGET - HEADER] */
         memmove(target + source_bytes - ZIPLIST_END_SIZE,
                 target + ZIPLIST_HEADER_SIZE,
diff --git a/src/zipmap.c b/src/zipmap.c
index 22bfa1a46..365c4aea4 100644
--- a/src/zipmap.c
+++ b/src/zipmap.c
@@ -133,7 +133,7 @@ static unsigned int zipmapEncodeLength(unsigned char *p, unsigned int len) {
  * zipmap. Returns NULL if the key is not found.
  *
  * If NULL is returned, and totlen is not NULL, it is set to the entire
- * size of the zimap, so that the calling function will be able to
+ * size of the zipmap, so that the calling function will be able to
  * reallocate the original zipmap to make room for more entries. */
 static unsigned char *zipmapLookupRaw(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned int *totlen) {
     unsigned char *p = zm+1, *k = NULL;
diff --git a/tests/cluster/tests/04-resharding.tcl b/tests/cluster/tests/04-resharding.tcl
index 33f861dc5..cee2ec5ba 100644
--- a/tests/cluster/tests/04-resharding.tcl
+++ b/tests/cluster/tests/04-resharding.tcl
@@ -1,7 +1,7 @@
 # Failover stress test.
 # In this test a different node is killed in a loop for N
 # iterations. The test checks that certain properties
-# are preseved across iterations.
+# are preserved across iterations.
 
 source "../tests/includes/init-tests.tcl"
 source "../../../tests/support/cli.tcl"
@@ -32,7 +32,7 @@ test "Enable AOF in all the instances" {
     }
 }
 
-# Return nno-zero if the specified PID is about a process still in execution,
+# Return non-zero if the specified PID is about a process still in execution,
 # otherwise 0 is returned.
 proc process_is_running {pid} {
     # PS should return with an error if PID is non existing,
@@ -45,7 +45,7 @@ proc process_is_running {pid} {
 #
 # - N commands are sent to the cluster in the course of the test.
 # - Every command selects a random key from key:0 to key:MAX-1.
-# - The operation RPUSH key <randomvalue> is perforemd.
+# - The operation RPUSH key <randomvalue> is performed.
 # - Tcl remembers into an array all the values pushed to each list.
 # - After N/2 commands, the resharding process is started in background.
 # - The test continues while the resharding is in progress.
diff --git a/tests/instances.tcl b/tests/instances.tcl
index 2199cfcd4..275db45c3 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -322,7 +322,7 @@ proc pause_on_error {} {
             puts "S <id> cmd ... arg     Call command in Sentinel <id>."
             puts "R <id> cmd ... arg     Call command in Redis <id>."
             puts "SI <id> <field>        Show Sentinel <id> INFO <field>."
-            puts "RI <id> <field>        Show Sentinel <id> INFO <field>."
+            puts "RI <id> <field>        Show Redis <id> INFO <field>."
             puts "continue               Resume test."
         } else {
             set errcode [catch {eval $line} retval]
diff --git a/tests/integration/replication-4.tcl b/tests/integration/replication-4.tcl
index 54891151b..8071c4f97 100644
--- a/tests/integration/replication-4.tcl
+++ b/tests/integration/replication-4.tcl
@@ -16,7 +16,7 @@ start_server {tags {"repl"}} {
             s 0 role
         } {slave}
 
-        test {Test replication with parallel clients writing in differnet DBs} {
+        test {Test replication with parallel clients writing in different DBs} {
             after 5000
             stop_bg_complex_data $load_handle0
             stop_bg_complex_data $load_handle1
diff --git a/tests/support/test.tcl b/tests/support/test.tcl
index 55937b8f4..23015b3a7 100644
--- a/tests/support/test.tcl
+++ b/tests/support/test.tcl
@@ -108,7 +108,7 @@ proc test {name code {okpattern undefined} {options undefined}} {
         return
     }
 
-    # abort if test name in skiptests
+    # abort if only_tests was set but test name is not included
     if {[llength $::only_tests] > 0 && [lsearch $::only_tests $name] < 0} {
         incr ::num_skipped
         send_data_packet $::test_server_fd skip $name
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index fe2d484b8..7e1c5c88f 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -471,7 +471,7 @@ proc signal_idle_client fd {
 # The the_end function gets called when all the test units were already
 # executed, so the test finished.
 proc the_end {} {
-    # TODO: print the status, exit with the rigth exit code.
+    # TODO: print the status, exit with the right exit code.
     puts "\n                   The End\n"
     puts "Execution time of different units:"
     foreach {time name} $::clients_time_history {
@@ -526,9 +526,10 @@ proc print_help_screen {} {
         "--stack-logging    Enable OSX leaks/malloc stack logging."
         "--accurate         Run slow randomized tests for more iterations."
         "--quiet            Don't show individual tests."
-        "--single <unit>    Just execute the specified unit (see next option). this option can be repeated."
+        "--single <unit>    Just execute the specified unit (see next option). This option can be repeated."
+        "--verbose          Increases verbosity."
         "--list-tests       List all the available test units."
-        "--only <test>      Just execute the specified test by test name. this option can be repeated."
+        "--only <test>      Just execute the specified test by test name. This option can be repeated."
         "--skip-till <unit> Skip all units until (and including) the specified one."
         "--skipunit <unit>  Skip one unit."
         "--clients <num>    Number of test clients (default 16)."
diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl
index 52d174d75..444525f36 100644
--- a/tests/unit/expire.tcl
+++ b/tests/unit/expire.tcl
@@ -72,7 +72,7 @@ start_server {tags {"expire"}} {
         list [r persist foo] [r persist nokeyatall]
     } {0 0}
 
-    test {EXPIRE pricision is now the millisecond} {
+    test {EXPIRE precision is now the millisecond} {
         # This test is very likely to do a false positive if the
         # server is under pressure, so if it does not work give it a few more
         # chances.
diff --git a/tests/unit/hyperloglog.tcl b/tests/unit/hyperloglog.tcl
index 712fcc641..db26a2e75 100644
--- a/tests/unit/hyperloglog.tcl
+++ b/tests/unit/hyperloglog.tcl
@@ -79,7 +79,7 @@ start_server {tags {"hll"}} {
         }
     }
 
-    test {Corrupted sparse HyperLogLogs are detected: Additionl at tail} {
+    test {Corrupted sparse HyperLogLogs are detected: Additional at tail} {
         r del hll
         r pfadd hll a b c
         r append hll "hello"
diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl
index 8b364b287..3283edc66 100644
--- a/tests/unit/scripting.tcl
+++ b/tests/unit/scripting.tcl
@@ -533,7 +533,7 @@ start_server {tags {"scripting"}} {
     # Note: keep this test at the end of this server stanza because it
     # kills the server.
     test {SHUTDOWN NOSAVE can kill a timedout script anyway} {
-        # The server could be still unresponding to normal commands.
+        # The server should be still unresponding to normal commands.
         catch {r ping} e
         assert_match {BUSY*} $e
         catch {r shutdown nosave}
diff --git a/utils/create-cluster/README b/utils/create-cluster/README
index 37a3080db..bcd745977 100644
--- a/utils/create-cluster/README
+++ b/utils/create-cluster/README
@@ -1,4 +1,4 @@
-Create-custer is a small script used to easily start a big number of Redis
+create-cluster is a small script used to easily start a big number of Redis
 instances configured to run in cluster mode. Its main goal is to allow manual
 testing in a condition which is not easy to replicate with the Redis cluster
 unit tests, for example when a lot of instances are needed in order to trigger
diff --git a/utils/hashtable/README b/utils/hashtable/README
index 87a76c9a5..87ffc2f08 100644
--- a/utils/hashtable/README
+++ b/utils/hashtable/README
@@ -5,7 +5,7 @@ rehashing.c
 
 Visually show buckets in the two hash tables between rehashings. Also stress
 test getRandomKeys() implementation, that may actually disappear from
-Redis soon, however visualization some code is reusable in new bugs
+Redis soon, However the visualization code is reusable in new bugs
 investigation.
 
 Compile with:

From 86511bbbefaeca7be4c7c39d65fd15e4549939ec Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuancode@163.com>
Date: Thu, 10 Sep 2020 22:02:27 +0800
Subject: [PATCH 0653/1098] Remove dead global variable 'lru_clock' (#7782)

---
 src/server.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index ed62dd402..a5b2538fa 100644
--- a/src/server.c
+++ b/src/server.c
@@ -71,7 +71,6 @@ double R_Zero, R_PosInf, R_NegInf, R_Nan;
 
 /* Global vars */
 struct redisServer server; /* Server global state */
-volatile unsigned long lru_clock; /* Server global current LRU time. */
 
 /* Our command table.
  *

From ed9bfe226289759756126f35583394e5db93048f Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 13 Sep 2020 13:50:23 +0300
Subject: [PATCH 0654/1098] fix broken PEXPIREAT test (#7791)

This test was nearly always failing on MacOS github actions.
This is because of bugs in the test that caused it to nearly always run
all 3 attempts and just look at the last one as the pass/fail creteria.

i.e. the test was nearly always running all 3 attempts and still sometimes
succeed. this is because the break condition was different than the test
completion condition.

The reason the test succeeded is because the break condition tested the
results of all 3 tests (PSETEX/PEXPIRE/PEXPIREAT), but the success check
at the end was only testing the result of PSETEX.

The reason the PEXPIREAT test nearly always failed is because it was
getting the current time wrong: getting the current second and loosing
the sub-section time, so the only chance for it to succeed is if it run
right when a certain second started.

Because i now get the time from redis, adding another round trip, i
added another 100ms to the PEXPIRE test to make it less fragile, and
also added many more attempts.

Adding many more attempts before failure to account for slow platforms,
github actions and valgrind
---
 tests/unit/expire.tcl | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl
index 444525f36..8bcdc16b7 100644
--- a/tests/unit/expire.tcl
+++ b/tests/unit/expire.tcl
@@ -92,7 +92,7 @@ start_server {tags {"expire"}} {
         # This test is very likely to do a false positive if the
         # server is under pressure, so if it does not work give it a few more
         # chances.
-        for {set j 0} {$j < 3} {incr j} {
+        for {set j 0} {$j < 30} {incr j} {
             r del x y z
             r psetex x 100 somevalue
             after 80
@@ -108,18 +108,22 @@ start_server {tags {"expire"}} {
             set d [r get x]
 
             r set x somevalue
-            r pexpireat x [expr ([clock seconds]*1000)+100]
-            after 80
+            set now [r time]
+            r pexpireat x [expr ([lindex $now 0]*1000)+([lindex $now 1]/1000)+200]
+            after 20
             set e [r get x]
-            after 120
+            after 220
             set f [r get x]
 
             if {$a eq {somevalue} && $b eq {} &&
                 $c eq {somevalue} && $d eq {} &&
                 $e eq {somevalue} && $f eq {}} break
         }
-        list $a $b
-    } {somevalue {}}
+        if {$::verbose} {
+            puts "sub-second expire test attempts: $j"
+        }
+        list $a $b $c $d $e $f
+    } {somevalue {} somevalue {} somevalue {}}
 
     test {TTL returns time to live in seconds} {
         r del x

From 78a6e5eb2b19adb40b5ec8fc867345b3c5479586 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 13 Sep 2020 13:51:21 +0300
Subject: [PATCH 0655/1098] Fix failing valgrind installation in github actions
 (#7792)

These tests started failing every day on http 404 (not being able to
install valgrind)
---
 .github/workflows/daily.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 07cd55c87..087b9f2ef 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -109,6 +109,7 @@ jobs:
       run: make valgrind
     - name: test
       run: |
+        sudo apt-get update
         sudo apt-get install tcl8.5 valgrind -y
         ./runtest --valgrind --verbose --clients 1
     - name: module api test

From cb2c7c4d34e01a4c33b5f50f2dc6ca6e857360e0 Mon Sep 17 00:00:00 2001
From: Mykhailo Pylyp <pylypsmol@gmail.com>
Date: Sun, 13 Sep 2020 18:39:59 +0300
Subject: [PATCH 0656/1098] Recalculate hardcoded variables from
 $::instances_count in sentinel tests (#7561)

Co-authored-by: MemuraiUser <githubuser@janeasystems.com>
---
 tests/sentinel/tests/06-ckquorum.tcl         | 13 +++++++------
 tests/sentinel/tests/07-down-conditions.tcl  | 13 +++++++------
 tests/sentinel/tests/includes/init-tests.tcl |  2 +-
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/tests/sentinel/tests/06-ckquorum.tcl b/tests/sentinel/tests/06-ckquorum.tcl
index 31e5fa2f8..4ea4e55d8 100644
--- a/tests/sentinel/tests/06-ckquorum.tcl
+++ b/tests/sentinel/tests/06-ckquorum.tcl
@@ -20,15 +20,16 @@ test "CKQUORUM detects quorum cannot be reached" {
 test "CKQUORUM detects failover authorization cannot be reached" {
     set orig_quorum [expr {$num_sentinels/2+1}]
     S 0 SENTINEL SET mymaster quorum 1
-    kill_instance sentinel 1
-    kill_instance sentinel 2
-    kill_instance sentinel 3
+    for {set i 0} {$i < $orig_quorum} {incr i} {
+        kill_instance sentinel [expr {$i + 1}]
+    }
+
     after 5000
     catch {[S 0 SENTINEL CKQUORUM mymaster]} err
     assert_match "*NOQUORUM*" $err
     S 0 SENTINEL SET mymaster quorum $orig_quorum
-    restart_instance sentinel 1
-    restart_instance sentinel 2
-    restart_instance sentinel 3
+    for {set i 0} {$i < $orig_quorum} {incr i} {
+        restart_instance sentinel [expr {$i + 1}]
+    }
 }
 
diff --git a/tests/sentinel/tests/07-down-conditions.tcl b/tests/sentinel/tests/07-down-conditions.tcl
index a12ea3151..0a696fa6b 100644
--- a/tests/sentinel/tests/07-down-conditions.tcl
+++ b/tests/sentinel/tests/07-down-conditions.tcl
@@ -3,9 +3,10 @@
 source "../tests/includes/init-tests.tcl"
 source "../../../tests/support/cli.tcl"
 
+set ::alive_sentinel [expr {$::instances_count/2+2}]
 proc ensure_master_up {} {
     wait_for_condition 1000 50 {
-        [dict get [S 4 sentinel master mymaster] flags] eq "master"
+        [dict get [S $::alive_sentinel sentinel master mymaster] flags] eq "master"
     } else {
         fail "Master flags are not just 'master'"
     }
@@ -14,7 +15,7 @@ proc ensure_master_up {} {
 proc ensure_master_down {} {
     wait_for_condition 1000 50 {
         [string match *down* \
-            [dict get [S 4 sentinel master mymaster] flags]]
+            [dict get [S $::alive_sentinel sentinel master mymaster] flags]]
     } else {
         fail "Master is not flagged SDOWN"
     }
@@ -27,7 +28,7 @@ test "Crash the majority of Sentinels to prevent failovers for this unit" {
 }
 
 test "SDOWN is triggered by non-responding but not crashed instance" {
-    lassign [S 4 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] host port
+    lassign [S $::alive_sentinel SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] host port
     ensure_master_up
     exec ../../../src/redis-cli -h $host -p $port {*}[rediscli_tls_config "../../../tests"] debug sleep 10 > /dev/null &
     ensure_master_down
@@ -35,7 +36,7 @@ test "SDOWN is triggered by non-responding but not crashed instance" {
 }
 
 test "SDOWN is triggered by crashed instance" {
-    lassign [S 4 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] host port
+    lassign [S $::alive_sentinel SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] host port
     ensure_master_up
     kill_instance redis 0
     ensure_master_down
@@ -72,8 +73,8 @@ test "SDOWN is triggered by misconfigured instance repling with errors" {
 # effect of the master going down if we send PONG instead of PING
 test "SDOWN is triggered if we rename PING to PONG" {
     ensure_master_up
-    S 4 SENTINEL SET mymaster rename-command PING PONG
+    S $::alive_sentinel SENTINEL SET mymaster rename-command PING PONG
     ensure_master_down
-    S 4 SENTINEL SET mymaster rename-command PING PING
+    S $::alive_sentinel SENTINEL SET mymaster rename-command PING PING
     ensure_master_up
 }
diff --git a/tests/sentinel/tests/includes/init-tests.tcl b/tests/sentinel/tests/includes/init-tests.tcl
index c8165dcfa..234f9c589 100644
--- a/tests/sentinel/tests/includes/init-tests.tcl
+++ b/tests/sentinel/tests/includes/init-tests.tcl
@@ -18,7 +18,7 @@ test "(init) Remove old master entry from sentinels" {
     }
 }
 
-set redis_slaves 4
+set redis_slaves [expr $::instances_count - 1]
 test "(init) Create a master-slaves cluster of [expr $redis_slaves+1] instances" {
     create_redis_master_slave_cluster [expr {$redis_slaves+1}]
 }

From 98c8ac0d705608e866a8c0055b09d4a40655f492 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Tue, 15 Sep 2020 13:27:42 +0800
Subject: [PATCH 0657/1098] Clarify help text of tcl scripts. (#7798)

Before this commit, following command did not show --tls option:
./runtest-cluster --help
./runtest-sentinel --help
---
 tests/instances.tcl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/instances.tcl b/tests/instances.tcl
index 275db45c3..5c4b665db 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -243,6 +243,7 @@ proc parse_options {} {
             puts "--pause-on-error        Pause for manual inspection on error."
             puts "--fail                  Simulate a test failure."
             puts "--valgrind              Run with valgrind."
+            puts "--tls                   Run tests in TLS mode."
             puts "--help                  Shows this help."
             exit 0
         } else {

From 6ff741b5cf4d1cf7e23610aa9eef7ed485bd3113 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Tue, 15 Sep 2020 01:58:21 -0400
Subject: [PATCH 0658/1098] correct OBJECT ENCODING response for stream type
 (#7797)

This commit makes stream object returning "stream" as encoding type in OBJECT ENCODING subcommand and DEBUG OBJECT command.

Till now, it would return "unknown"
---
 src/object.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/object.c b/src/object.c
index 732241e84..1380fffa5 100644
--- a/src/object.c
+++ b/src/object.c
@@ -740,6 +740,7 @@ char *strEncoding(int encoding) {
     case OBJ_ENCODING_INTSET: return "intset";
     case OBJ_ENCODING_SKIPLIST: return "skiplist";
     case OBJ_ENCODING_EMBSTR: return "embstr";
+    case OBJ_ENCODING_STREAM: return "stream";
     default: return "unknown";
     }
 }

From 810e28a397b77fa16037354477370b564474d9be Mon Sep 17 00:00:00 2001
From: Jim Brunner <brunnerj@amazon.com>
Date: Tue, 15 Sep 2020 23:16:01 -0700
Subject: [PATCH 0659/1098] Incremental eviction processing (#7653)

Rather than blindly evicting until maxmemory limit is achieved, this
update adds a time limit to eviction.  While over the maxmemory limit,
eviction will process before each command AND as a timeProc when no
commands are running.

This will reduce the latency impact on many cases, especially pathological
cases like massive used memory increase during dict rehashing.

There is a risk that some other edge cases (like massive pipelined use
of MGET) could cause Redis memory usage to keep growing despite the
eviction attempts, so a new maxmemory-eviction-tenacity config is
introduced to let users mitigate that.
---
 README.md        |   2 +-
 redis.conf       |   8 ++
 src/config.c     |   3 +-
 src/evict.c      | 227 +++++++++++++++++++++++++++++------------------
 src/networking.c |   6 +-
 src/server.c     |   6 +-
 src/server.h     |   8 +-
 7 files changed, 163 insertions(+), 97 deletions(-)

diff --git a/README.md b/README.md
index eec7e4bdd..e1cd3b87a 100644
--- a/README.md
+++ b/README.md
@@ -354,7 +354,7 @@ Inside server.c you can find code that handles other vital things of the Redis s
 
 * `call()` is used in order to call a given command in the context of a given client.
 * `activeExpireCycle()` handles eviction of keys with a time to live set via the `EXPIRE` command.
-* `freeMemoryIfNeeded()` is called when a new write command should be performed but Redis is out of memory according to the `maxmemory` directive.
+* `performEvictions()` is called when a new write command should be performed but Redis is out of memory according to the `maxmemory` directive.
 * The global variable `redisCommandTable` defines all the Redis commands, specifying the name of the command, the function implementing the command, the number of arguments required, and other properties of each command.
 
 networking.c
diff --git a/redis.conf b/redis.conf
index 51f71484c..424dfa09a 100644
--- a/redis.conf
+++ b/redis.conf
@@ -912,6 +912,14 @@ acllog-max-len 128
 #
 # maxmemory-samples 5
 
+# Eviction processing is designed to function well with the default setting.
+# If there is an unusually large amount of write traffic, this value may need to
+# be increased.  Decreasing this value may reduce latency at the risk of 
+# eviction processing effectiveness
+#   0 = minimum latency, 10 = default, 100 = process without regard to latency
+#
+# maxmemory-eviction-tenacity 10
+
 # Starting from Redis 5, by default a replica will ignore its maxmemory setting
 # (unless it is promoted to master after a failover or manually). It means
 # that the eviction of keys will be just handled by the master, sending the
diff --git a/src/config.c b/src/config.c
index 64a31fc67..be0494ccd 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2139,7 +2139,7 @@ static int updateMaxmemory(long long val, long long prev, char **err) {
         if ((unsigned long long)val < used) {
             serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET (%llu) is smaller than the current memory usage (%zu). This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy.", server.maxmemory, used);
         }
-        freeMemoryIfNeededAndSafe();
+        performEvictions();
     }
     return 1;
 }
@@ -2327,6 +2327,7 @@ standardConfig configs[] = {
     createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, server.slave_priority, 100, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_diskless_sync_delay, 5, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.maxmemory_samples, 5, INTEGER_CONFIG, NULL, NULL),
+    createIntConfig("maxmemory-eviction-tenacity", NULL, MODIFIABLE_CONFIG, 0, 100, server.maxmemory_eviction_tenacity, 10, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.maxidletime, 0, INTEGER_CONFIG, NULL, NULL), /* Default client timeout: infinite */
     createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, server.slave_announce_port, 0, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("tcp-backlog", NULL, IMMUTABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, 511, INTEGER_CONFIG, NULL, NULL), /* TCP listen backlog. */
diff --git a/src/evict.c b/src/evict.c
index 5d398c6c9..258e1a25e 100644
--- a/src/evict.c
+++ b/src/evict.c
@@ -33,13 +33,14 @@
 #include "server.h"
 #include "bio.h"
 #include "atomicvar.h"
+#include <math.h>
 
 /* ----------------------------------------------------------------------------
  * Data structures
  * --------------------------------------------------------------------------*/
 
 /* To improve the quality of the LRU approximation we take a set of keys
- * that are good candidate for eviction across freeMemoryIfNeeded() calls.
+ * that are good candidate for eviction across performEvictions() calls.
  *
  * Entries inside the eviction pool are taken ordered by idle time, putting
  * greater idle times to the right (ascending order).
@@ -97,25 +98,7 @@ unsigned long long estimateObjectIdleTime(robj *o) {
     }
 }
 
-/* freeMemoryIfNeeded() gets called when 'maxmemory' is set on the config
- * file to limit the max memory used by the server, before processing a
- * command.
- *
- * The goal of the function is to free enough memory to keep Redis under the
- * configured memory limit.
- *
- * The function starts calculating how many bytes should be freed to keep
- * Redis under the limit, and enters a loop selecting the best keys to
- * evict accordingly to the configured policy.
- *
- * If all the bytes needed to return back under the limit were freed the
- * function returns C_OK, otherwise C_ERR is returned, and the caller
- * should block the execution of commands that will result in more memory
- * used by the server.
- *
- * ------------------------------------------------------------------------
- *
- * LRU approximation algorithm
+/* LRU approximation algorithm
  *
  * Redis uses an approximation of the LRU algorithm that runs in constant
  * memory. Every time there is a key to expire, we sample N keys (with
@@ -150,7 +133,7 @@ void evictionPoolAlloc(void) {
     EvictionPoolLRU = ep;
 }
 
-/* This is an helper function for freeMemoryIfNeeded(), it is used in order
+/* This is an helper function for performEvictions(), it is used in order
  * to populate the evictionPool with a few entries every time we want to
  * expire a key. Keys with idle time smaller than one of the current
  * keys are added. Keys are always added if there are free entries.
@@ -341,11 +324,6 @@ unsigned long LFUDecrAndReturn(robj *o) {
     return counter;
 }
 
-/* ----------------------------------------------------------------------------
- * The external API for eviction: freeMemoryIfNeeded() is called by the
- * server when there is data to add in order to make space if needed.
- * --------------------------------------------------------------------------*/
-
 /* We don't want to count AOF buffers and slaves output buffers as
  * used memory: the eviction should use mostly data size. This function
  * returns the sum of AOF and slaves buffer. */
@@ -434,39 +412,113 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev
     return C_ERR;
 }
 
-/* This function is periodically called to see if there is memory to free
- * according to the current "maxmemory" settings. In case we are over the
- * memory limit, the function will try to free some memory to return back
- * under the limit.
- *
- * The function returns C_OK if we are under the memory limit or if we
- * were over the limit, but the attempt to free memory was successful.
- * Otherwise if we are over the memory limit, but not enough memory
- * was freed to return back under the limit, the function returns C_ERR. */
-int freeMemoryIfNeeded(void) {
-    int keys_freed = 0;
+
+/* The evictionTimeProc is started when "maxmemory" has been breached and
+ * could not immediately be resolved.  This will spin the event loop with short
+ * eviction cycles until the "maxmemory" condition has resolved or there are no
+ * more evictable items.  */
+static int isEvictionProcRunning = 0;
+static int evictionTimeProc(
+        struct aeEventLoop *eventLoop, long long id, void *clientData) {
+    UNUSED(eventLoop);
+    UNUSED(id);
+    UNUSED(clientData);
+
+    if (performEvictions() == EVICT_RUNNING) return 0;  /* keep evicting */
+
+    /* For EVICT_OK - things are good, no need to keep evicting.
+     * For EVICT_FAIL - there is nothing left to evict.  */
+    isEvictionProcRunning = 0;
+    return AE_NOMORE;
+}
+
+/* Check if it's safe to perform evictions.
+ *   Returns 1 if evictions can be performed
+ *   Returns 0 if eviction processing should be skipped
+ */
+static int isSafeToPerformEvictions(void) {
+    /* - There must be no script in timeout condition.
+     * - Nor we are loading data right now.  */
+    if (server.lua_timedout || server.loading) return 0;
+
     /* By default replicas should ignore maxmemory
      * and just be masters exact copies. */
-    if (server.masterhost && server.repl_slave_ignore_maxmemory) return C_OK;
-
-    size_t mem_reported, mem_tofree, mem_freed;
-    mstime_t latency, eviction_latency, lazyfree_latency;
-    long long delta;
-    int slaves = listLength(server.slaves);
-    int result = C_ERR;
+    if (server.masterhost && server.repl_slave_ignore_maxmemory) return 0;
 
     /* When clients are paused the dataset should be static not just from the
      * POV of clients not being able to write, but also from the POV of
      * expires and evictions of keys not being performed. */
-    if (clientsArePaused()) return C_OK;
+    if (clientsArePaused()) return 0;
+
+    return 1;
+}
+
+/* Algorithm for converting tenacity (0-100) to a time limit.  */
+static unsigned long evictionTimeLimitUs() {
+    serverAssert(server.maxmemory_eviction_tenacity >= 0);
+    serverAssert(server.maxmemory_eviction_tenacity <= 100);
+
+    if (server.maxmemory_eviction_tenacity <= 10) {
+        /* A linear progression from 0..500us */
+        return 50uL * server.maxmemory_eviction_tenacity;
+    }
+
+    if (server.maxmemory_eviction_tenacity < 100) {
+        /* A 15% geometric progression, resulting in a limit of ~2 min at tenacity==99  */
+        return (unsigned long)(500.0 * pow(1.15, server.maxmemory_eviction_tenacity - 10.0));
+    }
+
+    return ULONG_MAX;   /* No limit to eviction time */
+}
+
+/* Check that memory usage is within the current "maxmemory" limit.  If over
+ * "maxmemory", attempt to free memory by evicting data (if it's safe to do so).
+ *
+ * It's possible for Redis to suddenly be significantly over the "maxmemory"
+ * setting.  This can happen if there is a large allocation (like a hash table
+ * resize) or even if the "maxmemory" setting is manually adjusted.  Because of
+ * this, it's important to evict for a managed period of time - otherwise Redis
+ * would become unresponsive while evicting.
+ *
+ * The goal of this function is to improve the memory situation - not to
+ * immediately resolve it.  In the case that some items have been evicted but
+ * the "maxmemory" limit has not been achieved, an aeTimeProc will be started
+ * which will continue to evict items until memory limits are achieved or
+ * nothing more is evictable.
+ *
+ * This should be called before execution of commands.  If EVICT_FAIL is
+ * returned, commands which will result in increased memory usage should be
+ * rejected.
+ *
+ * Returns:
+ *   EVICT_OK       - memory is OK or it's not possible to perform evictions now
+ *   EVICT_RUNNING  - memory is over the limit, but eviction is still processing
+ *   EVICT_FAIL     - memory is over the limit, and there's nothing to evict
+ * */
+int performEvictions(void) {
+    if (!isSafeToPerformEvictions()) return EVICT_OK;
+
+    int keys_freed = 0;
+    size_t mem_reported, mem_tofree, mem_freed;
+    mstime_t latency, eviction_latency;
+    long long delta;
+    int slaves = listLength(server.slaves);
+    int result = EVICT_FAIL;
+
     if (getMaxmemoryState(&mem_reported,NULL,&mem_tofree,NULL) == C_OK)
-        return C_OK;
+        return EVICT_OK;
+
+    if (server.maxmemory_policy == MAXMEMORY_NO_EVICTION)
+        return EVICT_FAIL;  /* We need to free memory, but policy forbids. */
+
+    unsigned long eviction_time_limit_us = evictionTimeLimitUs();
 
     mem_freed = 0;
 
     latencyStartMonitor(latency);
-    if (server.maxmemory_policy == MAXMEMORY_NO_EVICTION)
-        goto cant_free; /* We need to free memory, but policy forbids. */
+
+    monotime evictionTimer;
+    elapsedStart(&evictionTimer);
 
     while (mem_freed < mem_tofree) {
         int j, k, i;
@@ -581,60 +633,61 @@ int freeMemoryIfNeeded(void) {
             decrRefCount(keyobj);
             keys_freed++;
 
-            /* When the memory to free starts to be big enough, we may
-             * start spending so much time here that is impossible to
-             * deliver data to the slaves fast enough, so we force the
-             * transmission here inside the loop. */
-            if (slaves) flushSlavesOutputBuffers();
+            if (keys_freed % 16 == 0) {
+                /* When the memory to free starts to be big enough, we may
+                 * start spending so much time here that is impossible to
+                 * deliver data to the replicas fast enough, so we force the
+                 * transmission here inside the loop. */
+                if (slaves) flushSlavesOutputBuffers();
 
-            /* Normally our stop condition is the ability to release
-             * a fixed, pre-computed amount of memory. However when we
-             * are deleting objects in another thread, it's better to
-             * check, from time to time, if we already reached our target
-             * memory, since the "mem_freed" amount is computed only
-             * across the dbAsyncDelete() call, while the thread can
-             * release the memory all the time. */
-            if (server.lazyfree_lazy_eviction && !(keys_freed % 16)) {
-                if (getMaxmemoryState(NULL,NULL,NULL,NULL) == C_OK) {
-                    /* Let's satisfy our stop condition. */
-                    mem_freed = mem_tofree;
+                /* Normally our stop condition is the ability to release
+                 * a fixed, pre-computed amount of memory. However when we
+                 * are deleting objects in another thread, it's better to
+                 * check, from time to time, if we already reached our target
+                 * memory, since the "mem_freed" amount is computed only
+                 * across the dbAsyncDelete() call, while the thread can
+                 * release the memory all the time. */
+                if (server.lazyfree_lazy_eviction) {
+                    if (getMaxmemoryState(NULL,NULL,NULL,NULL) == C_OK) {
+                        break;
+                    }
+                }
+
+                /* After some time, exit the loop early - even if memory limit
+                 * hasn't been reached.  If we suddenly need to free a lot of
+                 * memory, don't want to spend too much time here.  */
+                if (elapsedUs(evictionTimer) > eviction_time_limit_us) {
+                    // We still need to free memory - start eviction timer proc
+                    if (!isEvictionProcRunning) {
+                        isEvictionProcRunning = 1;
+                        aeCreateTimeEvent(server.el, 0,
+                                evictionTimeProc, NULL, NULL);
+                    }
+                    break;
                 }
             }
         } else {
             goto cant_free; /* nothing to free... */
         }
     }
-    result = C_OK;
+    /* at this point, the memory is OK, or we have reached the time limit */
+    result = (isEvictionProcRunning) ? EVICT_RUNNING : EVICT_OK;
 
 cant_free:
-    /* We are here if we are not able to reclaim memory. There is only one
-     * last thing we can try: check if the lazyfree thread has jobs in queue
-     * and wait... */
-    if (result != C_OK) {
-        latencyStartMonitor(lazyfree_latency);
-        while(bioPendingJobsOfType(BIO_LAZY_FREE)) {
+    if (result == EVICT_FAIL) {
+        /* At this point, we have run out of evictable items.  It's possible
+         * that some items are being freed in the lazyfree thread.  Perform a
+         * short wait here if such jobs exist, but don't wait long.  */
+        if (bioPendingJobsOfType(BIO_LAZY_FREE)) {
+            usleep(eviction_time_limit_us);
             if (getMaxmemoryState(NULL,NULL,NULL,NULL) == C_OK) {
-                result = C_OK;
-                break;
+                result = EVICT_OK;
             }
-            usleep(1000);
         }
-        latencyEndMonitor(lazyfree_latency);
-        latencyAddSampleIfNeeded("eviction-lazyfree",lazyfree_latency);
     }
+
     latencyEndMonitor(latency);
     latencyAddSampleIfNeeded("eviction-cycle",latency);
     return result;
 }
 
-/* This is a wrapper for freeMemoryIfNeeded() that only really calls the
- * function if right now there are the conditions to do so safely:
- *
- * - There must be no script in timeout condition.
- * - Nor we are loading data right now.
- *
- */
-int freeMemoryIfNeededAndSafe(void) {
-    if (server.lua_timedout || server.loading) return C_OK;
-    return freeMemoryIfNeeded();
-}
diff --git a/src/networking.c b/src/networking.c
index cf40b69e4..8090e391d 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1830,8 +1830,8 @@ int processCommandAndResetClient(client *c) {
     }
     if (server.current_client == NULL) deadclient = 1;
     server.current_client = NULL;
-    /* freeMemoryIfNeeded may flush slave output buffers. This may
-     * result into a slave, that may be the active client, to be
+    /* performEvictions may flush slave output buffers. This may
+     * result in a slave, that may be the active client, to be
      * freed. */
     return deadclient ? C_ERR : C_OK;
 }
@@ -2797,7 +2797,7 @@ void asyncCloseClientOnOutputBufferLimitReached(client *c) {
     }
 }
 
-/* Helper function used by freeMemoryIfNeeded() in order to flush slaves
+/* Helper function used by performEvictions() in order to flush slaves
  * output buffers without returning control to the event loop.
  * This is also called by SHUTDOWN for a best-effort attempt to send
  * slaves the latest writes. */
diff --git a/src/server.c b/src/server.c
index a5b2538fa..b0b12d59d 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3681,9 +3681,9 @@ int processCommand(client *c) {
      * condition, to avoid mixing the propagation of scripts with the
      * propagation of DELs due to eviction. */
     if (server.maxmemory && !server.lua_timedout) {
-        int out_of_memory = freeMemoryIfNeededAndSafe() == C_ERR;
-        /* freeMemoryIfNeeded may flush slave output buffers. This may result
-         * into a slave, that may be the active client, to be freed. */
+        int out_of_memory = (performEvictions() == EVICT_FAIL);
+        /* performEvictions may flush slave output buffers. This may result
+         * in a slave, that may be the active client, to be freed. */
         if (server.current_client == NULL) return C_ERR;
 
         int reject_cmd_on_oom = is_denyoom_command;
diff --git a/src/server.h b/src/server.h
index 8d26e6009..4122008dc 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1358,6 +1358,7 @@ struct redisServer {
     unsigned long long maxmemory;   /* Max number of memory bytes to use */
     int maxmemory_policy;           /* Policy for key eviction */
     int maxmemory_samples;          /* Precision of random sampling */
+    int maxmemory_eviction_tenacity;/* Aggressiveness of eviction processing */
     int lfu_log_factor;             /* LFU logarithmic counter factor. */
     int lfu_decay_time;             /* LFU counter decay factor. */
     long long proto_max_bulk_len;   /* Protocol bulk length maximum size. */
@@ -1996,8 +1997,6 @@ int zslLexValueLteMax(sds value, zlexrangespec *spec);
 /* Core functions */
 int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *level);
 size_t freeMemoryGetNotCountedMemory();
-int freeMemoryIfNeeded(void);
-int freeMemoryIfNeededAndSafe(void);
 int processCommand(client *c);
 void setupSignalHandlers(void);
 void removeSignalHandlers(void);
@@ -2234,6 +2233,11 @@ void evictionPoolAlloc(void);
 unsigned long LFUGetTimeInMinutes(void);
 uint8_t LFULogIncr(uint8_t value);
 unsigned long LFUDecrAndReturn(robj *o);
+#define EVICT_OK 0
+#define EVICT_RUNNING 1
+#define EVICT_FAIL 2
+int performEvictions(void);
+
 
 /* Keys hashing / comparison functions for dict.c hash tables. */
 uint64_t dictSdsHash(const void *key);

From 8b70cb0ef8e654d09d0d2974ad72388f46be9fde Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Tue, 15 Sep 2020 13:06:47 +0800
Subject: [PATCH 0660/1098] bio: fix doFastMemoryTest.

If one thread got SIGSEGV, function sigsegvHandler() would be triggered,
it would call bioKillThreads(). But call pthread_cancel() to cancel itself
would make it block. Also note that if SIGSEGV is caught by bio thread, it
should kill the main thread in order to give a positive report.
---
 src/bio.c    |  3 ++-
 src/debug.c  | 22 +++++++++++++++++++++-
 src/server.c |  2 +-
 src/server.h |  1 +
 4 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/src/bio.c b/src/bio.c
index cbbecb6f7..dd99442ae 100644
--- a/src/bio.c
+++ b/src/bio.c
@@ -268,10 +268,11 @@ void bioKillThreads(void) {
     int err, j;
 
     for (j = 0; j < BIO_NUM_OPS; j++) {
+        if (bio_threads[j] == pthread_self()) continue;
         if (bio_threads[j] && pthread_cancel(bio_threads[j]) == 0) {
             if ((err = pthread_join(bio_threads[j],NULL)) != 0) {
                 serverLog(LL_WARNING,
-                    "Bio thread for job type #%d can be joined: %s",
+                    "Bio thread for job type #%d can not be joined: %s",
                         j, strerror(err));
             } else {
                 serverLog(LL_WARNING,
diff --git a/src/debug.c b/src/debug.c
index a59c46046..785ad4e41 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1565,12 +1565,32 @@ int memtest_test_linux_anonymous_maps(void) {
 }
 #endif
 
+static void killMainThread(void) {
+    int err;
+    if (pthread_self() != server.main_thread_id && pthread_cancel(server.main_thread_id) == 0) {
+        if ((err = pthread_join(server.main_thread_id,NULL)) != 0) {
+            serverLog(LL_WARNING, "main thread can not be joined: %s", strerror(err));
+        } else {
+            serverLog(LL_WARNING, "main thread terminated");
+        }
+    }
+}
+
+/* Kill the running threads (other than current) in an unclean way. This function
+ * should be used only when it's critical to stop the threads for some reason.
+ * Currently Redis does this only on crash (for instance on SIGSEGV) in order
+ * to perform a fast memory check without other threads messing with memory. */
+static void killThreads(void) {
+    killMainThread();
+    bioKillThreads();
+}
+
 void doFastMemoryTest(void) {
 #if defined(HAVE_PROC_MAPS)
     if (server.memcheck_enabled) {
         /* Test memory */
         serverLogRaw(LL_WARNING|LL_RAW, "\n------ FAST MEMORY TEST ------\n");
-        bioKillThreads();
+        killThreads();
         if (memtest_test_linux_anonymous_maps()) {
             serverLogRaw(LL_WARNING|LL_RAW,
                 "!!! MEMORY ERROR DETECTED! Check your memory ASAP !!!\n");
diff --git a/src/server.c b/src/server.c
index b0b12d59d..e8c51ee86 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2879,6 +2879,7 @@ void initServer(void) {
     server.aof_state = server.aof_enabled ? AOF_ON : AOF_OFF;
     server.hz = server.config_hz;
     server.pid = getpid();
+    server.main_thread_id = pthread_self();
     server.current_client = NULL;
     server.fixed_time_expire = 0;
     server.clients = listCreate();
@@ -5174,7 +5175,6 @@ int iAmMaster(void) {
             (server.cluster_enabled && nodeIsMaster(server.cluster->myself)));
 }
 
-
 int main(int argc, char **argv) {
     struct timeval tv;
     int j;
diff --git a/src/server.h b/src/server.h
index 4122008dc..d4d8de95a 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1050,6 +1050,7 @@ struct clusterState;
 struct redisServer {
     /* General */
     pid_t pid;                  /* Main process pid. */
+    pthread_t main_thread_id;         /* Main thread id */
     char *configfile;           /* Absolute config file path, or NULL */
     char *executable;           /* Absolute executable file path. */
     char **exec_argv;           /* Executable argv vector (copy). */

From f86602339968cc89e31bd51fb9d6c771c3ab26ee Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Wed, 16 Sep 2020 09:58:24 +0800
Subject: [PATCH 0661/1098] bio: doFastMemoryTest should try to kill io threads
 as well.

---
 src/debug.c      |  1 +
 src/networking.c | 17 +++++++++++++++++
 src/server.h     |  1 +
 3 files changed, 19 insertions(+)

diff --git a/src/debug.c b/src/debug.c
index 785ad4e41..32e0c373c 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1583,6 +1583,7 @@ static void killMainThread(void) {
 static void killThreads(void) {
     killMainThread();
     bioKillThreads();
+    killIOThreads();
 }
 
 void doFastMemoryTest(void) {
diff --git a/src/networking.c b/src/networking.c
index 8090e391d..6becdc288 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -3028,6 +3028,23 @@ void initThreadedIO(void) {
     }
 }
 
+void killIOThreads(void) {
+    int err, j;
+    for (j = 0; j < server.io_threads_num; j++) {
+        if (io_threads[j] == pthread_self()) continue;
+        if (io_threads[j] && pthread_cancel(io_threads[j]) == 0) {
+            if ((err = pthread_join(io_threads[j],NULL)) != 0) {
+                serverLog(LL_WARNING,
+                    "IO thread(tid:%lu) can not be joined: %s",
+                        (unsigned long)io_threads[j], strerror(err));
+            } else {
+                serverLog(LL_WARNING,
+                    "IO thread(tid:%lu) terminated",(unsigned long)io_threads[j]);
+            }
+        }
+    }
+}
+
 void startThreadedIO(void) {
     if (tio_debug) { printf("S"); fflush(stdout); }
     if (tio_debug) printf("--- STARTING THREADED IO ---\n");
diff --git a/src/server.h b/src/server.h
index d4d8de95a..ae2cecf31 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2478,6 +2478,7 @@ void mixDigest(unsigned char *digest, void *ptr, size_t len);
 void xorDigest(unsigned char *digest, void *ptr, size_t len);
 int populateCommandTableParseFlags(struct redisCommand *c, char *strflags);
 void debugDelay(int usec);
+void killIOThreads(void);
 
 /* TLS stuff */
 void tlsInit(void);

From 092cfca5224e8e88053cd5b8a7b82f0f14b17011 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 16 Sep 2020 20:21:04 +0300
Subject: [PATCH 0662/1098] Add printf attribute and fix warnings and a minor
 bug (#7803)

The fix in error handling of rdbGenericLoadStringObject is an actual bugfix
---
 src/rdb.c    | 11 +++++++----
 src/server.h |  5 +++++
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/rdb.c b/src/rdb.c
index 80fe3c5a6..37e09ef96 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -52,6 +52,9 @@ extern int rdbCheckMode;
 void rdbCheckError(const char *fmt, ...);
 void rdbCheckSetError(const char *fmt, ...);
 
+#ifdef __GNUC__
+void rdbReportError(int corruption_error, int linenum, char *reason, ...) __attribute__ ((format (printf, 3, 4)));
+#endif
 void rdbReportError(int corruption_error, int linenum, char *reason, ...) {
     va_list ap;
     char msg[1024];
@@ -487,7 +490,7 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags, size_t *lenptr) {
     int plain = flags & RDB_LOAD_PLAIN;
     int sds = flags & RDB_LOAD_SDS;
     int isencoded;
-    uint64_t len;
+    unsigned long long len;
 
     len = rdbLoadLen(rdb,&isencoded);
     if (isencoded) {
@@ -499,8 +502,8 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags, size_t *lenptr) {
         case RDB_ENC_LZF:
             return rdbLoadLzfStringObject(rdb,flags,lenptr);
         default:
-            rdbExitReportCorruptRDB("Unknown RDB string encoding type %d",len);
-            return NULL; /* Never reached. */
+            rdbExitReportCorruptRDB("Unknown RDB string encoding type %llu",len);
+            return NULL;
         }
     }
 
@@ -2198,7 +2201,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
                 if (luaCreateFunction(NULL,server.lua,auxval) == NULL) {
                     rdbExitReportCorruptRDB(
                         "Can't load Lua script from RDB file! "
-                        "BODY: %s", auxval->ptr);
+                        "BODY: %s", (char*)auxval->ptr);
                 }
             } else if (!strcasecmp(auxkey->ptr,"redis-ver")) {
                 serverLog(LL_NOTICE,"Loading RDB produced by version %s",
diff --git a/src/server.h b/src/server.h
index ae2cecf31..8cba2f9b5 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2464,7 +2464,12 @@ void *realloc(void *ptr, size_t size) __attribute__ ((deprecated));
 /* Debugging stuff */
 void _serverAssertWithInfo(const client *c, const robj *o, const char *estr, const char *file, int line);
 void _serverAssert(const char *estr, const char *file, int line);
+#ifdef __GNUC__
+void _serverPanic(const char *file, int line, const char *msg, ...)
+    __attribute__ ((format (printf, 3, 4)));
+#else
 void _serverPanic(const char *file, int line, const char *msg, ...);
+#endif
 void serverLogObjectDebugInfo(const robj *o);
 void sigsegvHandler(int sig, siginfo_t *info, void *secret);
 sds genRedisInfoString(const char *section);

From 445a4b669a3a7232a18bf23340c5f7d580aa92c7 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuan21@baidu.com>
Date: Thu, 17 Sep 2020 21:01:45 +0800
Subject: [PATCH 0663/1098] Implement redisAtomic to replace _Atomic C11
 builtin (#7707)

Redis 6.0 introduces I/O threads, it is so cool and efficient, we use C11
_Atomic to establish inter-thread synchronization without mutex. But the
compiler that must supports C11 _Atomic can compile redis code, that brings a
lot of inconvenience since some common platforms can't support by default such
as CentOS7, so we want to implement redis atomic type to make it more portable.

We have implemented our atomic variable for redis that only has 'relaxed'
operations in src/atomicvar.h, so we implement some operations with
'sequentially-consistent', just like the default behavior of C11 _Atomic that
can establish inter-thread synchronization. And we replace all uses of C11
_Atomic with redis atomic variable.

Our implementation of redis atomic variable uses C11 _Atomic, __atomic or
__sync macros if available, it supports most common platforms, and we will
detect automatically which feature we use. In Makefile we use a dummy file to
detect if the compiler supports C11 _Atomic. Now for gcc, we can compile redis
code theoretically if your gcc version is not less than 4.1.2(starts to support
__sync_xxx operations). Otherwise, we remove use mutex fallback to implement
redis atomic variable for performance and test. You will get compiling errors
if your compiler doesn't support all features of above.

For cover redis atomic variable tests, we add other CI jobs that build redis on
CentOS6 and CentOS7 and workflow daily jobs that run the tests on them.
For them, we just install gcc by default in order to cover different compiler
versions, gcc is 4.4.7 by default installation on CentOS6 and 4.8.5 on CentOS7.

We restore the feature that we can test redis with Helgrind to find data race
errors. But you need install Valgrind in the default path configuration firstly
before running your tests, since we use macros in helgrind.h to tell Helgrind
inter-thread happens-before relationship explicitly for avoiding false positives.
Please open an issue on github if you find data race errors relate to this commit.

Unrelated:
- Fix redefinition of typedef 'RedisModuleUserChangedFunc'
  For some old version compilers, they will report errors or warnings, if we
  re-define function type.
---
 .github/workflows/ci.yml    |  19 +++++++
 .github/workflows/daily.yml |  29 ++++++++--
 src/Makefile                |  14 ++++-
 src/atomicvar.h             | 109 ++++++++++++++++++++++--------------
 src/evict.c                 |   2 +-
 src/lazyfree.c              |   3 +-
 src/module.c                |   5 --
 src/networking.c            |  41 +++++++++-----
 src/redis-benchmark.c       |  25 +++------
 src/replication.c           |  10 ++--
 src/server.c                |  44 ++++++++++-----
 src/server.h                |  23 ++++----
 src/zmalloc.c               |   3 +-
 13 files changed, 207 insertions(+), 120 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4d6c1c14c..d28d331db 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -47,3 +47,22 @@ jobs:
     - name: make
       run: make MALLOC=libc
 
+  build-centos7-jemalloc:
+    runs-on: ubuntu-latest
+    container: centos:7
+    steps:
+    - uses: actions/checkout@v2
+    - name: make
+      run: |
+        yum -y install gcc make
+        make
+
+  build-centos6-jemalloc:
+    runs-on: ubuntu-latest
+    container: centos:6
+    steps:
+    - uses: actions/checkout@v1
+    - name: make
+      run: |
+        yum -y install gcc make
+        make
diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 087b9f2ef..1ccd99805 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -124,12 +124,33 @@ jobs:
     - uses: actions/checkout@v2
     - name: make
       run: |
-        yum -y install centos-release-scl
-        yum -y install devtoolset-7
-        scl enable devtoolset-7 "make"
+        yum -y install gcc make
+        make
     - name: test
       run: |
-        yum -y install tcl
+        yum -y install which tcl
+        ./runtest --accurate --verbose
+    - name: module api test
+      run: ./runtest-moduleapi --verbose
+    - name: sentinel tests
+      run: ./runtest-sentinel
+    - name: cluster tests
+      run: ./runtest-cluster
+
+  test-centos6-jemalloc:
+    runs-on: ubuntu-latest
+    if: github.repository == 'redis/redis'
+    container: centos:6
+    timeout-minutes: 14400
+    steps:
+    - uses: actions/checkout@v1
+    - name: make
+      run: |
+        yum -y install gcc make
+        make
+    - name: test
+      run: |
+        yum -y install which tcl util-linux-ng
         ./runtest --accurate --verbose
     - name: module api test
       run: ./runtest-moduleapi --verbose
diff --git a/src/Makefile b/src/Makefile
index 9ff344c6a..6414b8c78 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -20,7 +20,7 @@ DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram
 NODEPS:=clean distclean
 
 # Default settings
-STD=-std=c11 -pedantic -DREDIS_STATIC=''
+STD=-pedantic -DREDIS_STATIC=''
 ifneq (,$(findstring clang,$(CC)))
 ifneq (,$(findstring FreeBSD,$(uname_S)))
   STD+=-Wno-c11-extensions
@@ -29,6 +29,16 @@ endif
 WARN=-Wall -W -Wno-missing-field-initializers
 OPT=$(OPTIMIZATION)
 
+# Detect if the compiler supports C11 _Atomic
+C11_ATOMIC := $(shell sh -c 'echo "\#include <stdatomic.h>" > foo.c; \
+	$(CC) -std=c11 -c foo.c -o foo.o &> /dev/null; \
+	if [ -f foo.o ]; then echo "yes"; rm foo.o; fi; rm foo.c')
+ifeq ($(C11_ATOMIC),yes)
+	STD+=-std=c11
+else
+	STD+=-std=c99
+endif
+
 PREFIX?=/usr/local
 INSTALL_BIN=$(PREFIX)/bin
 INSTALL=install
@@ -367,7 +377,7 @@ valgrind:
 	$(MAKE) OPTIMIZATION="-O0" MALLOC="libc"
 
 helgrind:
-	$(MAKE) OPTIMIZATION="-O0" MALLOC="libc" CFLAGS="-D__ATOMIC_VAR_FORCE_SYNC_MACROS"
+	$(MAKE) OPTIMIZATION="-O0" MALLOC="libc" CFLAGS="-D__ATOMIC_VAR_FORCE_SYNC_MACROS" REDIS_CFLAGS="-I/usr/local/include" REDIS_LDFLAGS="-L/usr/local/lib"
 
 src/help.h:
 	@../utils/generate-command-help.rb > help.h
diff --git a/src/atomicvar.h b/src/atomicvar.h
index ecd26ad70..6ac04c605 100644
--- a/src/atomicvar.h
+++ b/src/atomicvar.h
@@ -1,5 +1,5 @@
-/* This file implements atomic counters using __atomic or __sync macros if
- * available, otherwise synchronizing different threads using a mutex.
+/* This file implements atomic counters using c11 _Atomic, __atomic or __sync
+ * macros if available, otherwise we will throw an error when compile.
  *
  * The exported interface is composed of three macros:
  *
@@ -8,16 +8,8 @@
  * atomicDecr(var,count) -- Decrement the atomic counter
  * atomicGet(var,dstvar) -- Fetch the atomic counter value
  * atomicSet(var,value)  -- Set the atomic counter value
- *
- * The variable 'var' should also have a declared mutex with the same
- * name and the "_mutex" postfix, for instance:
- *
- *  long myvar;
- *  pthread_mutex_t myvar_mutex;
- *  atomicSet(myvar,12345);
- *
- * If atomic primitives are available (tested in config.h) the mutex
- * is not used.
+ * atomicGetWithSync(var,value)  -- 'atomicGet' with inter-thread synchronization
+ * atomicSetWithSync(var,value)  -- 'atomicSet' with inter-thread synchronization
  *
  * Never use return value from the macros, instead use the AtomicGetIncr()
  * if you need to get the current value and increment it atomically, like
@@ -58,17 +50,64 @@
  */
 
 #include <pthread.h>
+#include "config.h"
 
 #ifndef __ATOMIC_VAR_H
 #define __ATOMIC_VAR_H
 
+/* Define redisAtomic for atomic variable. */
+#define redisAtomic
+
 /* To test Redis with Helgrind (a Valgrind tool) it is useful to define
  * the following macro, so that __sync macros are used: those can be detected
  * by Helgrind (even if they are less efficient) so that no false positive
  * is reported. */
 // #define __ATOMIC_VAR_FORCE_SYNC_MACROS
 
-#if !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && defined(__ATOMIC_RELAXED) && !defined(__sun) && (!defined(__clang__) || !defined(__APPLE__) || __apple_build_version__ > 4210057)
+/* There will be many false positives if we test Redis with Helgrind, since
+ * Helgrind can't understand we have imposed ordering on the program, so
+ * we use macros in helgrind.h to tell Helgrind inter-thread happens-before
+ * relationship explicitly for avoiding false positives.
+ *
+ * For more details, please see: valgrind/helgrind.h and
+ * https://www.valgrind.org/docs/manual/hg-manual.html#hg-manual.effective-use
+ *
+ * These macros take effect only when 'make helgrind', and you must first
+ * install Valgrind in the default path configuration. */
+#ifdef __ATOMIC_VAR_FORCE_SYNC_MACROS
+#include <valgrind/helgrind.h>
+#else
+#define ANNOTATE_HAPPENS_BEFORE(v) ((void) v)
+#define ANNOTATE_HAPPENS_AFTER(v)  ((void) v)
+#endif
+
+#if !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && defined(__STDC_VERSION__) && \
+    (__STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_ATOMICS__)
+/* Use '_Atomic' keyword if the compiler supports. */
+#undef  redisAtomic
+#define redisAtomic _Atomic
+/* Implementation using _Atomic in C11. */
+
+#include <stdatomic.h>
+#define atomicIncr(var,count) atomic_fetch_add_explicit(&var,(count),memory_order_relaxed)
+#define atomicGetIncr(var,oldvalue_var,count) do { \
+    oldvalue_var = atomic_fetch_add_explicit(&var,(count),memory_order_relaxed); \
+} while(0)
+#define atomicDecr(var,count) atomic_fetch_sub_explicit(&var,(count),memory_order_relaxed)
+#define atomicGet(var,dstvar) do { \
+    dstvar = atomic_load_explicit(&var,memory_order_relaxed); \
+} while(0)
+#define atomicSet(var,value) atomic_store_explicit(&var,value,memory_order_relaxed)
+#define atomicGetWithSync(var,dstvar) do { \
+    dstvar = atomic_load_explicit(&var,memory_order_seq_cst); \
+} while(0)
+#define atomicSetWithSync(var,value) \
+    atomic_store_explicit(&var,value,memory_order_seq_cst)
+#define REDIS_ATOMIC_API "c11-builtin"
+
+#elif !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && !defined(__sun) && \
+    (!defined(__clang__) || !defined(__APPLE__) || __apple_build_version__ > 4210057) && \
+    defined(__ATOMIC_RELAXED) && defined(__ATOMIC_SEQ_CST)
 /* Implementation using __atomic macros. */
 
 #define atomicIncr(var,count) __atomic_add_fetch(&var,(count),__ATOMIC_RELAXED)
@@ -80,6 +119,11 @@
     dstvar = __atomic_load_n(&var,__ATOMIC_RELAXED); \
 } while(0)
 #define atomicSet(var,value) __atomic_store_n(&var,value,__ATOMIC_RELAXED)
+#define atomicGetWithSync(var,dstvar) do { \
+    dstvar = __atomic_load_n(&var,__ATOMIC_SEQ_CST); \
+} while(0)
+#define atomicSetWithSync(var,value) \
+    __atomic_store_n(&var,value,__ATOMIC_SEQ_CST)
 #define REDIS_ATOMIC_API "atomic-builtin"
 
 #elif defined(HAVE_ATOMIC)
@@ -96,38 +140,19 @@
 #define atomicSet(var,value) do { \
     while(!__sync_bool_compare_and_swap(&var,var,value)); \
 } while(0)
+/* Actually the builtin issues a full memory barrier by default. */
+#define atomicGetWithSync(var,dstvar) { \
+    dstvar = __sync_sub_and_fetch(&var,0,__sync_synchronize); \
+    ANNOTATE_HAPPENS_AFTER(&var); \
+} while(0)
+#define atomicSetWithSync(var,value) do { \
+    ANNOTATE_HAPPENS_BEFORE(&var);  \
+    while(!__sync_bool_compare_and_swap(&var,var,value,__sync_synchronize)); \
+} while(0)
 #define REDIS_ATOMIC_API "sync-builtin"
 
 #else
-/* Implementation using pthread mutex. */
-
-#define atomicIncr(var,count) do { \
-    pthread_mutex_lock(&var ## _mutex); \
-    var += (count); \
-    pthread_mutex_unlock(&var ## _mutex); \
-} while(0)
-#define atomicGetIncr(var,oldvalue_var,count) do { \
-    pthread_mutex_lock(&var ## _mutex); \
-    oldvalue_var = var; \
-    var += (count); \
-    pthread_mutex_unlock(&var ## _mutex); \
-} while(0)
-#define atomicDecr(var,count) do { \
-    pthread_mutex_lock(&var ## _mutex); \
-    var -= (count); \
-    pthread_mutex_unlock(&var ## _mutex); \
-} while(0)
-#define atomicGet(var,dstvar) do { \
-    pthread_mutex_lock(&var ## _mutex); \
-    dstvar = var; \
-    pthread_mutex_unlock(&var ## _mutex); \
-} while(0)
-#define atomicSet(var,value) do { \
-    pthread_mutex_lock(&var ## _mutex); \
-    var = value; \
-    pthread_mutex_unlock(&var ## _mutex); \
-} while(0)
-#define REDIS_ATOMIC_API "pthread-mutex"
+#error "Unable to determine atomic operations for your platform"
 
 #endif
 #endif /* __ATOMIC_VAR_H */
diff --git a/src/evict.c b/src/evict.c
index 258e1a25e..0e8f818ff 100644
--- a/src/evict.c
+++ b/src/evict.c
@@ -79,7 +79,7 @@ unsigned int getLRUClock(void) {
 unsigned int LRU_CLOCK(void) {
     unsigned int lruclock;
     if (1000/server.hz <= LRU_CLOCK_RESOLUTION) {
-        lruclock = server.lruclock;
+        atomicGet(server.lruclock,lruclock);
     } else {
         lruclock = getLRUClock();
     }
diff --git a/src/lazyfree.c b/src/lazyfree.c
index 821dc50df..b0fc26fcf 100644
--- a/src/lazyfree.c
+++ b/src/lazyfree.c
@@ -3,8 +3,7 @@
 #include "atomicvar.h"
 #include "cluster.h"
 
-static size_t lazyfree_objects = 0;
-pthread_mutex_t lazyfree_objects_mutex = PTHREAD_MUTEX_INITIALIZER;
+static redisAtomic size_t lazyfree_objects = 0;
 
 /* Return the number of currently pending objects to free. */
 size_t lazyfreeGetPendingObjectsCount(void) {
diff --git a/src/module.c b/src/module.c
index 315161ed9..a501a4830 100644
--- a/src/module.c
+++ b/src/module.c
@@ -357,11 +357,6 @@ unsigned long long ModulesInHooks = 0; /* Total number of modules in hooks
 
 /* Data structures related to the redis module users */
 
-/* This callback type is called by moduleNotifyUserChanged() every time
- * a user authenticated via the module API is associated with a different
- * user or gets disconnected. */
-typedef void (*RedisModuleUserChangedFunc) (uint64_t client_id, void *privdata);
-
 /* This is the object returned by RM_CreateModuleUser(). The module API is
  * able to create users, set ACLs to such users, and later authenticate
  * clients using such newly created users. */
diff --git a/src/networking.c b/src/networking.c
index 6becdc288..3fa298783 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -103,7 +103,8 @@ client *createClient(connection *conn) {
     }
 
     selectDb(c,0);
-    uint64_t client_id = ++server.next_client_id;
+    uint64_t client_id;
+    atomicGetIncr(server.next_client_id, client_id, 1);
     c->id = client_id;
     c->resp = 2;
     c->conn = conn;
@@ -1314,7 +1315,7 @@ client *lookupClientByID(uint64_t id) {
  * thread safe. */
 int writeToClient(client *c, int handler_installed) {
     /* Update total number of writes on server */
-    server.stat_total_writes_processed++;
+    atomicIncr(server.stat_total_writes_processed, 1);
 
     ssize_t nwritten = 0, totwritten = 0;
     size_t objlen;
@@ -1376,7 +1377,7 @@ int writeToClient(client *c, int handler_installed) {
              zmalloc_used_memory() < server.maxmemory) &&
             !(c->flags & CLIENT_SLAVE)) break;
     }
-    server.stat_net_output_bytes += totwritten;
+    atomicIncr(server.stat_net_output_bytes, totwritten);
     if (nwritten == -1) {
         if (connGetState(c->conn) == CONN_STATE_CONNECTED) {
             nwritten = 0;
@@ -1933,7 +1934,7 @@ void readQueryFromClient(connection *conn) {
     if (postponeClientRead(c)) return;
 
     /* Update total number of reads on server */
-    server.stat_total_reads_processed++;
+    atomicIncr(server.stat_total_reads_processed, 1);
 
     readlen = PROTO_IOBUF_LEN;
     /* If this is a multi bulk request, and we are processing a bulk reply
@@ -1979,7 +1980,7 @@ void readQueryFromClient(connection *conn) {
     sdsIncrLen(c->querybuf,nread);
     c->lastinteraction = server.unixtime;
     if (c->flags & CLIENT_MASTER) c->read_reploff += nread;
-    server.stat_net_input_bytes += nread;
+    atomicIncr(server.stat_net_input_bytes, nread);
     if (sdslen(c->querybuf) > server.client_max_querybuf_len) {
         sds ci = catClientInfoString(sdsempty(),c), bytes = sdsempty();
 
@@ -2938,7 +2939,7 @@ int tio_debug = 0;
 
 pthread_t io_threads[IO_THREADS_MAX_NUM];
 pthread_mutex_t io_threads_mutex[IO_THREADS_MAX_NUM];
-_Atomic unsigned long io_threads_pending[IO_THREADS_MAX_NUM];
+redisAtomic unsigned long io_threads_pending[IO_THREADS_MAX_NUM];
 int io_threads_op;      /* IO_THREADS_OP_WRITE or IO_THREADS_OP_READ. */
 
 /* This is the list of clients each thread will serve when threaded I/O is
@@ -2946,6 +2947,16 @@ int io_threads_op;      /* IO_THREADS_OP_WRITE or IO_THREADS_OP_READ. */
  * itself. */
 list *io_threads_list[IO_THREADS_MAX_NUM];
 
+static inline unsigned long getIOPendingCount(int i) {
+    unsigned long count = 0;
+    atomicGetWithSync(io_threads_pending[i], count);
+    return count;
+}
+
+static inline void setIOPendingCount(int i, unsigned long count) {
+    atomicSetWithSync(io_threads_pending[i], count);
+}
+
 void *IOThreadMain(void *myid) {
     /* The ID is the thread number (from 0 to server.iothreads_num-1), and is
      * used by the thread to just manipulate a single sub-array of clients. */
@@ -2959,17 +2970,17 @@ void *IOThreadMain(void *myid) {
     while(1) {
         /* Wait for start */
         for (int j = 0; j < 1000000; j++) {
-            if (io_threads_pending[id] != 0) break;
+            if (getIOPendingCount(id) != 0) break;
         }
 
         /* Give the main thread a chance to stop this thread. */
-        if (io_threads_pending[id] == 0) {
+        if (getIOPendingCount(id) == 0) {
             pthread_mutex_lock(&io_threads_mutex[id]);
             pthread_mutex_unlock(&io_threads_mutex[id]);
             continue;
         }
 
-        serverAssert(io_threads_pending[id] != 0);
+        serverAssert(getIOPendingCount(id) != 0);
 
         if (tio_debug) printf("[%ld] %d to handle\n", id, (int)listLength(io_threads_list[id]));
 
@@ -2989,7 +3000,7 @@ void *IOThreadMain(void *myid) {
             }
         }
         listEmpty(io_threads_list[id]);
-        io_threads_pending[id] = 0;
+        setIOPendingCount(id, 0);
 
         if (tio_debug) printf("[%ld] Done\n", id);
     }
@@ -3018,7 +3029,7 @@ void initThreadedIO(void) {
         /* Things we do only for the additional threads. */
         pthread_t tid;
         pthread_mutex_init(&io_threads_mutex[i],NULL);
-        io_threads_pending[i] = 0;
+        setIOPendingCount(i, 0);
         pthread_mutex_lock(&io_threads_mutex[i]); /* Thread will be stopped. */
         if (pthread_create(&tid,NULL,IOThreadMain,(void*)(long)i) != 0) {
             serverLog(LL_WARNING,"Fatal: Can't initialize IO thread.");
@@ -3124,7 +3135,7 @@ int handleClientsWithPendingWritesUsingThreads(void) {
     io_threads_op = IO_THREADS_OP_WRITE;
     for (int j = 1; j < server.io_threads_num; j++) {
         int count = listLength(io_threads_list[j]);
-        io_threads_pending[j] = count;
+        setIOPendingCount(j, count);
     }
 
     /* Also use the main thread to process a slice of clients. */
@@ -3139,7 +3150,7 @@ int handleClientsWithPendingWritesUsingThreads(void) {
     while(1) {
         unsigned long pending = 0;
         for (int j = 1; j < server.io_threads_num; j++)
-            pending += io_threads_pending[j];
+            pending += getIOPendingCount(j);
         if (pending == 0) break;
     }
     if (tio_debug) printf("I/O WRITE All threads finshed\n");
@@ -3214,7 +3225,7 @@ int handleClientsWithPendingReadsUsingThreads(void) {
     io_threads_op = IO_THREADS_OP_READ;
     for (int j = 1; j < server.io_threads_num; j++) {
         int count = listLength(io_threads_list[j]);
-        io_threads_pending[j] = count;
+        setIOPendingCount(j, count);
     }
 
     /* Also use the main thread to process a slice of clients. */
@@ -3229,7 +3240,7 @@ int handleClientsWithPendingReadsUsingThreads(void) {
     while(1) {
         unsigned long pending = 0;
         for (int j = 1; j < server.io_threads_num; j++)
-            pending += io_threads_pending[j];
+            pending += getIOPendingCount(j);
         if (pending == 0) break;
     }
     if (tio_debug) printf("I/O READ All threads finshed\n");
diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index a221ebdd2..7a7828184 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -76,11 +76,11 @@ static struct config {
     int hostport;
     const char *hostsocket;
     int numclients;
-    int liveclients;
+    redisAtomic int liveclients;
     int requests;
-    int requests_issued;
-    int requests_finished;
-    int previous_requests_finished;
+    redisAtomic int requests_issued;
+    redisAtomic int requests_finished;
+    redisAtomic int previous_requests_finished;
     int last_printed_bytes;
     long long previous_tick;
     int keysize;
@@ -113,18 +113,12 @@ static struct config {
     struct redisConfig *redis_config;
     struct hdr_histogram* latency_histogram;
     struct hdr_histogram* current_sec_latency_histogram;
-    int is_fetching_slots;
-    int is_updating_slots;
-    int slots_last_update;
+    redisAtomic int is_fetching_slots;
+    redisAtomic int is_updating_slots;
+    redisAtomic int slots_last_update;
     int enable_tracking;
-    /* Thread mutexes to be used as fallbacks by atomicvar.h */
-    pthread_mutex_t requests_issued_mutex;
-    pthread_mutex_t requests_finished_mutex;
     pthread_mutex_t liveclients_mutex;
-    pthread_mutex_t is_fetching_slots_mutex;
     pthread_mutex_t is_updating_slots_mutex;
-    pthread_mutex_t updating_slots_mutex;
-    pthread_mutex_t slots_last_update_mutex;
 } config;
 
 typedef struct _client {
@@ -1669,13 +1663,8 @@ int main(int argc, const char **argv) {
             fprintf(stderr, "WARN: could not fetch server CONFIG\n");
     }
     if (config.num_threads > 0) {
-        pthread_mutex_init(&(config.requests_issued_mutex), NULL);
-        pthread_mutex_init(&(config.requests_finished_mutex), NULL);
         pthread_mutex_init(&(config.liveclients_mutex), NULL);
-        pthread_mutex_init(&(config.is_fetching_slots_mutex), NULL);
         pthread_mutex_init(&(config.is_updating_slots_mutex), NULL);
-        pthread_mutex_init(&(config.updating_slots_mutex), NULL);
-        pthread_mutex_init(&(config.slots_last_update_mutex), NULL);
     }
 
     if (config.keepalive == 0) {
diff --git a/src/replication.c b/src/replication.c
index 3f98b1062..be05254e8 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1032,7 +1032,7 @@ void sendBulkToSlave(connection *conn) {
             freeClient(slave);
             return;
         }
-        server.stat_net_output_bytes += nwritten;
+        atomicIncr(server.stat_net_output_bytes, nwritten);
         sdsrange(slave->replpreamble,nwritten,-1);
         if (sdslen(slave->replpreamble) == 0) {
             sdsfree(slave->replpreamble);
@@ -1061,7 +1061,7 @@ void sendBulkToSlave(connection *conn) {
         return;
     }
     slave->repldboff += nwritten;
-    server.stat_net_output_bytes += nwritten;
+    atomicIncr(server.stat_net_output_bytes, nwritten);
     if (slave->repldboff == slave->repldbsize) {
         close(slave->repldbfd);
         slave->repldbfd = -1;
@@ -1102,7 +1102,7 @@ void rdbPipeWriteHandler(struct connection *conn) {
         return;
     } else {
         slave->repldboff += nwritten;
-        server.stat_net_output_bytes += nwritten;
+        atomicIncr(server.stat_net_output_bytes, nwritten);
         if (slave->repldboff < server.rdb_pipe_bufflen)
             return; /* more data to write.. */
     }
@@ -1180,7 +1180,7 @@ void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData,
                 /* Note: when use diskless replication, 'repldboff' is the offset
                  * of 'rdb_pipe_buff' sent rather than the offset of entire RDB. */
                 slave->repldboff = nwritten;
-                server.stat_net_output_bytes += nwritten;
+                atomicIncr(server.stat_net_output_bytes, nwritten);
             }
             /* If we were unable to write all the data to one of the replicas,
              * setup write handler (and disable pipe read handler, below) */
@@ -1558,7 +1558,7 @@ void readSyncBulkPayload(connection *conn) {
             cancelReplicationHandshake(1);
             return;
         }
-        server.stat_net_input_bytes += nread;
+        atomicIncr(server.stat_net_input_bytes, nread);
 
         /* When a mark is used, we want to detect EOF asap in order to avoid
          * writing the EOF mark into the file... */
diff --git a/src/server.c b/src/server.c
index e8c51ee86..31e082510 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1764,7 +1764,8 @@ void databasesCron(void) {
 void updateCachedTime(int update_daylight_info) {
     server.ustime = ustime();
     server.mstime = server.ustime / 1000;
-    server.unixtime = server.mstime / 1000;
+    time_t unixtime = server.mstime / 1000;
+    atomicSet(server.unixtime, unixtime);
 
     /* To get information about daylight saving time, we need to call
      * localtime_r and cache the result. However calling localtime_r in this
@@ -1913,11 +1914,15 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
     }
 
     run_with_period(100) {
+        long long stat_net_input_bytes, stat_net_output_bytes;
+        atomicGet(server.stat_net_input_bytes, stat_net_input_bytes);
+        atomicGet(server.stat_net_output_bytes, stat_net_output_bytes);
+
         trackInstantaneousMetric(STATS_METRIC_COMMAND,server.stat_numcommands);
         trackInstantaneousMetric(STATS_METRIC_NET_INPUT,
-                server.stat_net_input_bytes);
+                stat_net_input_bytes);
         trackInstantaneousMetric(STATS_METRIC_NET_OUTPUT,
-                server.stat_net_output_bytes);
+                stat_net_output_bytes);
     }
 
     /* We have just LRU_BITS bits per object for LRU information.
@@ -1931,7 +1936,8 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
      *
      * Note that you can change the resolution altering the
      * LRU_CLOCK_RESOLUTION define. */
-    server.lruclock = getLRUClock();
+    unsigned int lruclock = getLRUClock();
+    atomicSet(server.lruclock,lruclock);
 
     cronUpdateMemoryStats();
 
@@ -2443,7 +2449,8 @@ void initServerConfig(void) {
     server.next_client_id = 1; /* Client IDs, start from 1 .*/
     server.loading_process_events_interval_bytes = (1024*1024*2);
 
-    server.lruclock = getLRUClock();
+    unsigned int lruclock = getLRUClock();
+    atomicSet(server.lruclock,lruclock);
     resetServerSaveParams();
 
     appendServerSaveParams(60*60,1);  /* save after 1 hour and 1 change */
@@ -2846,9 +2853,9 @@ void resetServerStats(void) {
     server.stat_sync_partial_ok = 0;
     server.stat_sync_partial_err = 0;
     server.stat_io_reads_processed = 0;
-    server.stat_total_reads_processed = 0;
+    atomicSet(server.stat_total_reads_processed, 0);
     server.stat_io_writes_processed = 0;
-    server.stat_total_writes_processed = 0;
+    atomicSet(server.stat_total_writes_processed, 0);
     for (j = 0; j < STATS_METRIC_COUNT; j++) {
         server.inst_metric[j].idx = 0;
         server.inst_metric[j].last_sample_time = mstime();
@@ -2856,8 +2863,8 @@ void resetServerStats(void) {
         memset(server.inst_metric[j].samples,0,
             sizeof(server.inst_metric[j].samples));
     }
-    server.stat_net_input_bytes = 0;
-    server.stat_net_output_bytes = 0;
+    atomicSet(server.stat_net_input_bytes, 0);
+    atomicSet(server.stat_net_output_bytes, 0);
     server.stat_unexpected_error_replies = 0;
     server.aof_delayed_fsync = 0;
     server.blocked_last_cron = 0;
@@ -4186,6 +4193,8 @@ sds genRedisInfoString(const char *section) {
             call_uname = 0;
         }
 
+        unsigned int lruclock;
+        atomicGet(server.lruclock,lruclock);
         info = sdscatfmt(info,
             "# Server\r\n"
             "redis_version:%s\r\n"
@@ -4230,7 +4239,7 @@ sds genRedisInfoString(const char *section) {
             (int64_t)(uptime/(3600*24)),
             server.hz,
             server.config_hz,
-            server.lruclock,
+            lruclock,
             server.executable ? server.executable : "",
             server.configfile ? server.configfile : "",
             server.io_threads_active);
@@ -4472,6 +4481,13 @@ sds genRedisInfoString(const char *section) {
 
     /* Stats */
     if (allsections || defsections || !strcasecmp(section,"stats")) {
+        long long stat_total_reads_processed, stat_total_writes_processed;
+        long long stat_net_input_bytes, stat_net_output_bytes;
+        atomicGet(server.stat_total_reads_processed, stat_total_reads_processed);
+        atomicGet(server.stat_total_writes_processed, stat_total_writes_processed);
+        atomicGet(server.stat_net_input_bytes, stat_net_input_bytes);
+        atomicGet(server.stat_net_output_bytes, stat_net_output_bytes);
+
         if (sections++) info = sdscat(info,"\r\n");
         info = sdscatprintf(info,
             "# Stats\r\n"
@@ -4513,8 +4529,8 @@ sds genRedisInfoString(const char *section) {
             server.stat_numconnections,
             server.stat_numcommands,
             getInstantaneousMetric(STATS_METRIC_COMMAND),
-            server.stat_net_input_bytes,
-            server.stat_net_output_bytes,
+            stat_net_input_bytes,
+            stat_net_output_bytes,
             (float)getInstantaneousMetric(STATS_METRIC_NET_INPUT)/1024,
             (float)getInstantaneousMetric(STATS_METRIC_NET_OUTPUT)/1024,
             server.stat_rejected_conn,
@@ -4541,8 +4557,8 @@ sds genRedisInfoString(const char *section) {
             (unsigned long long) trackingGetTotalItems(),
             (unsigned long long) trackingGetTotalPrefixes(),
             server.stat_unexpected_error_replies,
-            server.stat_total_reads_processed,
-            server.stat_total_writes_processed,
+            stat_total_reads_processed,
+            stat_total_writes_processed,
             server.stat_io_reads_processed,
             server.stat_io_writes_processed);
     }
diff --git a/src/server.h b/src/server.h
index 8cba2f9b5..cac8544b9 100644
--- a/src/server.h
+++ b/src/server.h
@@ -34,6 +34,7 @@
 #include "config.h"
 #include "solarisfixes.h"
 #include "rio.h"
+#include "atomicvar.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -511,8 +512,10 @@ typedef void (*moduleTypeDigestFunc)(struct RedisModuleDigest *digest, void *val
 typedef size_t (*moduleTypeMemUsageFunc)(const void *value);
 typedef void (*moduleTypeFreeFunc)(void *value);
 
-/* A callback that is called when the client authentication changes. This
- * needs to be exposed since you can't cast a function pointer to (void *) */
+/* This callback type is called by moduleNotifyUserChanged() every time
+ * a user authenticated via the module API is associated with a different
+ * user or gets disconnected. This needs to be exposed since you can't cast
+ * a function pointer to (void *). */
 typedef void (*RedisModuleUserChangedFunc) (uint64_t client_id, void *privdata);
 
 
@@ -1063,7 +1066,7 @@ struct redisServer {
     dict *commands;             /* Command table */
     dict *orig_commands;        /* Command table before command renaming. */
     aeEventLoop *el;
-    _Atomic unsigned int lruclock; /* Clock for LRU eviction */
+    redisAtomic unsigned int lruclock; /* Clock for LRU eviction */
     int shutdown_asap;          /* SHUTDOWN needed ASAP */
     int activerehashing;        /* Incremental rehash in serverCron() */
     int active_defrag_running;  /* Active defragmentation running (holds current scan aggressiveness) */
@@ -1111,7 +1114,7 @@ struct redisServer {
     mstime_t clients_pause_end_time; /* Time when we undo clients_paused */
     char neterr[ANET_ERR_LEN];   /* Error buffer for anet.c */
     dict *migrate_cached_sockets;/* MIGRATE cached sockets */
-    _Atomic uint64_t next_client_id; /* Next client unique ID. Incremental. */
+    redisAtomic uint64_t next_client_id; /* Next client unique ID. Incremental. */
     int protected_mode;         /* Don't accept external connections. */
     int gopher_enabled;         /* If true the server will reply to gopher
                                    queries. Will still serve RESP2 queries. */
@@ -1160,8 +1163,8 @@ struct redisServer {
     long long slowlog_log_slower_than; /* SLOWLOG time limit (to get logged) */
     unsigned long slowlog_max_len;     /* SLOWLOG max number of items logged */
     struct malloc_stats cron_malloc_stats; /* sampled in serverCron(). */
-    _Atomic long long stat_net_input_bytes; /* Bytes read from network. */
-    _Atomic long long stat_net_output_bytes; /* Bytes written to network. */
+    redisAtomic long long stat_net_input_bytes; /* Bytes read from network. */
+    redisAtomic long long stat_net_output_bytes; /* Bytes written to network. */
     size_t stat_rdb_cow_bytes;      /* Copy on write bytes during RDB saving. */
     size_t stat_aof_cow_bytes;      /* Copy on write bytes during AOF rewrite. */
     size_t stat_module_cow_bytes;   /* Copy on write bytes during module fork. */
@@ -1169,8 +1172,8 @@ struct redisServer {
     long long stat_unexpected_error_replies; /* Number of unexpected (aof-loading, replica to master, etc.) error replies */
     long long stat_io_reads_processed; /* Number of read events processed by IO / Main threads */
     long long stat_io_writes_processed; /* Number of write events processed by IO / Main threads */
-    _Atomic long long stat_total_reads_processed; /* Total number of read events processed */
-    _Atomic long long stat_total_writes_processed; /* Total number of write events processed */
+    redisAtomic long long stat_total_reads_processed; /* Total number of read events processed */
+    redisAtomic long long stat_total_writes_processed; /* Total number of write events processed */
     /* The following two are used to track instantaneous metrics, like
      * number of operations per second, network traffic. */
     struct {
@@ -1193,7 +1196,7 @@ struct redisServer {
     int active_defrag_cycle_min;       /* minimal effort for defrag in CPU percentage */
     int active_defrag_cycle_max;       /* maximal effort for defrag in CPU percentage */
     unsigned long active_defrag_max_scan_fields; /* maximum number of fields of set/hash/zset/list to process from within the main dict scan */
-    _Atomic size_t client_max_querybuf_len; /* Limit for client query buffer length */
+    size_t client_max_querybuf_len; /* Limit for client query buffer length */
     int dbnum;                      /* Total number of configured DBs */
     int supervised;                 /* 1 if supervised, 0 otherwise. */
     int supervised_mode;            /* See SUPERVISED_* */
@@ -1393,7 +1396,7 @@ struct redisServer {
     int list_max_ziplist_size;
     int list_compress_depth;
     /* time cache */
-    _Atomic time_t unixtime;    /* Unix time sampled every cron cycle. */
+    redisAtomic time_t unixtime; /* Unix time sampled every cron cycle. */
     time_t timezone;            /* Cached timezone. As set by tzset(). */
     int daylight_active;        /* Currently in daylight saving time. */
     mstime_t mstime;            /* 'unixtime' in milliseconds. */
diff --git a/src/zmalloc.c b/src/zmalloc.c
index e36319241..cefa39aa8 100644
--- a/src/zmalloc.c
+++ b/src/zmalloc.c
@@ -74,8 +74,7 @@ void zlibc_free(void *ptr) {
 #define update_zmalloc_stat_alloc(__n) atomicIncr(used_memory,(__n))
 #define update_zmalloc_stat_free(__n) atomicDecr(used_memory,(__n))
 
-static size_t used_memory = 0;
-pthread_mutex_t used_memory_mutex = PTHREAD_MUTEX_INITIALIZER;
+static redisAtomic size_t used_memory = 0;
 
 static void zmalloc_default_oom(size_t size) {
     fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n",

From b002d2b4f1415f4db805081bc8f5b85d00f30e33 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuan21@baidu.com>
Date: Thu, 17 Sep 2020 23:20:10 +0800
Subject: [PATCH 0664/1098] Remove tmp rdb file in background thread (#7762)

We're already using bg_unlink in several places to delete the rdb file in the background,
and avoid paying the cost of the deletion from our main thread.
This commit uses bg_unlink to remove the temporary rdb file in the background too.

However, in case we delete that rdb file just before exiting, we don't actually wait for the
background thread or the main thread to delete it, and just let the OS clean up after us.
i.e. we open the file, unlink it and exit with the fd still open.

Furthermore, rdbRemoveTempFile can be called from a thread and was using snprintf which is
not async-signal-safe, we now use ll2string instead.
---
 src/rdb.c               | 29 +++++++++++++++++++-----
 src/rdb.h               |  2 +-
 src/server.c            |  6 ++++-
 src/server.h            |  1 +
 tests/test_helper.tcl   |  1 +
 tests/unit/shutdown.tcl | 49 +++++++++++++++++++++++++++++++++++++++++
 6 files changed, 81 insertions(+), 7 deletions(-)
 create mode 100644 tests/unit/shutdown.tcl

diff --git a/src/rdb.c b/src/rdb.c
index 37e09ef96..d0a3c3210 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -34,6 +34,7 @@
 #include "stream.h"
 
 #include <math.h>
+#include <fcntl.h>
 #include <sys/types.h>
 #include <sys/time.h>
 #include <sys/resource.h>
@@ -1413,11 +1414,29 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
     return C_OK; /* unreached */
 }
 
-void rdbRemoveTempFile(pid_t childpid) {
+/* Note that we may call this function in signal handle 'sigShutdownHandler',
+ * so we need guarantee all functions we call are async-signal-safe.
+ * If  we call this function from signal handle, we won't call bg_unlik that
+ * is not async-signal-safe. */
+void rdbRemoveTempFile(pid_t childpid, int from_signal) {
     char tmpfile[256];
+    char pid[32];
 
-    snprintf(tmpfile,sizeof(tmpfile),"temp-%d.rdb", (int) childpid);
-    unlink(tmpfile);
+    /* Generate temp rdb file name using aync-signal safe functions. */
+    int pid_len = ll2string(pid, sizeof(pid), childpid);
+    strcpy(tmpfile, "temp-");
+    strncpy(tmpfile+5, pid, pid_len);
+    strcpy(tmpfile+5+pid_len, ".rdb");
+
+    if (from_signal) {
+        /* bg_unlink is not async-signal-safe, but in this case we don't really
+         * need to close the fd, it'll be released when the process exists. */
+        int fd = open(tmpfile, O_RDONLY|O_NONBLOCK);
+        UNUSED(fd);
+        unlink(tmpfile);
+    } else {
+        bg_unlink(tmpfile);
+    }
 }
 
 /* This function is called by rdbLoadObject() when the code is in RDB-check
@@ -2419,7 +2438,7 @@ void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal) {
         serverLog(LL_WARNING,
             "Background saving terminated by signal %d", bysignal);
         latencyStartMonitor(latency);
-        rdbRemoveTempFile(server.rdb_child_pid);
+        rdbRemoveTempFile(server.rdb_child_pid, 0);
         latencyEndMonitor(latency);
         latencyAddSampleIfNeeded("rdb-unlink-temp-file",latency);
         /* SIGUSR1 is whitelisted, so we have a way to kill a child without
@@ -2488,7 +2507,7 @@ void backgroundSaveDoneHandler(int exitcode, int bysignal) {
  * the cleanup needed. */
 void killRDBChild(void) {
     kill(server.rdb_child_pid,SIGUSR1);
-    rdbRemoveTempFile(server.rdb_child_pid);
+    rdbRemoveTempFile(server.rdb_child_pid, 0);
     closeChildInfoPipe();
     updateDictResizePolicy();
 }
diff --git a/src/rdb.h b/src/rdb.h
index aae682dbc..885cf49c6 100644
--- a/src/rdb.h
+++ b/src/rdb.h
@@ -141,7 +141,7 @@ int rdbLoadObjectType(rio *rdb);
 int rdbLoad(char *filename, rdbSaveInfo *rsi, int rdbflags);
 int rdbSaveBackground(char *filename, rdbSaveInfo *rsi);
 int rdbSaveToSlavesSockets(rdbSaveInfo *rsi);
-void rdbRemoveTempFile(pid_t childpid);
+void rdbRemoveTempFile(pid_t childpid, int from_signal);
 int rdbSave(char *filename, rdbSaveInfo *rsi);
 ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key);
 size_t rdbSavedObjectLen(robj *o, robj *key);
diff --git a/src/server.c b/src/server.c
index 31e082510..1ce20334c 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3878,6 +3878,10 @@ int prepareForShutdown(int flags) {
        overwrite the synchronous saving did by SHUTDOWN. */
     if (server.rdb_child_pid != -1) {
         serverLog(LL_WARNING,"There is a child saving an .rdb. Killing it!");
+        /* Note that, in killRDBChild, we call rdbRemoveTempFile that will
+         * do close fd(in order to unlink file actully) in background thread.
+         * The temp rdb file fd may won't be closed when redis exits quickly,
+         * but OS will close this fd when process exits. */
         killRDBChild();
     }
 
@@ -4934,7 +4938,7 @@ static void sigShutdownHandler(int sig) {
      * on disk. */
     if (server.shutdown_asap && sig == SIGINT) {
         serverLogFromHandler(LL_WARNING, "You insist... exiting now.");
-        rdbRemoveTempFile(getpid());
+        rdbRemoveTempFile(getpid(), 1);
         exit(1); /* Exit with an error since this was not a clean shutdown. */
     } else if (server.loading) {
         serverLogFromHandler(LL_WARNING, "Received shutdown signal during loading, exiting now.");
diff --git a/src/server.h b/src/server.h
index cac8544b9..a6859f554 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1880,6 +1880,7 @@ int writeCommandsDeniedByDiskError(void);
 /* RDB persistence */
 #include "rdb.h"
 void killRDBChild(void);
+int bg_unlink(const char *filename);
 
 /* AOF persistence */
 void flushAppendOnlyFile(int force);
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 7e1c5c88f..b60adb881 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -69,6 +69,7 @@ set ::all_tests {
     unit/tls
     unit/tracking
     unit/oom-score-adj
+    unit/shutdown
 }
 # Index to the next test to run in the ::all_tests list.
 set ::next_test 0
diff --git a/tests/unit/shutdown.tcl b/tests/unit/shutdown.tcl
new file mode 100644
index 000000000..21ea8545d
--- /dev/null
+++ b/tests/unit/shutdown.tcl
@@ -0,0 +1,49 @@
+start_server {tags {"shutdown"}} {
+    test {Temp rdb will be deleted if we use bg_unlink when shutdown} {
+        for {set i 0} {$i < 20} {incr i} {
+            r set $i $i
+        }
+        # It will cost 2s(20 * 100ms) to dump rdb
+        r config set rdb-key-save-delay 100000
+
+        # Child is dumping rdb
+        r bgsave
+        after 100
+        set dir [lindex [r config get dir] 1]
+        set child_pid [get_child_pid 0]
+        set temp_rdb [file join [lindex [r config get dir] 1] temp-${child_pid}.rdb]
+        # Temp rdb must be existed
+        assert {[file exists $temp_rdb]}
+
+        catch {r shutdown nosave}
+        # Make sure the server was killed
+        catch {set rd [redis_deferring_client]} e
+        assert_match {*connection refused*} $e
+
+        # Temp rdb file must be deleted
+        assert {![file exists $temp_rdb]}
+    }
+}
+
+start_server {tags {"shutdown"}} {
+    test {Temp rdb will be deleted in signal handle} {
+        for {set i 0} {$i < 20} {incr i} {
+            r set $i $i
+        }
+        # It will cost 2s(20 * 100ms) to dump rdb
+        r config set rdb-key-save-delay 100000
+        
+        set pid [s process_id]
+        set temp_rdb [file join [lindex [r config get dir] 1] temp-${pid}.rdb]
+
+        exec kill -SIGINT $pid
+        after 100
+        # Temp rdb must be existed
+        assert {[file exists $temp_rdb]}
+
+        # Temp rdb file must be deleted
+        exec kill -SIGINT $pid
+        after 100
+        assert {![file exists $temp_rdb]}
+    }
+}

From eabe3eaec02e293d479c695bc973b8780c989da9 Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Sat, 19 Sep 2020 10:24:40 +0100
Subject: [PATCH 0665/1098] debug.c: NetBSD build warning fix. (#7810)

The symbol base address is a const on this system.
---
 src/debug.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/debug.c b/src/debug.c
index 32e0c373c..153619334 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1647,13 +1647,14 @@ void dumpCodeAroundEIP(void *eip) {
             /* Find the address of the next page, which is our "safety"
              * limit when dumping. Then try to dump just 128 bytes more
              * than EIP if there is room, or stop sooner. */
+            void *base = (void *)info.dli_saddr;
             unsigned long next = ((unsigned long)eip + sz) & ~(sz-1);
             unsigned long end = (unsigned long)eip + 128;
             if (end > next) end = next;
-            len = end - (unsigned long)info.dli_saddr;
+            len = end - (unsigned long)base;
             serverLogHexDump(LL_WARNING, "dump of function",
-                info.dli_saddr ,len);
-            dumpX86Calls(info.dli_saddr,len);
+                base, len);
+            dumpX86Calls(base, len);
         }
     }
 }

From 1b3b75208c243fe9e18f34c4fb5f0233c2dc5a7f Mon Sep 17 00:00:00 2001
From: Daniel Dai <764122422@qq.com>
Date: Sun, 20 Sep 2020 05:06:17 -0400
Subject: [PATCH 0666/1098] fix make warnings in debug.c MacOS (#7805)

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/debug.c  | 6 +++---
 src/server.h | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/debug.c b/src/debug.c
index 153619334..13373b7c7 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1563,7 +1563,7 @@ int memtest_test_linux_anonymous_maps(void) {
     closeDirectLogFiledes(fd);
     return errors;
 }
-#endif
+#endif /* HAVE_PROC_MAPS */
 
 static void killMainThread(void) {
     int err;
@@ -1580,7 +1580,7 @@ static void killMainThread(void) {
  * should be used only when it's critical to stop the threads for some reason.
  * Currently Redis does this only on crash (for instance on SIGSEGV) in order
  * to perform a fast memory check without other threads messing with memory. */
-static void killThreads(void) {
+void killThreads(void) {
     killMainThread();
     bioKillThreads();
     killIOThreads();
@@ -1600,7 +1600,7 @@ void doFastMemoryTest(void) {
                 "Fast memory test PASSED, however your memory can still be broken. Please run a memory test for several hours if possible.\n");
         }
     }
-#endif
+#endif /* HAVE_PROC_MAPS */
 }
 
 /* Scans the (assumed) x86 code starting at addr, for a max of `len`
diff --git a/src/server.h b/src/server.h
index a6859f554..1025b1654 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2488,6 +2488,7 @@ void xorDigest(unsigned char *digest, void *ptr, size_t len);
 int populateCommandTableParseFlags(struct redisCommand *c, char *strflags);
 void debugDelay(int usec);
 void killIOThreads(void);
+void killThreads(void);
 
 /* TLS stuff */
 void tlsInit(void);

From dfe9714c867ba69f9ace30c9a9270ae95a111556 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Sun, 20 Sep 2020 06:36:20 -0400
Subject: [PATCH 0667/1098] Add Swapdb Module Event (#7804)

---
 src/db.c                       |  2 ++
 src/module.c                   | 16 ++++++++++++++++
 src/redismodule.h              | 16 ++++++++++++++++
 tests/modules/hooks.c          | 12 ++++++++++++
 tests/unit/moduleapi/hooks.tcl |  6 ++++++
 5 files changed, 52 insertions(+)

diff --git a/src/db.c b/src/db.c
index 2405cc2ab..668a1193a 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1157,6 +1157,8 @@ void swapdbCommand(client *c) {
         addReplyError(c,"DB index is out of range");
         return;
     } else {
+        RedisModuleSwapDbInfo si = {REDISMODULE_SWAPDBINFO_VERSION,id1,id2};
+        moduleFireServerEvent(REDISMODULE_EVENT_SWAPDB,0,&si);
         server.dirty++;
         addReply(c,shared.ok);
     }
diff --git a/src/module.c b/src/module.c
index a501a4830..c6ea0cbbb 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7174,6 +7174,20 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) {
  *              int32_t progress;  // Approximate progress between 0 and 1024,
  *                                    or -1 if unknown.
  *
+ *      RedisModuleEvent_SwapDB
+ *
+ *          This event is called when a swap db command has been successfully 
+ *          Executed. 
+ *          For this event call currently there is no subevents available.
+ *
+ *          The data pointer can be casted to a RedisModuleSwapDbInfo
+ *          structure with the following fields:
+ *
+ *             int32_t dbnum_first;    // Swap Db first dbnum 
+ *             int32_t dbnum_second;   // Swap Db second dbnum 
+ *
+ *
+ *
  * The function returns REDISMODULE_OK if the module was successfully subscribed
  * for the specified event. If the API is called from a wrong context then
  * REDISMODULE_ERR is returned. */
@@ -7278,6 +7292,8 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) {
                 moduledata = data;
             } else if (eid == REDISMODULE_EVENT_CRON_LOOP) {
                 moduledata = data;
+            } else if (eid == REDISMODULE_EVENT_SWAPDB) {
+                moduledata = data;
             }
 
             ModulesInHooks++;
diff --git a/src/redismodule.h b/src/redismodule.h
index 4a0e5bf15..56011fae0 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -190,6 +190,7 @@ typedef uint64_t RedisModuleTimerID;
 #define REDISMODULE_EVENT_CRON_LOOP 8
 #define REDISMODULE_EVENT_MODULE_CHANGE 9
 #define REDISMODULE_EVENT_LOADING_PROGRESS 10
+#define REDISMODULE_EVENT_SWAPDB 11
 
 typedef struct RedisModuleEvent {
     uint64_t id;        /* REDISMODULE_EVENT_... defines. */
@@ -243,6 +244,10 @@ static const RedisModuleEvent
     RedisModuleEvent_LoadingProgress = {
         REDISMODULE_EVENT_LOADING_PROGRESS,
         1
+    },
+    RedisModuleEvent_SwapDB = {
+        REDISMODULE_EVENT_SWAPDB,
+        1
     };
 
 /* Those are values that are used for the 'subevent' callback argument. */
@@ -374,6 +379,17 @@ typedef struct RedisModuleLoadingProgressInfo {
 
 #define RedisModuleLoadingProgress RedisModuleLoadingProgressV1
 
+#define REDISMODULE_SWAPDBINFO_VERSION 1
+typedef struct RedisModuleSwapDbInfo {
+    uint64_t version;       /* Not used since this structure is never passed
+                               from the module to the core right now. Here
+                               for future compatibility. */
+    int32_t dbnum_first;    /* Swap Db first dbnum */
+    int32_t dbnum_second;   /* Swap Db second dbnum */
+} RedisModuleSwapDbInfoV1;
+
+#define RedisModuleSwapDbInfo RedisModuleSwapDbInfoV1
+
 /* ------------------------- End of common defines ------------------------ */
 
 #ifndef REDISMODULE_CORE
diff --git a/tests/modules/hooks.c b/tests/modules/hooks.c
index 665a20481..54f84aa23 100644
--- a/tests/modules/hooks.c
+++ b/tests/modules/hooks.c
@@ -253,6 +253,16 @@ void moduleChangeCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub,
     LogStringEvent(ctx, keyname, ei->module_name);
 }
 
+void swapDbCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data)
+{
+    REDISMODULE_NOT_USED(e);
+    REDISMODULE_NOT_USED(sub);
+
+    RedisModuleSwapDbInfo *ei = data;
+    LogNumericEvent(ctx, "swapdb-first", ei->dbnum_first);
+    LogNumericEvent(ctx, "swapdb-second", ei->dbnum_second);
+}
+
 /* This function must be present on each Redis module. It is used in order to
  * register the commands into the Redis server. */
 int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
@@ -289,6 +299,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
         RedisModuleEvent_CronLoop, cronLoopCallback);
     RedisModule_SubscribeToServerEvent(ctx,
         RedisModuleEvent_ModuleChange, moduleChangeCallback);
+    RedisModule_SubscribeToServerEvent(ctx,
+        RedisModuleEvent_SwapDB, swapDbCallback);
 
     event_log = RedisModule_CreateDict(ctx);
 
diff --git a/tests/unit/moduleapi/hooks.tcl b/tests/unit/moduleapi/hooks.tcl
index da0307ce6..c4af59bd2 100644
--- a/tests/unit/moduleapi/hooks.tcl
+++ b/tests/unit/moduleapi/hooks.tcl
@@ -147,6 +147,12 @@ tags "modules" {
             set replica_stdout [srv 0 stdout]
         }
 
+        test {Test swapdb hooks} {
+            r swapdb 0 10
+            assert_equal [r hooks.event_last swapdb-first] 0
+            assert_equal [r hooks.event_last swapdb-second] 10
+
+        }
 
         # look into the log file of the server that just exited
         test {Test shutdown hook} {

From 2458e5481434bb9c7b99813a449f6cbdf521c028 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 20 Sep 2020 13:43:28 +0300
Subject: [PATCH 0668/1098] RM_GetContextFlags provides indication that we're
 in a fork child (#7783)

---
 src/aof.c         |  4 ++--
 src/childinfo.c   |  6 +++---
 src/module.c      |  5 +++--
 src/rdb.c         |  8 ++++----
 src/redismodule.h |  2 ++
 src/scripting.c   |  2 +-
 src/server.c      |  8 ++++++--
 src/server.h      | 11 +++++++----
 8 files changed, 28 insertions(+), 18 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index e01a11342..c444c299c 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1603,7 +1603,7 @@ int rewriteAppendOnlyFileBackground(void) {
     if (hasActiveChildProcess()) return C_ERR;
     if (aofCreatePipes() != C_OK) return C_ERR;
     openChildInfoPipe();
-    if ((childpid = redisFork()) == 0) {
+    if ((childpid = redisFork(CHILD_TYPE_AOF)) == 0) {
         char tmpfile[256];
 
         /* Child */
@@ -1611,7 +1611,7 @@ int rewriteAppendOnlyFileBackground(void) {
         redisSetCpuAffinity(server.aof_rewrite_cpulist);
         snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
         if (rewriteAppendOnlyFile(tmpfile) == C_OK) {
-            sendChildCOWInfo(CHILD_INFO_TYPE_AOF, "AOF rewrite");
+            sendChildCOWInfo(CHILD_TYPE_AOF, "AOF rewrite");
             exitFromChild(0);
         } else {
             exitFromChild(1);
diff --git a/src/childinfo.c b/src/childinfo.c
index fa0600552..f95ae9647 100644
--- a/src/childinfo.c
+++ b/src/childinfo.c
@@ -76,11 +76,11 @@ void receiveChildInfo(void) {
     if (read(server.child_info_pipe[0],&server.child_info_data,wlen) == wlen &&
         server.child_info_data.magic == CHILD_INFO_MAGIC)
     {
-        if (server.child_info_data.process_type == CHILD_INFO_TYPE_RDB) {
+        if (server.child_info_data.process_type == CHILD_TYPE_RDB) {
             server.stat_rdb_cow_bytes = server.child_info_data.cow_size;
-        } else if (server.child_info_data.process_type == CHILD_INFO_TYPE_AOF) {
+        } else if (server.child_info_data.process_type == CHILD_TYPE_AOF) {
             server.stat_aof_cow_bytes = server.child_info_data.cow_size;
-        } else if (server.child_info_data.process_type == CHILD_INFO_TYPE_MODULE) {
+        } else if (server.child_info_data.process_type == CHILD_TYPE_MODULE) {
             server.stat_module_cow_bytes = server.child_info_data.cow_size;
         }
     }
diff --git a/src/module.c b/src/module.c
index c6ea0cbbb..17b72d059 100644
--- a/src/module.c
+++ b/src/module.c
@@ -1991,6 +1991,7 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) {
 
     /* Presence of children processes. */
     if (hasActiveChildProcess()) flags |= REDISMODULE_CTX_FLAGS_ACTIVE_CHILD;
+    if (server.in_fork_child) flags |= REDISMODULE_CTX_FLAGS_IS_CHILD;
 
     return flags;
 }
@@ -6899,7 +6900,7 @@ int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) {
     }
 
     openChildInfoPipe();
-    if ((childpid = redisFork()) == 0) {
+    if ((childpid = redisFork(CHILD_TYPE_MODULE)) == 0) {
         /* Child */
         redisSetProcTitle("redis-module-fork");
     } else if (childpid == -1) {
@@ -6919,7 +6920,7 @@ int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) {
  * retcode will be provided to the done handler executed on the parent process.
  */
 int RM_ExitFromChild(int retcode) {
-    sendChildCOWInfo(CHILD_INFO_TYPE_MODULE, "Module fork");
+    sendChildCOWInfo(CHILD_TYPE_MODULE, "Module fork");
     exitFromChild(retcode);
     return REDISMODULE_OK;
 }
diff --git a/src/rdb.c b/src/rdb.c
index d0a3c3210..d2c085d4d 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1385,7 +1385,7 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
     server.lastbgsave_try = time(NULL);
     openChildInfoPipe();
 
-    if ((childpid = redisFork()) == 0) {
+    if ((childpid = redisFork(CHILD_TYPE_RDB)) == 0) {
         int retval;
 
         /* Child */
@@ -1393,7 +1393,7 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
         redisSetCpuAffinity(server.bgsave_cpulist);
         retval = rdbSave(filename,rsi);
         if (retval == C_OK) {
-            sendChildCOWInfo(CHILD_INFO_TYPE_RDB, "RDB");
+            sendChildCOWInfo(CHILD_TYPE_RDB, "RDB");
         }
         exitFromChild((retval == C_OK) ? 0 : 1);
     } else {
@@ -2561,7 +2561,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
 
     /* Create the child process. */
     openChildInfoPipe();
-    if ((childpid = redisFork()) == 0) {
+    if ((childpid = redisFork(CHILD_TYPE_RDB)) == 0) {
         /* Child */
         int retval, dummy;
         rio rdb;
@@ -2576,7 +2576,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
             retval = C_ERR;
 
         if (retval == C_OK) {
-            sendChildCOWInfo(CHILD_INFO_TYPE_RDB, "RDB");
+            sendChildCOWInfo(CHILD_TYPE_RDB, "RDB");
         }
 
         rioFreeFd(&rdb);
diff --git a/src/redismodule.h b/src/redismodule.h
index 56011fae0..c0eedc221 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -112,6 +112,8 @@
 #define REDISMODULE_CTX_FLAGS_ACTIVE_CHILD (1<<18)
 /* The next EXEC will fail due to dirty CAS (touched keys). */
 #define REDISMODULE_CTX_FLAGS_MULTI_DIRTY (1<<19)
+/* Redis is currently running inside background child process. */
+#define REDISMODULE_CTX_FLAGS_IS_CHILD (1<<20)
 
 /* Keyspace changes notification classes. Every class is associated with a
  * character for configuration purposes.
diff --git a/src/scripting.c b/src/scripting.c
index fdccb5db7..c8ff51b91 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -1858,7 +1858,7 @@ void ldbSendLogs(void) {
 int ldbStartSession(client *c) {
     ldb.forked = (c->flags & CLIENT_LUA_DEBUG_SYNC) == 0;
     if (ldb.forked) {
-        pid_t cp = redisFork();
+        pid_t cp = redisFork(CHILD_TYPE_LDB);
         if (cp == -1) {
             addReplyError(c,"Fork() failed: can't run EVAL in debugging mode.");
             return 0;
diff --git a/src/server.c b/src/server.c
index 1ce20334c..3363d476f 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2886,6 +2886,7 @@ void initServer(void) {
     server.aof_state = server.aof_enabled ? AOF_ON : AOF_OFF;
     server.hz = server.config_hz;
     server.pid = getpid();
+    server.in_fork_child = CHILD_TYPE_NONE;
     server.main_thread_id = pthread_self();
     server.current_client = NULL;
     server.fixed_time_expire = 0;
@@ -4989,7 +4990,8 @@ void removeSignalHandlers(void) {
  * accepting writes because of a write error condition. */
 static void sigKillChildHandler(int sig) {
     UNUSED(sig);
-    serverLogFromHandler(LL_WARNING, "Received SIGUSR1 in child, exiting now.");
+    int level = server.in_fork_child == CHILD_TYPE_MODULE? LL_VERBOSE: LL_WARNING;
+    serverLogFromHandler(level, "Received SIGUSR1 in child, exiting now.");
     exitFromChild(SERVER_CHILD_NOERROR_RETVAL);
 }
 
@@ -5015,11 +5017,13 @@ void closeClildUnusedResourceAfterFork() {
         close(server.cluster_config_file_lock_fd);  /* don't care if this fails */
 }
 
-int redisFork() {
+/* purpose is one of CHILD_TYPE_ types */
+int redisFork(int purpose) {
     int childpid;
     long long start = ustime();
     if ((childpid = fork()) == 0) {
         /* Child */
+        server.in_fork_child = purpose;
         setOOMScoreAdj(CONFIG_OOM_BGCHILD);
         setupChildSignalHandlers();
         closeClildUnusedResourceAfterFork();
diff --git a/src/server.h b/src/server.h
index 1025b1654..7d27626a7 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1046,9 +1046,11 @@ struct clusterState;
 #endif
 
 #define CHILD_INFO_MAGIC 0xC17DDA7A12345678LL
-#define CHILD_INFO_TYPE_RDB 0
-#define CHILD_INFO_TYPE_AOF 1
-#define CHILD_INFO_TYPE_MODULE 3
+#define CHILD_TYPE_NONE 0
+#define CHILD_TYPE_RDB 1
+#define CHILD_TYPE_AOF 2
+#define CHILD_TYPE_LDB 3
+#define CHILD_TYPE_MODULE 4
 
 struct redisServer {
     /* General */
@@ -1062,6 +1064,7 @@ struct redisServer {
                                    the actual 'hz' field value if dynamic-hz
                                    is enabled. */
     int hz;                     /* serverCron() calls frequency in hertz */
+    int in_fork_child;          /* indication that this is a fork child */
     redisDb *db;
     dict *commands;             /* Command table */
     dict *orig_commands;        /* Command table before command renaming. */
@@ -1904,7 +1907,7 @@ void sendChildInfo(int process_type);
 void receiveChildInfo(void);
 
 /* Fork helpers */
-int redisFork();
+int redisFork(int type);
 int hasActiveChildProcess();
 void sendChildCOWInfo(int ptype, char *pname);
 

From f11a0c8f3016fd6d62116d606cd68a01ec06f951 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 21 Sep 2020 11:17:48 +0300
Subject: [PATCH 0669/1098] Fix C11 detection in the makefile (#7822)

445a4b6 introudced a makefile script that detects if the toolchain
supports c11, and it looked that it was passing on MacOS and fails on
Ubuntu, looks like Ubuntu's Dash was spawning a background process,
deleted foo.c before gcc tried to compile it.
---
 src/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Makefile b/src/Makefile
index 6414b8c78..715316a15 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -31,7 +31,7 @@ OPT=$(OPTIMIZATION)
 
 # Detect if the compiler supports C11 _Atomic
 C11_ATOMIC := $(shell sh -c 'echo "\#include <stdatomic.h>" > foo.c; \
-	$(CC) -std=c11 -c foo.c -o foo.o &> /dev/null; \
+	$(CC) -std=c11 -c foo.c -o foo.o > /dev/null 2>&1; \
 	if [ -f foo.o ]; then echo "yes"; rm foo.o; fi; rm foo.c')
 ifeq ($(C11_ATOMIC),yes)
 	STD+=-std=c11

From 647cac5bb469171a5b97eade276335ab9c552edc Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Fri, 18 Sep 2020 16:08:52 +0800
Subject: [PATCH 0670/1098] Make main thread killable so that it can be
 canceled at any time. Refine comment of makeThreadKillable().

This commit can be backported to 5.0, only if we also backport 8b70cb0.

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/bio.c    | 5 +----
 src/server.c | 9 +++++++++
 src/server.h | 1 +
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/bio.c b/src/bio.c
index dd99442ae..a512def0a 100644
--- a/src/bio.c
+++ b/src/bio.c
@@ -168,10 +168,7 @@ void *bioProcessBackgroundJobs(void *arg) {
 
     redisSetCpuAffinity(server.bio_cpulist);
 
-    /* Make the thread killable at any time, so that bioKillThreads()
-     * can work reliably. */
-    pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
-    pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
+    makeThreadKillable();
 
     pthread_mutex_lock(&bio_mutex[type]);
     /* Block SIGALRM so we are sure that only the main thread will
diff --git a/src/server.c b/src/server.c
index 3363d476f..8ae5c64d9 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2870,12 +2870,21 @@ void resetServerStats(void) {
     server.blocked_last_cron = 0;
 }
 
+/* Make the thread killable at any time, so that kill threads functions
+ * can work reliably (default cancelability type is PTHREAD_CANCEL_DEFERRED).
+ * Needed for pthread_cancel used by the fast memory test used by the crash report. */
+void makeThreadKillable(void) {
+    pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
+    pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
+}
+
 void initServer(void) {
     int j;
 
     signal(SIGHUP, SIG_IGN);
     signal(SIGPIPE, SIG_IGN);
     setupSignalHandlers();
+    makeThreadKillable();
 
     if (server.syslog_enabled) {
         openlog(server.syslog_ident, LOG_PID | LOG_NDELAY | LOG_NOWAIT,
diff --git a/src/server.h b/src/server.h
index 7d27626a7..c0ca89b8b 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2492,6 +2492,7 @@ int populateCommandTableParseFlags(struct redisCommand *c, char *strflags);
 void debugDelay(int usec);
 void killIOThreads(void);
 void killThreads(void);
+void makeThreadKillable(void);
 
 /* TLS stuff */
 void tlsInit(void);

From c37acb82aa4b1c3317f9bc1b05fc5ab8af23ba00 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Fri, 18 Sep 2020 16:10:54 +0800
Subject: [PATCH 0671/1098] Make IO threads killable so that they can be
 canceled at any time.

This commit can be cherry picked to 6.0 only if we also cherry pick f866023.
---
 src/networking.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/networking.c b/src/networking.c
index 3fa298783..d3b0cd241 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2966,6 +2966,7 @@ void *IOThreadMain(void *myid) {
     snprintf(thdname, sizeof(thdname), "io_thd_%ld", id);
     redis_set_thread_title(thdname);
     redisSetCpuAffinity(server.server_cpulist);
+    makeThreadKillable();
 
     while(1) {
         /* Wait for start */

From d3faed875d4d50dca28be814e8898b4f8674922b Mon Sep 17 00:00:00 2001
From: YoongHM <yoonghm@users.noreply.github.com>
Date: Mon, 21 Sep 2020 21:57:15 +0800
Subject: [PATCH 0672/1098] Fix warning from jemalloc configure script (#7790)

jemalloc configure shows this:
    configure: WARNING: unrecognized options: --enable-cc-silence

The changelog of jemalloc 4.0 has:
  - Replace --enable-cc-silence with --disable-cc-silence to suppress spurious
    warnings by default.
---
 deps/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deps/Makefile b/deps/Makefile
index 0815d15de..b90bbd2b7 100644
--- a/deps/Makefile
+++ b/deps/Makefile
@@ -88,7 +88,7 @@ JEMALLOC_LDFLAGS= $(LDFLAGS)
 
 jemalloc: .make-prerequisites
 	@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
-	cd jemalloc && ./configure --with-version=5.1.0-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ --enable-cc-silence CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)"
+	cd jemalloc && ./configure --with-version=5.1.0-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)"
 	cd jemalloc && $(MAKE) CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" lib/libjemalloc.a
 
 .PHONY: jemalloc

From 9216b96b4117009c75214bcc8aca048d472d2d14 Mon Sep 17 00:00:00 2001
From: YoongHM <yoonghm@users.noreply.github.com>
Date: Mon, 21 Sep 2020 22:04:34 +0800
Subject: [PATCH 0673/1098] Fix compilation warning in jemalloc's
 malloc_vsnprintf (#7789)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Change `val` to `unsigned char` before being tested.
The fix is identical to the one that's been made in upstream jemalloc.

warning is:
src/malloc_io.c: In function ‘malloc_vsnprintf’:
src/malloc_io.c:369:2: warning: case label value exceeds maximum value for type
  369 |  case '?' | 0x80:      \
      |  ^~~~
src/malloc_io.c:581:5: note: in expansion of macro ‘GET_ARG_NUMERIC’
  581 |     GET_ARG_NUMERIC(val, 'p');
      |     ^~~~~~~~~~~~~~~
---
 deps/jemalloc/src/malloc_io.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deps/jemalloc/src/malloc_io.c b/deps/jemalloc/src/malloc_io.c
index 7bdc13f95..dd8826511 100644
--- a/deps/jemalloc/src/malloc_io.c
+++ b/deps/jemalloc/src/malloc_io.c
@@ -362,7 +362,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
 	}								\
 } while (0)
 #define GET_ARG_NUMERIC(val, len) do {					\
-	switch (len) {							\
+	switch ((unsigned char)len) {					\
 	case '?':							\
 		val = va_arg(ap, int);					\
 		break;							\

From 23b50bccccf4efac4a1672a332e470a82b7bd514 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Tue, 22 Sep 2020 02:05:47 -0400
Subject: [PATCH 0674/1098] refactor rewriteStreamObject code for adding
 missing streamIteratorStop call (#7829)

This commit adds streamIteratorStop call in rewriteStreamObject function in some of the return statement. Although currently this will not cause memory leak since stream id is only 16 bytes long.
---
 src/aof.c | 54 ++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 36 insertions(+), 18 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index c444c299c..f1490a39c 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1201,16 +1201,24 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) {
              * the ID, the second is an array of field-value pairs. */
 
             /* Emit the XADD <key> <id> ...fields... command. */
-            if (rioWriteBulkCount(r,'*',3+numfields*2) == 0) return 0;
-            if (rioWriteBulkString(r,"XADD",4) == 0) return 0;
-            if (rioWriteBulkObject(r,key) == 0) return 0;
-            if (rioWriteBulkStreamID(r,&id) == 0) return 0;
+            if (!rioWriteBulkCount(r,'*',3+numfields*2) || 
+                !rioWriteBulkString(r,"XADD",4) ||
+                !rioWriteBulkObject(r,key) ||
+                !rioWriteBulkStreamID(r,&id)) 
+            {
+                streamIteratorStop(&si);
+                return 0;
+            }
             while(numfields--) {
                 unsigned char *field, *value;
                 int64_t field_len, value_len;
                 streamIteratorGetField(&si,&field,&value,&field_len,&value_len);
-                if (rioWriteBulkString(r,(char*)field,field_len) == 0) return 0;
-                if (rioWriteBulkString(r,(char*)value,value_len) == 0) return 0;
+                if (!rioWriteBulkString(r,(char*)field,field_len) ||
+                    !rioWriteBulkString(r,(char*)value,value_len)) 
+                {
+                    streamIteratorStop(&si);
+                    return 0;                  
+                }
             }
         }
     } else {
@@ -1218,22 +1226,30 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) {
          * the key we are serializing is an empty string, which is possible
          * for the Stream type. */
         id.ms = 0; id.seq = 1; 
-        if (rioWriteBulkCount(r,'*',7) == 0) return 0;
-        if (rioWriteBulkString(r,"XADD",4) == 0) return 0;
-        if (rioWriteBulkObject(r,key) == 0) return 0;
-        if (rioWriteBulkString(r,"MAXLEN",6) == 0) return 0;
-        if (rioWriteBulkString(r,"0",1) == 0) return 0;
-        if (rioWriteBulkStreamID(r,&id) == 0) return 0;
-        if (rioWriteBulkString(r,"x",1) == 0) return 0;
-        if (rioWriteBulkString(r,"y",1) == 0) return 0;
+        if (!rioWriteBulkCount(r,'*',7) ||
+            !rioWriteBulkString(r,"XADD",4) ||
+            !rioWriteBulkObject(r,key) ||
+            !rioWriteBulkString(r,"MAXLEN",6) ||
+            !rioWriteBulkString(r,"0",1) ||
+            !rioWriteBulkStreamID(r,&id) ||
+            !rioWriteBulkString(r,"x",1) ||
+            !rioWriteBulkString(r,"y",1))
+        {
+            streamIteratorStop(&si);
+            return 0;     
+        }
     }
 
     /* Append XSETID after XADD, make sure lastid is correct,
      * in case of XDEL lastid. */
-    if (rioWriteBulkCount(r,'*',3) == 0) return 0;
-    if (rioWriteBulkString(r,"XSETID",6) == 0) return 0;
-    if (rioWriteBulkObject(r,key) == 0) return 0;
-    if (rioWriteBulkStreamID(r,&s->last_id) == 0) return 0;
+    if (!rioWriteBulkCount(r,'*',3) ||
+        !rioWriteBulkString(r,"XSETID",6) ||
+        !rioWriteBulkObject(r,key) ||
+        !rioWriteBulkStreamID(r,&s->last_id)) 
+    {
+        streamIteratorStop(&si);
+        return 0; 
+    }
 
 
     /* Create all the stream consumer groups. */
@@ -1252,6 +1268,7 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) {
                 !rioWriteBulkStreamID(r,&group->last_id)) 
             {
                 raxStop(&ri);
+                streamIteratorStop(&si);
                 return 0;
             }
 
@@ -1277,6 +1294,7 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) {
                         raxStop(&ri_pel);
                         raxStop(&ri_cons);
                         raxStop(&ri);
+                        streamIteratorStop(&si);
                         return 0;
                     }
                 }

From 1bb5794a1fa1527ea9ce181269f2c8c1714fc045 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuan21@baidu.com>
Date: Tue, 22 Sep 2020 14:47:58 +0800
Subject: [PATCH 0675/1098] Kill disk-based fork child when all replicas drop
 and 'save' is not enabled (#7819)

When all replicas waiting for a bgsave get disconnected (possibly due to output buffer limit),
It may be good to kill the bgsave child. in diskless replication it already happens, but in
disk-based, the child may still serve some purpose (for persistence).

By killing the child, we prevent it from eating COW memory in vain, and we also allow a new child fork sooner for the next full synchronization or bgsave.
We do that only if rdb persistence wasn't enabled in the configuration.

Btw, now, rdbRemoveTempFile in killRDBChild won't block server, so we can killRDBChild safely.
---
 src/networking.c                  | 34 +++++++++++++++++
 tests/integration/replication.tcl | 61 +++++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index d3b0cd241..7ac8ed1c4 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1063,6 +1063,25 @@ void disconnectSlaves(void) {
     }
 }
 
+/* Check if there is any other slave waiting dumping RDB finished expect me.
+ * This function is useful to judge current dumping RDB can be used for full
+ * synchronization or not. */
+int anyOtherSlaveWaitRdb(client *except_me) {
+    listIter li;
+    listNode *ln;
+
+    listRewind(server.slaves, &li);
+    while((ln = listNext(&li))) {
+        client *slave = ln->value;
+        if (slave != except_me &&
+            slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END)
+        {
+            return 1;
+        }
+    }
+    return 0;
+}
+
 /* Remove the specified client from global lists where the client could
  * be referenced, not including the Pub/Sub channels.
  * This is used by freeClient() and replicationCacheMaster(). */
@@ -1213,6 +1232,21 @@ void freeClient(client *c) {
     /* Master/slave cleanup Case 1:
      * we lost the connection with a slave. */
     if (c->flags & CLIENT_SLAVE) {
+        /* If there is no any other slave waiting dumping RDB finished, the
+         * current child process need not continue to dump RDB, then we kill it.
+         * So child process won't use more memory, and we also can fork a new
+         * child process asap to dump rdb for next full synchronization or bgsave.
+         * But we also need to check if users enable 'save' RDB, if enable, we
+         * should not remove directly since that means RDB is important for users
+         * to keep data safe and we may delay configured 'save' for full sync. */
+        if (server.saveparamslen == 0 &&
+            c->replstate == SLAVE_STATE_WAIT_BGSAVE_END &&
+            server.rdb_child_pid != -1 &&
+            server.rdb_child_type == RDB_CHILD_TYPE_DISK &&
+            anyOtherSlaveWaitRdb(c) == 0)
+        {
+            killRDBChild();
+        }
         if (c->replstate == SLAVE_STATE_SEND_BULK) {
             if (c->repldbfd != -1) close(c->repldbfd);
             if (c->replpreamble) sdsfree(c->replpreamble);
diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl
index 5438b0eba..d2c8b8e56 100644
--- a/tests/integration/replication.tcl
+++ b/tests/integration/replication.tcl
@@ -752,3 +752,64 @@ test {replicaof right after disconnection} {
         }
     }
 }
+
+test {Kill rdb child process if its dumping RDB is not useful} {
+    start_server {tags {"repl"}} {
+        set slave1 [srv 0 client]
+        start_server {} {
+            set slave2 [srv 0 client]
+            start_server {} {
+                set master [srv 0 client]
+                set master_host [srv 0 host]
+                set master_port [srv 0 port]
+                for {set i 0} {$i < 10} {incr i} {
+                    $master set $i $i
+                }
+                # Generating RDB will cost 10s(10 * 1s)
+                $master config set rdb-key-save-delay 1000000
+                $master config set repl-diskless-sync no
+                $master config set save ""
+
+                $slave1 slaveof $master_host $master_port
+                $slave2 slaveof $master_host $master_port
+
+                # Wait for starting child
+                wait_for_condition 50 100 {
+                    [s 0 rdb_bgsave_in_progress] == 1
+                } else {
+                    fail "rdb child didn't start"
+                }
+
+                # Slave1 disconnect with master
+                $slave1 slaveof no one
+                # Shouldn't kill child since another slave wait for rdb
+                after 100
+                assert {[s 0 rdb_bgsave_in_progress] == 1}
+
+                # Slave2 disconnect with master
+                $slave2 slaveof no one
+                # Should kill child
+                wait_for_condition 20 10 {
+                    [s 0 rdb_bgsave_in_progress] eq 0
+                } else {
+                    fail "can't kill rdb child"
+                }
+
+                # If have save parameters, won't kill child
+                $master config set save "900 1"
+                $slave1 slaveof $master_host $master_port
+                $slave2 slaveof $master_host $master_port
+                wait_for_condition 50 100 {
+                    [s 0 rdb_bgsave_in_progress] == 1
+                } else {
+                    fail "rdb child didn't start"
+                }
+                $slave1 slaveof no one
+                $slave2 slaveof no one
+                after 200
+                assert {[s 0 rdb_bgsave_in_progress] == 1}
+                catch {$master shutdown nosave}
+            }
+        }
+    }
+}

From 63a05dde462c1be4bd74c32630eca6e794ae440a Mon Sep 17 00:00:00 2001
From: Ariel Shtul <ashtul@gmail.com>
Date: Tue, 22 Sep 2020 10:18:07 +0300
Subject: [PATCH 0676/1098] Fix redis-check-rdb support for modules aux data
 (#7826)

redis-check-rdb was unable to parse rdb files containing module aux data.

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/rdb.c             |  2 ++
 src/rdb.h             |  1 +
 src/redis-check-rdb.c | 19 ++++++++++++++++++-
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/rdb.c b/src/rdb.c
index d2c085d4d..00cc184af 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1146,6 +1146,8 @@ ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt) {
     /* Save a module-specific aux value. */
     RedisModuleIO io;
     int retval = rdbSaveType(rdb, RDB_OPCODE_MODULE_AUX);
+    if (retval == -1) return -1;
+    io.bytes += retval;
 
     /* Write the "module" identifier as prefix, so that we'll be able
      * to call the right module during loading. */
diff --git a/src/rdb.h b/src/rdb.h
index 885cf49c6..f22fbecd1 100644
--- a/src/rdb.h
+++ b/src/rdb.h
@@ -149,6 +149,7 @@ robj *rdbLoadObject(int type, rio *rdb, sds key);
 void backgroundSaveDoneHandler(int exitcode, int bysignal);
 int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime);
 ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt);
+robj *rdbLoadCheckModuleValue(rio *rdb, char *modulename);
 robj *rdbLoadStringObject(rio *rdb);
 ssize_t rdbSaveStringObject(rio *rdb, robj *obj);
 ssize_t rdbSaveRawString(rio *rdb, unsigned char *s, size_t len);
diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c
index 592feaf42..a9d110aa8 100644
--- a/src/redis-check-rdb.c
+++ b/src/redis-check-rdb.c
@@ -58,6 +58,7 @@ struct {
 #define RDB_CHECK_DOING_CHECK_SUM 5
 #define RDB_CHECK_DOING_READ_LEN 6
 #define RDB_CHECK_DOING_READ_AUX 7
+#define RDB_CHECK_DOING_READ_MODULE_AUX 8
 
 char *rdb_check_doing_string[] = {
     "start",
@@ -67,7 +68,8 @@ char *rdb_check_doing_string[] = {
     "read-object-value",
     "check-sum",
     "read-len",
-    "read-aux"
+    "read-aux",
+    "read-module-aux"
 };
 
 char *rdb_type_string[] = {
@@ -272,6 +274,21 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) {
             decrRefCount(auxkey);
             decrRefCount(auxval);
             continue; /* Read type again. */
+        } else if (type == RDB_OPCODE_MODULE_AUX) {
+            /* AUX: Auxiliary data for modules. */
+            uint64_t moduleid, when_opcode, when;
+            rdbstate.doing = RDB_CHECK_DOING_READ_MODULE_AUX;
+            if ((moduleid = rdbLoadLen(&rdb,NULL)) == RDB_LENERR) goto eoferr;
+            if ((when_opcode = rdbLoadLen(&rdb,NULL)) == RDB_LENERR) goto eoferr;
+            if ((when = rdbLoadLen(&rdb,NULL)) == RDB_LENERR) goto eoferr;
+
+            char name[10];
+            moduleTypeNameByID(name,moduleid);
+            rdbCheckInfo("MODULE AUX for: %s", name);
+
+            robj *o = rdbLoadCheckModuleValue(&rdb,name);
+            decrRefCount(o);
+            continue; /* Read type again. */
         } else {
             if (!rdbIsObjectType(type)) {
                 rdbCheckError("Invalid object type: %d", type);

From 1980f639b161f46da2944d60f1c2facaf547dc1a Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 22 Sep 2020 11:38:52 +0300
Subject: [PATCH 0677/1098] Fix occasional hangs on replication reconnection.
 (#7830)

This happens only on diskless replicas when attempting to reconnect after
failing to load an RDB file. It is more likely to occur with larger datasets.

After reconnection is initiated, replicationEmptyDbCallback() may get called
and try to write to an unconnected socket. This triggered another issue where
the connection is put into an error state and the connect handler never gets
called. The problem is a regression introduced by commit c17e597.
---
 src/connection.c  | 14 ++++++++++++--
 src/replication.c |  3 ++-
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/connection.c b/src/connection.c
index 23b44a314..415cbdf78 100644
--- a/src/connection.c
+++ b/src/connection.c
@@ -168,7 +168,12 @@ static int connSocketWrite(connection *conn, const void *data, size_t data_len)
     int ret = write(conn->fd, data, data_len);
     if (ret < 0 && errno != EAGAIN) {
         conn->last_errno = errno;
-        conn->state = CONN_STATE_ERROR;
+
+        /* Don't overwrite the state of a connection that is not already
+         * connected, not to mess with handler callbacks.
+         */
+        if (conn->state == CONN_STATE_CONNECTED)
+            conn->state = CONN_STATE_ERROR;
     }
 
     return ret;
@@ -180,7 +185,12 @@ static int connSocketRead(connection *conn, void *buf, size_t buf_len) {
         conn->state = CONN_STATE_CLOSED;
     } else if (ret < 0 && errno != EAGAIN) {
         conn->last_errno = errno;
-        conn->state = CONN_STATE_ERROR;
+
+        /* Don't overwrite the state of a connection that is not already
+         * connected, not to mess with handler callbacks.
+         */
+        if (conn->state == CONN_STATE_CONNECTED)
+            conn->state = CONN_STATE_ERROR;
     }
 
     return ret;
diff --git a/src/replication.c b/src/replication.c
index be05254e8..445ee1970 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1361,7 +1361,8 @@ void replicationSendNewlineToMaster(void) {
  * the new dataset received by the master. */
 void replicationEmptyDbCallback(void *privdata) {
     UNUSED(privdata);
-    replicationSendNewlineToMaster();
+    if (server.repl_state == REPL_STATE_TRANSFER)
+        replicationSendNewlineToMaster();
 }
 
 /* Once we have a link with the master and the synchronization was

From 26ca04a8a2d46ee243e97cc80535c7c1e28d5c56 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 22 Sep 2020 12:11:19 +0300
Subject: [PATCH 0678/1098] RM_GetContextFlags - document missing flags (#7821)

---
 src/module.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/module.c b/src/module.c
index 17b72d059..02a3b1ddc 100644
--- a/src/module.c
+++ b/src/module.c
@@ -1917,6 +1917,12 @@ int RM_GetSelectedDb(RedisModuleCtx *ctx) {
  *
  *  * REDISMODULE_CTX_FLAGS_ACTIVE_CHILD: There is currently some background
  *                                        process active (RDB, AUX or module).
+ *
+ *  * REDISMODULE_CTX_FLAGS_MULTI_DIRTY: The next EXEC will fail due to dirty
+ *                                       CAS (touched keys).
+ *
+ *  * REDISMODULE_CTX_FLAGS_IS_CHILD: Redis is currently running inside
+ *                                    background child process.
  */
 int RM_GetContextFlags(RedisModuleCtx *ctx) {
 

From b96c3595af817721fc838746b859249bf9cf3aaf Mon Sep 17 00:00:00 2001
From: yixiang <zhouyxamp@gmail.com>
Date: Tue, 22 Sep 2020 17:53:36 +0800
Subject: [PATCH 0679/1098] Fix connGetSocketError usage (#7811)

---
 src/connection.c | 5 +++--
 src/tls.c        | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/connection.c b/src/connection.c
index 415cbdf78..83fb84d6d 100644
--- a/src/connection.c
+++ b/src/connection.c
@@ -261,8 +261,9 @@ static void connSocketEventHandler(struct aeEventLoop *el, int fd, void *clientD
     if (conn->state == CONN_STATE_CONNECTING &&
             (mask & AE_WRITABLE) && conn->conn_handler) {
 
-        if (connGetSocketError(conn)) {
-            conn->last_errno = errno;
+        int conn_error = connGetSocketError(conn);
+        if (conn_error) {
+            conn->last_errno = conn_error;
             conn->state = CONN_STATE_ERROR;
         } else {
             conn->state = CONN_STATE_CONNECTED;
diff --git a/src/tls.c b/src/tls.c
index 52887cd23..f55d25c78 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -464,8 +464,9 @@ static void tlsHandleEvent(tls_connection *conn, int mask) {
 
     switch (conn->c.state) {
         case CONN_STATE_CONNECTING:
-            if (connGetSocketError((connection *) conn)) {
-                conn->c.last_errno = errno;
+            int conn_error = connGetSocketError((connection *) conn);
+            if (conn_error) {
+                conn->c.last_errno = conn_error;
                 conn->c.state = CONN_STATE_ERROR;
             } else {
                 if (!(conn->flags & TLS_CONN_FLAG_FD_SET)) {

From 63cd4d4e2044059c44a65e2ce424b9ce1fcdbd39 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Wed, 23 Sep 2020 14:09:48 +0800
Subject: [PATCH 0680/1098] Fix redundancy use of semicolon in do-while macros
 in ziplist.c. (#7832)

this is very dangerous bug, but it looks like it didn't cause any harm.
---
 src/ziplist.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/ziplist.c b/src/ziplist.c
index 6bfe34461..ac5cdafb9 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -390,7 +390,7 @@ unsigned int zipStoreEntryEncoding(unsigned char *p, unsigned char encoding, uns
         (lensize) = 1;                                                         \
         (len) = zipIntSize(encoding);                                          \
     }                                                                          \
-} while(0);
+} while(0)
 
 /* Encode the length of the previous entry and write it to "p". This only
  * uses the larger encoding (required in __ziplistCascadeUpdate). */
@@ -426,7 +426,7 @@ unsigned int zipStorePrevEntryLength(unsigned char *p, unsigned int len) {
     } else {                                                                   \
         (prevlensize) = 5;                                                     \
     }                                                                          \
-} while(0);
+} while(0)
 
 /* Return the length of the previous element, and the number of bytes that
  * are used in order to encode the previous element length.
@@ -444,7 +444,7 @@ unsigned int zipStorePrevEntryLength(unsigned char *p, unsigned int len) {
         memcpy(&(prevlen), ((char*)(ptr)) + 1, 4);                             \
         memrev32ifbe(&prevlen);                                                \
     }                                                                          \
-} while(0);
+} while(0)
 
 /* Given a pointer 'p' to the prevlen info that prefixes an entry, this
  * function returns the difference in number of bytes needed to encode

From c3edaa79413bdf908abb56f088217c01f347ecff Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Wed, 23 Sep 2020 08:00:31 +0100
Subject: [PATCH 0681/1098] Further NetBSD update and build fixes. (#7831)

mainly backtrace and register dump support.
---
 src/Makefile | 12 ++++++++++-
 src/config.h |  4 ++--
 src/debug.c  | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 72 insertions(+), 3 deletions(-)

diff --git a/src/Makefile b/src/Makefile
index 715316a15..f6e5f3e3f 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -131,12 +131,21 @@ ifeq ($(uname_S),OpenBSD)
     	endif
 
 else
+ifeq ($(uname_S),NetBSD)
+	# NetBSD
+	FINAL_LIBS+= -lpthread
+	ifeq ($(USE_BACKTRACE),yes)
+	    FINAL_CFLAGS+= -DUSE_BACKTRACE -I/usr/pkg/include
+	    FINAL_LDFLAGS+= -L/usr/pkg/lib
+	    FINAL_LIBS+= -lexecinfo
+    	endif
+else
 ifeq ($(uname_S),FreeBSD)
 	# FreeBSD
 	FINAL_LIBS+= -lpthread -lexecinfo
 else
 ifeq ($(uname_S),DragonFly)
-	# FreeBSD
+	# DragonFly
 	FINAL_LIBS+= -lpthread -lexecinfo
 else
 ifeq ($(uname_S),OpenBSD)
@@ -158,6 +167,7 @@ endif
 endif
 endif
 endif
+endif
 # Include paths to dependencies
 FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram
 
diff --git a/src/config.h b/src/config.h
index f9ec7e44a..d391508fa 100644
--- a/src/config.h
+++ b/src/config.h
@@ -64,7 +64,7 @@
 
 /* Test for backtrace() */
 #if defined(__APPLE__) || (defined(__linux__) && defined(__GLIBC__)) || \
-    defined(__FreeBSD__) || (defined(__OpenBSD__) && defined(USE_BACKTRACE))\
+    defined(__FreeBSD__) || ((defined(__OpenBSD__) || defined(__NetBSD__)) && defined(USE_BACKTRACE))\
  || defined(__DragonFly__)
 #define HAVE_BACKTRACE 1
 #endif
@@ -236,7 +236,7 @@ void setproctitle(const char *fmt, ...);
 #define redis_set_thread_title(name) pthread_set_name_np(pthread_self(), name)
 #elif defined __NetBSD__
 #include <pthread.h>
-#define redis_set_thread_title(name) pthread_setname_np(pthread_self(), name, NULL)
+#define redis_set_thread_title(name) pthread_setname_np(pthread_self(), "%s", name)
 #else
 #if (defined __APPLE__ && defined(MAC_OS_X_VERSION_10_7))
 int pthread_setname_np(const char *name);
diff --git a/src/debug.c b/src/debug.c
index 13373b7c7..87b4bec71 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -984,6 +984,12 @@ static void *getMcontextEip(ucontext_t *uc) {
     #elif defined(__x86_64__)
     return (void*) uc->sc_rip;
     #endif
+#elif defined(__NetBSD__)
+    #if defined(__i386__)
+    return (void*) uc->uc_mcontext.__gregs[_REG_EIP];
+    #elif defined(__x86_64__)
+    return (void*) uc->uc_mcontext.__gregs[_REG_RIP];
+    #endif
 #elif defined(__DragonFly__)
     return (void*) uc->uc_mcontext.mc_rip;
 #else
@@ -1342,6 +1348,59 @@ void logRegisters(ucontext_t *uc) {
     );
     logStackContent((void**)uc->sc_esp);
     #endif
+#elif defined(__NetBSD__)
+    #if defined(__x86_64__)
+    serverLog(LL_WARNING,
+    "\n"
+    "RAX:%016lx RBX:%016lx\nRCX:%016lx RDX:%016lx\n"
+    "RDI:%016lx RSI:%016lx\nRBP:%016lx RSP:%016lx\n"
+    "R8 :%016lx R9 :%016lx\nR10:%016lx R11:%016lx\n"
+    "R12:%016lx R13:%016lx\nR14:%016lx R15:%016lx\n"
+    "RIP:%016lx EFL:%016lx\nCSGSFS:%016lx",
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_RAX],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_RBX],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_RCX],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_RDX],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_RDI],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_RSI],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_RBP],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_RSP],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_R8],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_R9],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_R10],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_R11],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_R12],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_R13],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_R14],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_R15],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_RIP],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_RFLAGS],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_CS]
+    );
+    logStackContent((void**)uc->uc_mcontext.__gregs[_REG_RSP]);
+    #elif defined(__i386__)
+    serverLog(LL_WARNING,
+    "\n"
+    "EAX:%08lx EBX:%08lx ECX:%08lx EDX:%08lx\n"
+    "EDI:%08lx ESI:%08lx EBP:%08lx ESP:%08lx\n"
+    "SS :%08lx EFL:%08lx EIP:%08lx CS:%08lx\n"
+    "DS :%08lx ES :%08lx FS :%08lx GS:%08lx",
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_EAX],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_EBX],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_EDX],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_EDI],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_ESI],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_EBP],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_ESP],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_SS],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_EFLAGS],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_EIP],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_CS],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_ES],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_FS],
+        (unsigned long) uc->uc_mcontext.__gregs[_REG_GS]
+    );
+    #endif
 #elif defined(__DragonFly__)
     serverLog(LL_WARNING,
     "\n"

From a735bf5c2a8a3d3d18c3a7e9cc728768c2086e89 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 23 Sep 2020 11:30:24 +0300
Subject: [PATCH 0682/1098] fix recently broken TLS build error, and add
 coverage for CI (#7833)

---
 .github/workflows/ci.yml | 3 ++-
 src/tls.c                | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d28d331db..68593717f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -9,7 +9,8 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: make
-      run: make
+      # build with TLS just for compilatoin coverage
+      run: make BUILD_TLS=yes
     - name: test
       run: |
         sudo apt-get install tcl8.5
diff --git a/src/tls.c b/src/tls.c
index f55d25c78..d3173fab0 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -454,7 +454,7 @@ void updateSSLEvent(tls_connection *conn) {
 }
 
 static void tlsHandleEvent(tls_connection *conn, int mask) {
-    int ret;
+    int ret, conn_error;
 
     TLSCONN_DEBUG("tlsEventHandler(): fd=%d, state=%d, mask=%d, r=%d, w=%d, flags=%d",
             fd, conn->c.state, mask, conn->c.read_handler != NULL, conn->c.write_handler != NULL,
@@ -464,7 +464,7 @@ static void tlsHandleEvent(tls_connection *conn, int mask) {
 
     switch (conn->c.state) {
         case CONN_STATE_CONNECTING:
-            int conn_error = connGetSocketError((connection *) conn);
+            conn_error = connGetSocketError((connection *) conn);
             if (conn_error) {
                 conn->c.last_errno = conn_error;
                 conn->c.state = CONN_STATE_ERROR;

From 66a13ccbdfe7cfe982e73a5fafc945d6e64f26f2 Mon Sep 17 00:00:00 2001
From: alexronke-channeladvisor
 <41968397+alexronke-channeladvisor@users.noreply.github.com>
Date: Wed, 23 Sep 2020 14:56:16 -0400
Subject: [PATCH 0683/1098] Add GT and LT options to ZADD for conditional score
 updates (#7818)

Co-authored-by: Alex Ronke <w.alex.ronke@gmail.com>
---
 .gitignore               |  1 +
 src/module.c             |  6 +++++
 src/redismodule.h        |  2 ++
 src/server.h             |  2 ++
 src/t_zset.c             | 31 +++++++++++++++++++--
 tests/unit/type/zset.tcl | 58 ++++++++++++++++++++++++++++++++++++++++
 6 files changed, 98 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 721f1d74e..dd9d4b80e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,3 +37,4 @@ Makefile.dep
 .ccls
 .ccls-cache/*
 compile_commands.json
+redis.code-workspace
diff --git a/src/module.c b/src/module.c
index 02a3b1ddc..77468700e 100644
--- a/src/module.c
+++ b/src/module.c
@@ -2380,6 +2380,8 @@ int RM_ZsetAddFlagsToCoreFlags(int flags) {
     int retflags = 0;
     if (flags & REDISMODULE_ZADD_XX) retflags |= ZADD_XX;
     if (flags & REDISMODULE_ZADD_NX) retflags |= ZADD_NX;
+    if (flags & REDISMODULE_ZADD_GT) retflags |= ZADD_GT;
+    if (flags & REDISMODULE_ZADD_LT) retflags |= ZADD_LT;
     return retflags;
 }
 
@@ -2406,6 +2408,10 @@ int RM_ZsetAddFlagsFromCoreFlags(int flags) {
  *
  *     REDISMODULE_ZADD_XX: Element must already exist. Do nothing otherwise.
  *     REDISMODULE_ZADD_NX: Element must not exist. Do nothing otherwise.
+ *     REDISMODULE_ZADD_GT: If element exists, new score must be greater than the current score. 
+ *                          Do nothing otherwise. Can optionally be combined with XX.
+ *     REDISMODULE_ZADD_LT: If element exists, new score must be less than the current score.
+ *                          Do nothing otherwise. Can optionally be combined with XX.
  *
  * The output flags are:
  *
diff --git a/src/redismodule.h b/src/redismodule.h
index c0eedc221..00ca1f578 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -55,6 +55,8 @@
 #define REDISMODULE_ZADD_ADDED   (1<<2)
 #define REDISMODULE_ZADD_UPDATED (1<<3)
 #define REDISMODULE_ZADD_NOP     (1<<4)
+#define REDISMODULE_ZADD_GT      (1<<5)
+#define REDISMODULE_ZADD_LT      (1<<6)
 
 /* Hash API flags. */
 #define REDISMODULE_HASH_NONE       0
diff --git a/src/server.h b/src/server.h
index c0ca89b8b..80df48881 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1945,6 +1945,8 @@ void addACLLogEntry(client *c, int reason, int keypos, sds username);
 #define ZADD_INCR (1<<0)    /* Increment the score instead of setting it. */
 #define ZADD_NX (1<<1)      /* Don't touch elements not already existing. */
 #define ZADD_XX (1<<2)      /* Only touch elements already existing. */
+#define ZADD_GT (1<<7)      /* Only update existing when new scores are higher. */ 
+#define ZADD_LT (1<<8)      /* Only update existing when new scores are lower. */
 
 /* Output flags. */
 #define ZADD_NOP (1<<3)     /* Operation not performed because of conditionals.*/
diff --git a/src/t_zset.c b/src/t_zset.c
index 4b28c24ef..2ef02ac8a 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -1283,6 +1283,10 @@ int zsetScore(robj *zobj, sds member, double *score) {
  *            assume 0 as previous score.
  * ZADD_NX:   Perform the operation only if the element does not exist.
  * ZADD_XX:   Perform the operation only if the element already exist.
+ * ZADD_GT:   Perform the operation on existing elements only if the new score is 
+ *            greater than the current score.
+ * ZADD_LT:   Perform the operation on existing elements only if the new score is 
+ *            less than the current score.
  *
  * When ZADD_INCR is used, the new score of the element is stored in
  * '*newscore' if 'newscore' is not NULL.
@@ -1317,6 +1321,8 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) {
     int incr = (*flags & ZADD_INCR) != 0;
     int nx = (*flags & ZADD_NX) != 0;
     int xx = (*flags & ZADD_XX) != 0;
+    int gt = (*flags & ZADD_GT) != 0;
+    int lt = (*flags & ZADD_LT) != 0;
     *flags = 0; /* We'll return our response flags. */
     double curscore;
 
@@ -1348,7 +1354,12 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) {
             }
 
             /* Remove and re-insert when score changed. */
-            if (score != curscore) {
+            if (score != curscore &&  
+                /* LT? Only update if score is less than current. */
+                (!lt || score < curscore) &&
+                /* GT? Only update if score is greater than current. */
+                (!gt || score > curscore)) 
+            {
                 zobj->ptr = zzlDelete(zobj->ptr,eptr);
                 zobj->ptr = zzlInsert(zobj->ptr,ele,score);
                 *flags |= ZADD_UPDATED;
@@ -1393,7 +1404,12 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) {
             }
 
             /* Remove and re-insert when score changes. */
-            if (score != curscore) {
+            if (score != curscore &&  
+                /* LT? Only update if score is less than current. */
+                (!lt || score < curscore) &&
+                /* GT? Only update if score is greater than current. */
+                (!gt || score > curscore)) 
+            {
                 znode = zslUpdateScore(zs->zsl,curscore,ele,score);
                 /* Note that we did not removed the original element from
                  * the hash table representing the sorted set, so we just
@@ -1555,6 +1571,8 @@ void zaddGenericCommand(client *c, int flags) {
         else if (!strcasecmp(opt,"xx")) flags |= ZADD_XX;
         else if (!strcasecmp(opt,"ch")) flags |= ZADD_CH;
         else if (!strcasecmp(opt,"incr")) flags |= ZADD_INCR;
+        else if (!strcasecmp(opt,"gt")) flags |= ZADD_GT;
+        else if (!strcasecmp(opt,"lt")) flags |= ZADD_LT;
         else break;
         scoreidx++;
     }
@@ -1564,6 +1582,8 @@ void zaddGenericCommand(client *c, int flags) {
     int nx = (flags & ZADD_NX) != 0;
     int xx = (flags & ZADD_XX) != 0;
     int ch = (flags & ZADD_CH) != 0;
+    int gt = (flags & ZADD_GT) != 0;
+    int lt = (flags & ZADD_LT) != 0;
 
     /* After the options, we expect to have an even number of args, since
      * we expect any number of score-element pairs. */
@@ -1580,6 +1600,13 @@ void zaddGenericCommand(client *c, int flags) {
             "XX and NX options at the same time are not compatible");
         return;
     }
+    
+    if ((gt && nx) || (lt && nx) || (gt && lt)) {
+        addReplyError(c,
+            "GT, LT, and/or NX options at the same time are not compatible");
+        return;
+    }
+    /* Note that XX is compatible with either GT or LT */
 
     if (incr && elements > 1) {
         addReplyError(c,
diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl
index d098840b0..e997bc66d 100644
--- a/tests/unit/type/zset.tcl
+++ b/tests/unit/type/zset.tcl
@@ -78,6 +78,46 @@ start_server {tags {"zset"}} {
             assert {[r zscore ztmp y] == 21}
         }
 
+        test "ZADD GT updates existing elements when new scores are greater" {
+            r del ztmp
+            r zadd ztmp 10 x 20 y 30 z
+            assert {[r zadd ztmp gt ch 5 foo 11 x 21 y 29 z] == 3}
+            assert {[r zcard ztmp] == 4}
+            assert {[r zscore ztmp x] == 11}
+            assert {[r zscore ztmp y] == 21}
+            assert {[r zscore ztmp z] == 30}
+        }
+
+        test "ZADD LT updates existing elements when new scores are lower" {
+            r del ztmp
+            r zadd ztmp 10 x 20 y 30 z
+            assert {[r zadd ztmp lt ch 5 foo 11 x 21 y 29 z] == 2}
+            assert {[r zcard ztmp] == 4}
+            assert {[r zscore ztmp x] == 10}
+            assert {[r zscore ztmp y] == 20}
+            assert {[r zscore ztmp z] == 29}
+        }
+
+        test "ZADD GT XX updates existing elements when new scores are greater and skips new elements" {
+            r del ztmp
+            r zadd ztmp 10 x 20 y 30 z
+            assert {[r zadd ztmp gt xx ch 5 foo 11 x 21 y 29 z] == 2}
+            assert {[r zcard ztmp] == 3}
+            assert {[r zscore ztmp x] == 11}
+            assert {[r zscore ztmp y] == 21}
+            assert {[r zscore ztmp z] == 30}
+        }
+
+        test "ZADD LT XX updates existing elements when new scores are lower and skips new elements" {
+            r del ztmp
+            r zadd ztmp 10 x 20 y 30 z
+            assert {[r zadd ztmp lt xx ch 5 foo 11 x 21 y 29 z] == 1}
+            assert {[r zcard ztmp] == 3}
+            assert {[r zscore ztmp x] == 10}
+            assert {[r zscore ztmp y] == 20}
+            assert {[r zscore ztmp z] == 29}
+        }
+
         test "ZADD XX and NX are not compatible" {
             r del ztmp
             catch {r zadd ztmp xx nx 10 x} err
@@ -100,6 +140,24 @@ start_server {tags {"zset"}} {
             assert {[r zscore ztmp b] == 200}
         }
 
+        test "ZADD GT and NX are not compatible" {
+            r del ztmp
+            catch {r zadd ztmp gt nx 10 x} err
+            set err
+        } {ERR*}
+
+        test "ZADD LT and NX are not compatible" {
+            r del ztmp
+            catch {r zadd ztmp lt nx 10 x} err
+            set err
+        } {ERR*}
+
+        test "ZADD LT and GT are not compatible" {
+            r del ztmp
+            catch {r zadd ztmp lt gt 10 x} err
+            set err
+        } {ERR*}
+
         test "ZADD INCR works like ZINCRBY" {
             r del ztmp
             r zadd ztmp 10 x 20 y 30 z

From e08bf16637db65ed0f2cfc44d3dbc788d6fa89c0 Mon Sep 17 00:00:00 2001
From: "bodong.ybd" <bodong.ybd@alibaba-inc.com>
Date: Fri, 11 Sep 2020 16:59:15 +0800
Subject: [PATCH 0684/1098] Add ZINTER/ZUNION command

Syntax: ZINTER/ZUNION numkeys key [key ...] [WEIGHTS weight [weight ...]]
[AGGREGATE SUM|MIN|MAX] [WITHSCORES]

see #7624
---
 src/db.c                 | 27 +++++++++++++-
 src/server.c             | 10 +++++-
 src/server.h             |  3 ++
 src/t_zset.c             | 77 ++++++++++++++++++++++++++++++----------
 tests/unit/type/zset.tcl | 50 ++++++++++++++++++++++++++
 5 files changed, 146 insertions(+), 21 deletions(-)

diff --git a/src/db.c b/src/db.c
index 668a1193a..0a6481c8f 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1388,7 +1388,7 @@ void getKeysFreeResult(int *result) {
 /* Helper function to extract keys from following commands:
  * ZUNIONSTORE <destkey> <num-keys> <key> <key> ... <key> <options>
  * ZINTERSTORE <destkey> <num-keys> <key> <key> ... <key> <options> */
-int *zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
+int *zunionInterStoreGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
     int i, num, *keys;
     UNUSED(cmd);
 
@@ -1416,6 +1416,31 @@ int *zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *nu
     return keys;
 }
 
+/* Helper function to extract keys from following commands:
+ * ZUNION <num-keys> <key> <key> ... <key> <options>
+ * ZINTER <num-keys> <key> <key> ... <key> <options> */
+int *zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
+    int i, num, *keys;
+    UNUSED(cmd);
+
+    num = atoi(argv[1]->ptr);
+    /* Sanity check. Don't return any key if the command is going to
+     * reply with syntax error. */
+    if (num < 1 || num > (argc-2)) {
+        *numkeys = 0;
+        return NULL;
+    }
+
+    keys = getKeysTempBuffer;
+    if (num>MAX_KEYS_BUFFER)
+        keys = zmalloc(sizeof(int)*(num));
+
+    /* Add all key positions for argv[2...n] to keys[] */
+    for (i = 0; i < num; i++) keys[i] = 2+i;
+    *numkeys = num;
+    return keys;
+}
+
 /* Helper function to extract keys from the following commands:
  * EVAL <script> <num-keys> <key> <key> ... <key> [more stuff]
  * EVALSHA <script> <num-keys> <key> <key> ... <key> [more stuff] */
diff --git a/src/server.c b/src/server.c
index 8ae5c64d9..6179d2f29 100644
--- a/src/server.c
+++ b/src/server.c
@@ -428,10 +428,18 @@ struct redisCommand redisCommandTable[] = {
 
     {"zunionstore",zunionstoreCommand,-4,
      "write use-memory @sortedset",
-     0,zunionInterGetKeys,0,0,0,0,0,0},
+     0,zunionInterStoreGetKeys,0,0,0,0,0,0},
 
     {"zinterstore",zinterstoreCommand,-4,
      "write use-memory @sortedset",
+     0,zunionInterStoreGetKeys,0,0,0,0,0,0},
+
+    {"zunion",zunionCommand,-3,
+     "read-only @sortedset",
+     0,zunionInterGetKeys,0,0,0,0,0,0},
+
+    {"zinter",zinterCommand,-3,
+     "read-only @sortedset",
      0,zunionInterGetKeys,0,0,0,0,0,0},
 
     {"zrange",zrangeCommand,-4,
diff --git a/src/server.h b/src/server.h
index 80df48881..685f929f5 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2175,6 +2175,7 @@ void freeObjAsync(robj *o);
 int *getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
 void getKeysFreeResult(int *result);
 int *zunionInterGetKeys(struct redisCommand *cmd,robj **argv, int argc, int *numkeys);
+int *zunionInterStoreGetKeys(struct redisCommand *cmd,robj **argv, int argc, int *numkeys);
 int *evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
 int *sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
 int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
@@ -2391,6 +2392,8 @@ void hstrlenCommand(client *c);
 void zremrangebyrankCommand(client *c);
 void zunionstoreCommand(client *c);
 void zinterstoreCommand(client *c);
+void zunionCommand(client *c);
+void zinterCommand(client *c);
 void zscanCommand(client *c);
 void hkeysCommand(client *c);
 void hvalsCommand(client *c);
diff --git a/src/t_zset.c b/src/t_zset.c
index 2ef02ac8a..9fddf37d1 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -2196,7 +2196,15 @@ dictType setAccumulatorDictType = {
     NULL                       /* val destructor */
 };
 
-void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
+/* The zunionInterGenericCommand() function is called in order to implement the
+ * following commands: ZUNION, ZINTER, ZUNIONSTORE, ZINTERSTORE.
+ *
+ * 'numkeysIndex' parameter position of key number. for ZUNION/ZINTER command, this
+ * value is 1, for ZUNIONSTORE/ZINTERSTORE command, this value is 2.
+ *
+ * 'op' SET_OP_INTER or SET_OP_UNION.
+ */
+void zunionInterGenericCommand(client *c, robj *dstkey, int numkeysIndex, int op) {
     int i, j;
     long setnum;
     int aggregate = REDIS_AGGR_SUM;
@@ -2207,9 +2215,10 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
     robj *dstobj;
     zset *dstzset;
     zskiplistNode *znode;
+    int withscores = 0;
 
     /* expect setnum input keys to be given */
-    if ((getLongFromObjectOrReply(c, c->argv[2], &setnum, NULL) != C_OK))
+    if ((getLongFromObjectOrReply(c, c->argv[numkeysIndex], &setnum, NULL) != C_OK))
         return;
 
     if (setnum < 1) {
@@ -2219,14 +2228,14 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
     }
 
     /* test if the expected number of keys would overflow */
-    if (setnum > c->argc-3) {
+    if (setnum > (c->argc-(numkeysIndex+1))) {
         addReply(c,shared.syntaxerr);
         return;
     }
 
     /* read keys to be used for input */
     src = zcalloc(sizeof(zsetopsrc) * setnum);
-    for (i = 0, j = 3; i < setnum; i++, j++) {
+    for (i = 0, j = numkeysIndex+1; i < setnum; i++, j++) {
         robj *obj = lookupKeyWrite(c->db,c->argv[j]);
         if (obj != NULL) {
             if (obj->type != OBJ_ZSET && obj->type != OBJ_SET) {
@@ -2279,6 +2288,11 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
                     return;
                 }
                 j++; remaining--;
+            } else if (remaining >= 1 &&
+                       !strcasecmp(c->argv[j]->ptr,"withscores"))
+            {
+                j++; remaining--;
+                withscores = 1;
             } else {
                 zfree(src);
                 addReply(c,shared.syntaxerr);
@@ -2399,20 +2413,37 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
         serverPanic("Unknown operator");
     }
 
-    if (dstzset->zsl->length) {
-        zsetConvertToZiplistIfNeeded(dstobj,maxelelen);
-        setKey(c,c->db,dstkey,dstobj);
-        addReplyLongLong(c,zsetLength(dstobj));
-        notifyKeyspaceEvent(NOTIFY_ZSET,
-            (op == SET_OP_UNION) ? "zunionstore" : "zinterstore",
-            dstkey,c->db->id);
-        server.dirty++;
-    } else {
-        addReply(c,shared.czero);
-        if (dbDelete(c->db,dstkey)) {
-            signalModifiedKey(c,c->db,dstkey);
-            notifyKeyspaceEvent(NOTIFY_GENERIC,"del",dstkey,c->db->id);
+    if (dstkey) {
+        if (dstzset->zsl->length) {
+            zsetConvertToZiplistIfNeeded(dstobj, maxelelen);
+            setKey(c, c->db, dstkey, dstobj);
+            addReplyLongLong(c, zsetLength(dstobj));
+            notifyKeyspaceEvent(NOTIFY_ZSET,
+                                (op == SET_OP_UNION) ? "zunionstore" : "zinterstore",
+                                dstkey, c->db->id);
             server.dirty++;
+        } else {
+            addReply(c, shared.czero);
+            if (dbDelete(c->db, dstkey)) {
+                signalModifiedKey(c, c->db, dstkey);
+                notifyKeyspaceEvent(NOTIFY_GENERIC, "del", dstkey, c->db->id);
+                server.dirty++;
+            }
+        }
+    } else {
+        unsigned long length = dstzset->zsl->length;
+        zskiplist *zsl = dstzset->zsl;
+        zskiplistNode *zn = zsl->header->level[0].forward;
+        if (withscores && c->resp == 2)
+            addReplyArrayLen(c, length*2);
+        else
+            addReplyArrayLen(c, length);
+
+        while (zn != NULL) {
+            if (withscores && c->resp > 2) addReplyArrayLen(c,2);
+            addReplyBulkCBuffer(c,zn->ele,sdslen(zn->ele));
+            if (withscores) addReplyDouble(c,zn->score);
+            zn = zn->level[0].forward;
         }
     }
     decrRefCount(dstobj);
@@ -2420,11 +2451,19 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
 }
 
 void zunionstoreCommand(client *c) {
-    zunionInterGenericCommand(c,c->argv[1], SET_OP_UNION);
+    zunionInterGenericCommand(c, c->argv[1], 2, SET_OP_UNION);
 }
 
 void zinterstoreCommand(client *c) {
-    zunionInterGenericCommand(c,c->argv[1], SET_OP_INTER);
+    zunionInterGenericCommand(c, c->argv[1], 2, SET_OP_INTER);
+}
+
+void zunionCommand(client *c) {
+    zunionInterGenericCommand(c, NULL, 1, SET_OP_UNION);
+}
+
+void zinterCommand(client *c) {
+    zunionInterGenericCommand(c, NULL, 1, SET_OP_INTER);
 }
 
 void zrangeGenericCommand(client *c, int reverse) {
diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl
index e997bc66d..2f9eb823f 100644
--- a/tests/unit/type/zset.tcl
+++ b/tests/unit/type/zset.tcl
@@ -612,6 +612,12 @@ start_server {tags {"zset"}} {
             assert_equal 0 [r exists dst_key]
         }
 
+        test "ZUNION/ZINTER against non-existing key - $encoding" {
+            r del zseta
+            assert_equal {} [r zunion 1 zseta]
+            assert_equal {} [r zinter 1 zseta]
+        }
+
         test "ZUNIONSTORE with empty set - $encoding" {
             r del zseta zsetb
             r zadd zseta 1 a
@@ -620,6 +626,14 @@ start_server {tags {"zset"}} {
             r zrange zsetc 0 -1 withscores
         } {a 1 b 2}
 
+        test "ZUNION/ZINTER with empty set - $encoding" {
+            r del zseta zsetb
+            r zadd zseta 1 a
+            r zadd zseta 2 b
+            assert_equal {a 1 b 2} [r zunion 2 zseta zsetb withscores]
+            assert_equal {} [r zinter 2 zseta zsetb withscores]
+        }
+
         test "ZUNIONSTORE basics - $encoding" {
             r del zseta zsetb zsetc
             r zadd zseta 1 a
@@ -633,11 +647,29 @@ start_server {tags {"zset"}} {
             assert_equal {a 1 b 3 d 3 c 5} [r zrange zsetc 0 -1 withscores]
         }
 
+        test "ZUNION/ZINTER with integer members - $encoding" {
+            r del zsetd zsetf
+            r zadd zsetd 1 1
+            r zadd zsetd 2 2
+            r zadd zsetd 3 3
+            r zadd zsetf 1 1
+            r zadd zsetf 3 3
+            r zadd zsetf 4 4
+
+            assert_equal {1 2 2 2 4 4 3 6} [r zunion 2 zsetd zsetf withscores]
+            assert_equal {1 2 3 6} [r zinter 2 zsetd zsetf withscores]
+        }
+
         test "ZUNIONSTORE with weights - $encoding" {
             assert_equal 4 [r zunionstore zsetc 2 zseta zsetb weights 2 3]
             assert_equal {a 2 b 7 d 9 c 12} [r zrange zsetc 0 -1 withscores]
         }
 
+        test "ZUNION with weights - $encoding" {
+            assert_equal {a 2 b 7 d 9 c 12} [r zunion 2 zseta zsetb weights 2 3 withscores]
+            assert_equal {b 7 c 12} [r zinter 2 zseta zsetb weights 2 3 withscores]
+        }
+
         test "ZUNIONSTORE with a regular set and weights - $encoding" {
             r del seta
             r sadd seta a
@@ -653,21 +685,39 @@ start_server {tags {"zset"}} {
             assert_equal {a 1 b 1 c 2 d 3} [r zrange zsetc 0 -1 withscores]
         }
 
+        test "ZUNION/ZINTER with AGGREGATE MIN - $encoding" {
+            assert_equal {a 1 b 1 c 2 d 3} [r zunion 2 zseta zsetb aggregate min withscores]
+            assert_equal {b 1 c 2} [r zinter 2 zseta zsetb aggregate min withscores]
+        }
+
         test "ZUNIONSTORE with AGGREGATE MAX - $encoding" {
             assert_equal 4 [r zunionstore zsetc 2 zseta zsetb aggregate max]
             assert_equal {a 1 b 2 c 3 d 3} [r zrange zsetc 0 -1 withscores]
         }
 
+        test "ZUNION/ZINTER with AGGREGATE MAX - $encoding" {
+            assert_equal {a 1 b 2 c 3 d 3} [r zunion 2 zseta zsetb aggregate max withscores]
+            assert_equal {b 2 c 3} [r zinter 2 zseta zsetb aggregate max withscores]
+        }
+
         test "ZINTERSTORE basics - $encoding" {
             assert_equal 2 [r zinterstore zsetc 2 zseta zsetb]
             assert_equal {b 3 c 5} [r zrange zsetc 0 -1 withscores]
         }
 
+        test "ZINTER basics - $encoding" {
+            assert_equal {b 3 c 5} [r zinter 2 zseta zsetb withscores]
+        }
+
         test "ZINTERSTORE with weights - $encoding" {
             assert_equal 2 [r zinterstore zsetc 2 zseta zsetb weights 2 3]
             assert_equal {b 7 c 12} [r zrange zsetc 0 -1 withscores]
         }
 
+        test "ZINTER with weights - $encoding" {
+            assert_equal {b 7 c 12} [r zinter 2 zseta zsetb weights 2 3 withscores]
+        }
+
         test "ZINTERSTORE with a regular set and weights - $encoding" {
             r del seta
             r sadd seta a

From b7ce583a5e31dbb674e0aa24366d4648c2a45dfb Mon Sep 17 00:00:00 2001
From: "bodong.ybd" <bodong.ybd@alibaba-inc.com>
Date: Thu, 17 Sep 2020 15:54:37 +0800
Subject: [PATCH 0685/1098] Refactor multi-key command get keys proc

---
 src/db.c | 96 +++++++++++++++++++-------------------------------------
 1 file changed, 32 insertions(+), 64 deletions(-)

diff --git a/src/db.c b/src/db.c
index 0a6481c8f..c004468d8 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1386,86 +1386,54 @@ void getKeysFreeResult(int *result) {
 }
 
 /* Helper function to extract keys from following commands:
- * ZUNIONSTORE <destkey> <num-keys> <key> <key> ... <key> <options>
- * ZINTERSTORE <destkey> <num-keys> <key> <key> ... <key> <options> */
-int *zunionInterStoreGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
-    int i, num, *keys;
-    UNUSED(cmd);
-
-    num = atoi(argv[2]->ptr);
-    /* Sanity check. Don't return any key if the command is going to
-     * reply with syntax error. */
-    if (num < 1 || num > (argc-3)) {
-        *numkeys = 0;
-        return NULL;
-    }
-
-    /* Keys in z{union,inter}store come from two places:
-     * argv[1] = storage key,
-     * argv[3...n] = keys to intersect */
-    keys = getKeysTempBuffer;
-    if (num+1>MAX_KEYS_BUFFER)
-        keys = zmalloc(sizeof(int)*(num+1));
-
-    /* Add all key positions for argv[3...n] to keys[] */
-    for (i = 0; i < num; i++) keys[i] = 3+i;
-
-    /* Finally add the argv[1] key position (the storage key target). */
-    keys[num] = 1;
-    *numkeys = num+1;  /* Total keys = {union,inter} keys + storage key */
-    return keys;
-}
-
-/* Helper function to extract keys from following commands:
+ * COMMAND [destkey] <num-keys> <key> [...] <key> [...] ... <options>
+ *
+ * eg:
  * ZUNION <num-keys> <key> <key> ... <key> <options>
- * ZINTER <num-keys> <key> <key> ... <key> <options> */
-int *zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
+ * ZUNIONSTORE <destkey> <num-keys> <key> <key> ... <key> <options>
+ *
+ * 'storeKeyOfs': destkey index, 0 means destkey not exists.
+ * 'keyCountOfs': num-keys index.
+ * 'firstKeyOfs': firstkey index.
+ * 'keyStep': the interval of each key, usually this value is 1.
+ * */
+int *genericGetKeys(int storeKeyOfs, int keyCountOfs, int firstKeyOfs, int keyStep,
+                    robj **argv, int argc, int *numkeys) {
     int i, num, *keys;
-    UNUSED(cmd);
 
-    num = atoi(argv[1]->ptr);
+    num = atoi(argv[keyCountOfs]->ptr);
     /* Sanity check. Don't return any key if the command is going to
-     * reply with syntax error. */
-    if (num < 1 || num > (argc-2)) {
+     * reply with syntax error. (no input keys). */
+    if (num < 1 || num > (argc - firstKeyOfs)/keyStep) {
         *numkeys = 0;
         return NULL;
     }
 
     keys = getKeysTempBuffer;
-    if (num>MAX_KEYS_BUFFER)
-        keys = zmalloc(sizeof(int)*(num));
+    *numkeys = storeKeyOfs ? num + 1 : num;
+    if (*numkeys > MAX_KEYS_BUFFER)
+        keys = zmalloc(sizeof(int)*(*numkeys));
 
-    /* Add all key positions for argv[2...n] to keys[] */
-    for (i = 0; i < num; i++) keys[i] = 2+i;
-    *numkeys = num;
+    /* Add all key positions for argv[firstKeyOfs...n] to keys[] */
+    for (i = 0; i < num; i++) keys[i] = firstKeyOfs+(i*keyStep);
+
+    if (storeKeyOfs) keys[num] = storeKeyOfs;
     return keys;
 }
 
-/* Helper function to extract keys from the following commands:
- * EVAL <script> <num-keys> <key> <key> ... <key> [more stuff]
- * EVALSHA <script> <num-keys> <key> <key> ... <key> [more stuff] */
-int *evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
-    int i, num, *keys;
+int *zunionInterStoreGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
     UNUSED(cmd);
+    return genericGetKeys(1, 2, 3, 1, argv, argc, numkeys);
+}
 
-    num = atoi(argv[2]->ptr);
-    /* Sanity check. Don't return any key if the command is going to
-     * reply with syntax error. */
-    if (num <= 0 || num > (argc-3)) {
-        *numkeys = 0;
-        return NULL;
-    }
+int *zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
+    UNUSED(cmd);
+    return genericGetKeys(0, 1, 2, 1, argv, argc, numkeys);
+}
 
-    keys = getKeysTempBuffer;
-    if (num>MAX_KEYS_BUFFER)
-        keys = zmalloc(sizeof(int)*num);
-
-    *numkeys = num;
-
-    /* Add all key positions for argv[3...n] to keys[] */
-    for (i = 0; i < num; i++) keys[i] = 3+i;
-
-    return keys;
+int *evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
+    UNUSED(cmd);
+    return genericGetKeys(0, 2, 3, 1, argv, argc, numkeys);
 }
 
 /* Helper function to extract keys from the SORT command.

From 795c454db1ce74b6eb2aee9db124f8316e9f188c Mon Sep 17 00:00:00 2001
From: valentinogeron <valentino@redislabs.com>
Date: Thu, 24 Sep 2020 12:02:40 +0300
Subject: [PATCH 0686/1098] Stream: Inconsistency between master and replica
 some XREADGROUP case (#7526)

XREADGROUP auto-creates the consumer inside the consumer group the
first time it saw it.
When XREADGROUP is being used with NOACK option, the message will not
be added into the client's PEL and XGROUP SETID would be propagated.
When the replica gets the XGROUP SETID it will only update the last delivered
id of the group, but will not create the consumer.

So, in this commit XGROUP CREATECONSUMER is being added.
Command pattern: XGROUP CREATECONSUMER <key> <group> <consumer>.

When NOACK option is being used, createconsumer command would be
propagated as well.

In case of AOFREWRITE, consumer with an empty PEL would be saved with
XGROUP CREATECONSUMER whereas consumer with pending entries would be
saved with XCLAIM
---
 src/aof.c                          | 27 ++++++++-
 src/blocked.c                      |  9 ++-
 src/rdb.c                          |  2 +-
 src/stream.h                       |  3 +-
 src/t_stream.c                     | 77 +++++++++++++++++++-----
 tests/unit/type/stream-cgroups.tcl | 96 ++++++++++++++++++++++++++++++
 6 files changed, 193 insertions(+), 21 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index f1490a39c..c51fdbeff 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1185,6 +1185,20 @@ int rioWriteStreamPendingEntry(rio *r, robj *key, const char *groupname, size_t
     return 1;
 }
 
+/* Helper for rewriteStreamObject(): emit the XGROUP CREATECONSUMER is
+ * needed in order to create consumers that do not have any pending entries.
+ * All this in the context of the specified key and group. */
+int rioWriteStreamEmptyConsumer(rio *r, robj *key, const char *groupname, size_t groupname_len, streamConsumer *consumer) {
+    /* XGROUP CREATECONSUMER <key> <group> <consumer> */
+    if (rioWriteBulkCount(r,'*',5) == 0) return 0;
+    if (rioWriteBulkString(r,"XGROUP",6) == 0) return 0;
+    if (rioWriteBulkString(r,"CREATECONSUMER",14) == 0) return 0;
+    if (rioWriteBulkObject(r,key) == 0) return 0;
+    if (rioWriteBulkString(r,groupname,groupname_len) == 0) return 0;
+    if (rioWriteBulkString(r,consumer->name,sdslen(consumer->name)) == 0) return 0;
+    return 1;
+}
+
 /* Emit the commands needed to rebuild a stream object.
  * The function returns 0 on error, 1 on success. */
 int rewriteStreamObject(rio *r, robj *key, robj *o) {
@@ -1273,13 +1287,22 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) {
             }
 
             /* Generate XCLAIMs for each consumer that happens to
-             * have pending entries. Empty consumers have no semantical
-             * value so they are discarded. */
+             * have pending entries. Empty consumers would be generated with
+             * XGROUP CREATECONSUMER. */
             raxIterator ri_cons;
             raxStart(&ri_cons,group->consumers);
             raxSeek(&ri_cons,"^",NULL,0);
             while(raxNext(&ri_cons)) {
                 streamConsumer *consumer = ri_cons.data;
+                /* If there are no pending entries, just emit XGROUP CREATECONSUMER */
+                if (raxSize(consumer->pel) == 0) {
+                    if (rioWriteStreamEmptyConsumer(r,key,(char*)ri.key,
+                                                    ri.key_len,consumer) == 0)
+                    {
+                        return 0;
+                    }
+                    continue;
+                }
                 /* For the current consumer, iterate all the PEL entries
                  * to emit the XCLAIM protocol. */
                 raxIterator ri_pel;
diff --git a/src/blocked.c b/src/blocked.c
index 08de63485..87f816faa 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -371,11 +371,18 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) {
                 int noack = 0;
 
                 if (group) {
+                    int created = 0;
                     consumer =
                         streamLookupConsumer(group,
                                              receiver->bpop.xread_consumer->ptr,
-                                             SLC_NONE);
+                                             SLC_NONE,
+                                             &created);
                     noack = receiver->bpop.xread_group_noack;
+                    if (created && noack) {
+                        streamPropagateConsumerCreation(receiver,rl->key,
+                                                        receiver->bpop.xread_group,
+                                                        consumer->name);
+                    }
                 }
 
                 /* Emit the two elements sub-array consisting of
diff --git a/src/rdb.c b/src/rdb.c
index 00cc184af..6420e9924 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1912,7 +1912,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                     return NULL;
                 }
                 streamConsumer *consumer =
-                    streamLookupConsumer(cgroup,cname,SLC_NONE);
+                    streamLookupConsumer(cgroup,cname,SLC_NONE,NULL);
                 sdsfree(cname);
                 consumer->seen_time = rdbLoadMillisecondTime(rdb,RDB_VERSION);
                 if (rioGetReadError(rdb)) {
diff --git a/src/stream.h b/src/stream.h
index e4c5ff78d..3d692cd9d 100644
--- a/src/stream.h
+++ b/src/stream.h
@@ -110,12 +110,13 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields);
 void streamIteratorGetField(streamIterator *si, unsigned char **fieldptr, unsigned char **valueptr, int64_t *fieldlen, int64_t *valuelen);
 void streamIteratorStop(streamIterator *si);
 streamCG *streamLookupCG(stream *s, sds groupname);
-streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int flags);
+streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int flags, int *created);
 streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id);
 streamNACK *streamCreateNACK(streamConsumer *consumer);
 void streamDecodeID(void *buf, streamID *id);
 int streamCompareID(streamID *a, streamID *b);
 void streamFreeNACK(streamNACK *na);
 void streamIncrID(streamID *id);
+void streamPropagateConsumerCreation(client *c, robj *key, robj *groupname, sds consumername);
 
 #endif
diff --git a/src/t_stream.c b/src/t_stream.c
index d565dd1f6..285684987 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -894,6 +894,30 @@ void streamPropagateGroupID(client *c, robj *key, streamCG *group, robj *groupna
     decrRefCount(argv[4]);
 }
 
+/* We need this when we want to propagate creation of consumer that was created
+ * by XREADGROUP with the NOACK option. In that case, the only way to create
+ * the consumer at the replica is by using XGROUP CREATECONSUMER (see issue #7140)
+ *
+ *  XGROUP CREATECONSUMER <key> <groupname> <consumername>
+ */
+void streamPropagateConsumerCreation(client *c, robj *key, robj *groupname, sds consumername) {
+    robj *argv[5];
+    argv[0] = createStringObject("XGROUP",6);
+    argv[1] = createStringObject("CREATECONSUMER",14);
+    argv[2] = key;
+    argv[3] = groupname;
+    argv[4] = createObject(OBJ_STRING,sdsdup(consumername));
+
+    /* We use progagate() because this code path is not always called from
+     * the command execution context. Moreover this will just alter the
+     * consumer group state, and we don't need MULTI/EXEC wrapping because
+     * there is no message state cross-message atomicity required. */
+    propagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL);
+    decrRefCount(argv[0]);
+    decrRefCount(argv[1]);
+    decrRefCount(argv[4]);
+}
+
 /* Send the stream items in the specified range to the client 'c'. The range
  * the client will receive is between start and end inclusive, if 'count' is
  * non zero, no more than 'count' elements are sent.
@@ -1565,11 +1589,17 @@ void xreadCommand(client *c) {
              * of the stream and the data we extracted from it. */
             if (c->resp == 2) addReplyArrayLen(c,2);
             addReplyBulk(c,c->argv[streams_arg+i]);
+            int created = 0;
             streamConsumer *consumer = NULL;
             if (groups) consumer = streamLookupConsumer(groups[i],
                                                         consumername->ptr,
-                                                        SLC_NONE);
+                                                        SLC_NONE,
+                                                        &created);
             streamPropInfo spi = {c->argv[i+streams_arg],groupname};
+            if (created && noack)
+                streamPropagateConsumerCreation(c,spi.keyname,
+                                                spi.groupname,
+                                                consumer->name);
             int flags = 0;
             if (noack) flags |= STREAM_RWR_NOACK;
             if (serve_history) flags |= STREAM_RWR_HISTORY;
@@ -1701,10 +1731,10 @@ streamCG *streamLookupCG(stream *s, sds groupname) {
 }
 
 /* Lookup the consumer with the specified name in the group 'cg': if the
- * consumer does not exist it is automatically created as a side effect
- * of calling this function, otherwise its last seen time is updated and
- * the existing consumer reference returned. */
-streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int flags) {
+ * consumer does not exist it is created unless SLC_NOCREAT flag was specified.
+ * Its last seen time is updated unless SLC_NOREFRESH flag was specified. */
+streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int flags, int *created) {
+    if (created) *created = 0;
     int create = !(flags & SLC_NOCREAT);
     int refresh = !(flags & SLC_NOREFRESH);
     streamConsumer *consumer = raxFind(cg->consumers,(unsigned char*)name,
@@ -1716,8 +1746,10 @@ streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int flags) {
         consumer->pel = raxNew();
         raxInsert(cg->consumers,(unsigned char*)name,sdslen(name),
                   consumer,NULL);
-    }
-    if (refresh) consumer->seen_time = mstime();
+        consumer->seen_time = mstime();
+        if (created) *created = 1;
+    } else if (refresh)
+        consumer->seen_time = mstime();
     return consumer;
 }
 
@@ -1726,7 +1758,7 @@ streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int flags) {
  * of pending messages "lost" is returned. */
 uint64_t streamDelConsumer(streamCG *cg, sds name) {
     streamConsumer *consumer =
-        streamLookupConsumer(cg,name,SLC_NOCREAT|SLC_NOREFRESH);
+        streamLookupConsumer(cg,name,SLC_NOCREAT|SLC_NOREFRESH,NULL);
     if (consumer == NULL) return 0;
 
     uint64_t retval = raxSize(consumer->pel);
@@ -1756,15 +1788,17 @@ uint64_t streamDelConsumer(streamCG *cg, sds name) {
 /* XGROUP CREATE <key> <groupname> <id or $> [MKSTREAM]
  * XGROUP SETID <key> <groupname> <id or $>
  * XGROUP DESTROY <key> <groupname>
+ * CREATECONSUMER <key> <groupname> <consumer>
  * XGROUP DELCONSUMER <key> <groupname> <consumername> */
 void xgroupCommand(client *c) {
     const char *help[] = {
-"CREATE      <key> <groupname> <id or $> [opt] -- Create a new consumer group.",
-"            option MKSTREAM: create the empty stream if it does not exist.",
-"SETID       <key> <groupname> <id or $>  -- Set the current group ID.",
-"DESTROY     <key> <groupname>            -- Remove the specified group.",
-"DELCONSUMER <key> <groupname> <consumer> -- Remove the specified consumer.",
-"HELP                                     -- Prints this help.",
+"CREATE         <key> <groupname> <id or $> [opt] -- Create a new consumer group.",
+"               option MKSTREAM: create the empty stream if it does not exist.",
+"SETID          <key> <groupname> <id or $>  -- Set the current group ID.",
+"DESTROY        <key> <groupname>            -- Remove the specified group.",
+"CREATECONSUMER <key> <groupname> <consumer> -- Create new consumer in the specified group.",
+"DELCONSUMER    <key> <groupname> <consumer> -- Remove the specified consumer.",
+"HELP                                        -- Prints this help.",
 NULL
     };
     stream *s = NULL;
@@ -1809,6 +1843,7 @@ NULL
         /* Certain subcommands require the group to exist. */
         if ((cg = streamLookupCG(s,grpname)) == NULL &&
             (!strcasecmp(opt,"SETID") ||
+             !strcasecmp(opt,"CREATECONSUMER") ||
              !strcasecmp(opt,"DELCONSUMER")))
         {
             addReplyErrorFormat(c, "-NOGROUP No such consumer group '%s' "
@@ -1875,6 +1910,15 @@ NULL
         } else {
             addReply(c,shared.czero);
         }
+    } else if (!strcasecmp(opt,"CREATECONSUMER") && c->argc == 5) {
+        int created = 0;
+        streamLookupConsumer(cg,c->argv[4]->ptr,SLC_NOREFRESH,&created);
+        if (created) {
+            server.dirty++;
+            notifyKeyspaceEvent(NOTIFY_STREAM,"xgroup-createconsumer",
+                                c->argv[2],c->db->id);
+        }
+        addReplyLongLong(c,created);
     } else if (!strcasecmp(opt,"DELCONSUMER") && c->argc == 5) {
         /* Delete the consumer and returns the number of pending messages
          * that were yet associated with such a consumer. */
@@ -2077,7 +2121,8 @@ void xpendingCommand(client *c) {
         if (consumername) {
             consumer = streamLookupConsumer(group,
                                             consumername->ptr,
-                                            SLC_NOCREAT|SLC_NOREFRESH);
+                                            SLC_NOCREAT|SLC_NOREFRESH,
+                                            NULL);
 
             /* If a consumer name was mentioned but it does not exist, we can
              * just return an empty array. */
@@ -2348,7 +2393,7 @@ void xclaimCommand(client *c) {
                 raxRemove(nack->consumer->pel,buf,sizeof(buf),NULL);
             /* Update the consumer and idle time. */
             if (consumer == NULL)
-                consumer = streamLookupConsumer(group,c->argv[3]->ptr,SLC_NONE);
+                consumer = streamLookupConsumer(group,c->argv[3]->ptr,SLC_NONE,NULL);
             nack->consumer = consumer;
             nack->delivery_time = deliverytime;
             /* Set the delivery attempts counter if given, otherwise
diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl
index dfcd735f6..74ddc98ba 100644
--- a/tests/unit/type/stream-cgroups.tcl
+++ b/tests/unit/type/stream-cgroups.tcl
@@ -328,6 +328,102 @@ start_server {
         assert_equal [lindex $reply 9] "{100-0 {a 1}}"
     }
 
+    test {XGROUP CREATECONSUMER: create consumer if does not exist} {
+        r del mystream
+        r XGROUP CREATE mystream mygroup $ MKSTREAM
+        r XADD mystream * f v
+
+        set reply [r xinfo groups mystream]
+        set group_info [lindex $reply 0]
+        set n_consumers [lindex $group_info 3]
+        assert_equal $n_consumers 0 ;# consumers number in cg
+
+        # create consumer using XREADGROUP
+        r XREADGROUP GROUP mygroup Alice COUNT 1 STREAMS mystream >
+
+        set reply [r xinfo groups mystream]
+        set group_info [lindex $reply 0]
+        set n_consumers [lindex $group_info 3]
+        assert_equal $n_consumers 1 ;# consumers number in cg
+
+        set reply [r xinfo consumers mystream mygroup]
+        set consumer_info [lindex $reply 0]
+        assert_equal [lindex $consumer_info 1] "Alice" ;# consumer name
+
+        # create group using XGROUP CREATECONSUMER when Alice already exists
+        set created [r XGROUP CREATECONSUMER mystream mygroup Alice]
+        assert_equal $created 0
+
+        # create group using XGROUP CREATECONSUMER when Bob does not exist
+        set created [r XGROUP CREATECONSUMER mystream mygroup Bob]
+        assert_equal $created 1
+
+        set reply [r xinfo groups mystream]
+        set group_info [lindex $reply 0]
+        set n_consumers [lindex $group_info 3]
+        assert_equal $n_consumers 2 ;# consumers number in cg
+
+        set reply [r xinfo consumers mystream mygroup]
+        set consumer_info [lindex $reply 0]
+        assert_equal [lindex $consumer_info 1] "Alice" ;# consumer name
+        set consumer_info [lindex $reply 1]
+        assert_equal [lindex $consumer_info 1] "Bob" ;# consumer name
+    }
+
+    test {XGROUP CREATECONSUMER: group must exist} {
+        r del mystream
+        r XADD mystream * f v
+        assert_error "*NOGROUP*" {r XGROUP CREATECONSUMER mystream mygroup consumer}
+    }
+
+    start_server {tags {"stream"} overrides {appendonly yes aof-use-rdb-preamble no appendfsync always}} {
+        test {XREADGROUP with NOACK creates consumer} {
+            r del mystream
+            r XGROUP CREATE mystream mygroup $ MKSTREAM
+            r XADD mystream * f1 v1
+            r XREADGROUP GROUP mygroup Alice NOACK STREAMS mystream ">"
+            set rd [redis_deferring_client]
+            $rd XREADGROUP GROUP mygroup Bob BLOCK 0 NOACK STREAMS mystream ">"
+            r XADD mystream * f2 v2
+            set grpinfo [r xinfo groups mystream]
+
+            r debug loadaof
+            assert {[r xinfo groups mystream] == $grpinfo}
+            set reply [r xinfo consumers mystream mygroup]
+            set consumer_info [lindex $reply 0]
+            assert_equal [lindex $consumer_info 1] "Alice" ;# consumer name
+            set consumer_info [lindex $reply 1]
+            assert_equal [lindex $consumer_info 1] "Bob" ;# consumer name
+        }
+
+        test {Consumer without PEL is present in AOF after AOFRW} {
+            r del mystream
+            r XGROUP CREATE mystream mygroup $ MKSTREAM
+            r XADD mystream * f v
+            r XREADGROUP GROUP mygroup Alice NOACK STREAMS mystream ">"
+            set rd [redis_deferring_client]
+            $rd XREADGROUP GROUP mygroup Bob BLOCK 0 NOACK STREAMS mystream ">"
+            r XGROUP CREATECONSUMER mystream mygroup Charlie
+            set grpinfo [lindex [r xinfo groups mystream] 0]
+
+            r bgrewriteaof
+            waitForBgrewriteaof r
+            r debug loadaof
+
+            set curr_grpinfo [lindex [r xinfo groups mystream] 0]
+            assert {$curr_grpinfo == $grpinfo}
+            set n_consumers [lindex $grpinfo 3]
+
+            # Bob should be created only when there will be new data for this client
+            assert_equal $n_consumers 2
+            set reply [r xinfo consumers mystream mygroup]
+            set consumer_info [lindex $reply 0]
+            assert_equal [lindex $consumer_info 1] "Alice"
+            set consumer_info [lindex $reply 1]
+            assert_equal [lindex $consumer_info 1] "Charlie"
+        }
+    }
+
     start_server {} {
         set master [srv -1 client]
         set master_host [srv -1 host]

From b464afb9e2c16277a5cf62cf1290c8e2ad4b0aa1 Mon Sep 17 00:00:00 2001
From: Guy Korland <gkorland@gmail.com>
Date: Thu, 24 Sep 2020 12:45:30 +0300
Subject: [PATCH 0687/1098] Fix RedisModule_HashGet examples (#6697)

---
 src/module.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/module.c b/src/module.c
index 77468700e..ddf8ee888 100644
--- a/src/module.c
+++ b/src/module.c
@@ -2953,22 +2953,22 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) {
  * As with RedisModule_HashSet() the behavior of the command can be specified
  * passing flags different than REDISMODULE_HASH_NONE:
  *
- * REDISMODULE_HASH_CFIELD: field names as null terminated C strings.
+ * REDISMODULE_HASH_CFIELDS: field names as null terminated C strings.
  *
  * REDISMODULE_HASH_EXISTS: instead of setting the value of the field
  * expecting a RedisModuleString pointer to pointer, the function just
  * reports if the field exists or not and expects an integer pointer
  * as the second element of each pair.
  *
- * Example of REDISMODULE_HASH_CFIELD:
+ * Example of REDISMODULE_HASH_CFIELDS:
  *
  *      RedisModuleString *username, *hashedpass;
- *      RedisModule_HashGet(mykey,"username",&username,"hp",&hashedpass, NULL);
+ *      RedisModule_HashGet(mykey,REDISMODULE_HASH_CFIELDS,"username",&username,"hp",&hashedpass, NULL);
  *
  * Example of REDISMODULE_HASH_EXISTS:
  *
  *      int exists;
- *      RedisModule_HashGet(mykey,argv[1],&exists,NULL);
+ *      RedisModule_HashGet(mykey,REDISMODULE_HASH_EXISTS,argv[1],&exists,NULL);
  *
  * The function returns REDISMODULE_OK on success and REDISMODULE_ERR if
  * the key is not an hash value.

From 57709c4bc663ddcb9313777c551a92dabf07095e Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuancode@163.com>
Date: Thu, 24 Sep 2020 21:01:41 +0800
Subject: [PATCH 0688/1098] Don't write replies if close the client ASAP
 (#7202)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Before this commit, we would have continued to add replies to the reply buffer even if client
output buffer limit is reached, so the used memory would keep increasing over the configured limit.
What's more, we shouldn’t write any reply to the client if it is set 'CLIENT_CLOSE_ASAP' flag
because that doesn't conform to its definition and we will close all clients flagged with
'CLIENT_CLOSE_ASAP' in ‘beforeSleep’.

Because of code execution order, before this, we may firstly write to part of the replies to
the socket before disconnecting it, but in fact, we may can’t send the full replies to clients
since OS socket buffer is limited. But this unexpected behavior makes some commands work well,
for instance ACL DELUSER, if the client deletes the current user, we need to send reply to client
and close the connection, but before, we close the client firstly and write the reply to reply
buffer. secondly, we shouldn't do this despite the fact it works well in most cases.

We add a flag 'CLIENT_CLOSE_AFTER_COMMAND' to mark clients, this flag means we will close the
client after executing commands and send all entire replies, so that we can write replies to
reply buffer during executing commands, send replies to clients, and close them later.

We also fix some implicit problems. If client output buffer limit is enforced in 'multi/exec',
all commands will be executed completely in redis and clients will not read any reply instead of
partial replies. Even more, if the client executes 'ACL deluser' the using user in 'multi/exec',
it will not read the replies after 'ACL deluser' just like before executing 'client kill' itself
in 'multi/exec'.

We added some tests for output buffer limit breach during multi-exec and using a pipeline of
many small commands rather than one with big response.

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/acl.c                  |  8 +++-
 src/module.c               |  8 +++-
 src/networking.c           | 14 ++++++
 src/server.c               |  7 +++
 src/server.h               |  2 +
 tests/unit/acl.tcl         | 17 +++++++
 tests/unit/obuf-limits.tcl | 90 ++++++++++++++++++++++++++++++++++++++
 7 files changed, 144 insertions(+), 2 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 7af3d6242..b4fb52c50 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -297,7 +297,13 @@ void ACLFreeUserAndKillClients(user *u) {
              * it in non authenticated mode. */
             c->user = DefaultUser;
             c->authenticated = 0;
-            freeClientAsync(c);
+            /* We will write replies to this client later, so we can't
+             * close it directly even if async. */
+            if (c == server.current_client) {
+                c->flags |= CLIENT_CLOSE_AFTER_COMMAND;
+            } else {
+                freeClientAsync(c);
+            }
         }
     }
     ACLFreeUser(u);
diff --git a/src/module.c b/src/module.c
index ddf8ee888..b76b57698 100644
--- a/src/module.c
+++ b/src/module.c
@@ -5538,7 +5538,13 @@ void revokeClientAuthentication(client *c) {
 
     c->user = DefaultUser;
     c->authenticated = 0;
-    freeClientAsync(c);
+    /* We will write replies to this client later, so we can't close it
+     * directly even if async. */
+    if (c == server.current_client) {
+        c->flags |= CLIENT_CLOSE_AFTER_COMMAND;
+    } else {
+        freeClientAsync(c);
+    }
 }
 
 /* Cleanup all clients that have been authenticated with this module. This
diff --git a/src/networking.c b/src/networking.c
index 7ac8ed1c4..4cbddb660 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -224,6 +224,9 @@ int prepareClientToWrite(client *c) {
      * handler since there is no socket at all. */
     if (c->flags & (CLIENT_LUA|CLIENT_MODULE)) return C_OK;
 
+    /* If CLIENT_CLOSE_ASAP flag is set, we need not write anything. */
+    if (c->flags & CLIENT_CLOSE_ASAP) return C_ERR;
+
     /* CLIENT REPLY OFF / SKIP handling: don't send replies. */
     if (c->flags & (CLIENT_REPLY_OFF|CLIENT_REPLY_SKIP)) return C_ERR;
 
@@ -1471,6 +1474,9 @@ int handleClientsWithPendingWrites(void) {
          * that may trigger write error or recreate handler. */
         if (c->flags & CLIENT_PROTECTED) continue;
 
+        /* Don't write to clients that are going to be closed anyway. */
+        if (c->flags & CLIENT_CLOSE_ASAP) continue;
+
         /* Try to write buffers to the client socket. */
         if (writeToClient(c,0) == C_ERR) continue;
 
@@ -3160,6 +3166,14 @@ int handleClientsWithPendingWritesUsingThreads(void) {
     while((ln = listNext(&li))) {
         client *c = listNodeValue(ln);
         c->flags &= ~CLIENT_PENDING_WRITE;
+
+        /* Remove clients from the list of pending writes since
+         * they are going to be closed ASAP. */
+        if (c->flags & CLIENT_CLOSE_ASAP) {
+            listDelNode(server.clients_pending_write, ln);
+            continue;
+        }
+
         int target_id = item_id % server.io_threads_num;
         listAddNodeTail(io_threads_list[target_id],c);
         item_id++;
diff --git a/src/server.c b/src/server.c
index 6179d2f29..24238c595 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3427,6 +3427,13 @@ void call(client *c, int flags) {
     dirty = server.dirty-dirty;
     if (dirty < 0) dirty = 0;
 
+    /* After executing command, we will close the client after writing entire
+     * reply if it is set 'CLIENT_CLOSE_AFTER_COMMAND' flag. */
+    if (c->flags & CLIENT_CLOSE_AFTER_COMMAND) {
+        c->flags &= ~CLIENT_CLOSE_AFTER_COMMAND;
+        c->flags |= CLIENT_CLOSE_AFTER_REPLY;
+    }
+
     /* When EVAL is called loading the AOF we don't want commands called
      * from Lua to go into the slowlog or to populate statistics. */
     if (server.loading && c->flags & CLIENT_LUA)
diff --git a/src/server.h b/src/server.h
index 685f929f5..5c266f61d 100644
--- a/src/server.h
+++ b/src/server.h
@@ -265,6 +265,8 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
                                              about writes performed by myself.*/
 #define CLIENT_IN_TO_TABLE (1ULL<<38) /* This client is in the timeout table. */
 #define CLIENT_PROTOCOL_ERROR (1ULL<<39) /* Protocol error chatting with it. */
+#define CLIENT_CLOSE_AFTER_COMMAND (1ULL<<40) /* Close after executing commands
+                                               * and writing entire reply. */
 
 /* Client block type (btype field in client structure)
  * if CLIENT_BLOCKED flag is set. */
diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl
index 381f2f95f..f015f75a0 100644
--- a/tests/unit/acl.tcl
+++ b/tests/unit/acl.tcl
@@ -260,6 +260,23 @@ start_server {tags {"acl"}} {
         catch {r ACL help xxx} e
         assert_match "*Unknown subcommand or wrong number of arguments*" $e
     }
+
+    test {Delete a user that the client doesn't use} {
+        r ACL setuser not_used on >passwd
+        assert {[r ACL deluser not_used] == 1}
+        # The client is not closed
+        assert {[r ping] eq {PONG}}
+    }
+
+    test {Delete a user that the client is using} {
+        r ACL setuser using on +acl >passwd
+        r AUTH using passwd
+        # The client will receive reply normally
+        assert {[r ACL deluser using] == 1}
+        # The client is closed
+        catch {[r ping]} e
+        assert_match "*I/O error*" $e
+    }
 }
 
 set server_path [tmpdir "server.acl"]
diff --git a/tests/unit/obuf-limits.tcl b/tests/unit/obuf-limits.tcl
index c45bf8e86..20ba32fd5 100644
--- a/tests/unit/obuf-limits.tcl
+++ b/tests/unit/obuf-limits.tcl
@@ -70,4 +70,94 @@ start_server {tags {"obuf-limits"}} {
         assert {$omem >= 100000 && $time_elapsed < 6}
         $rd1 close
     }
+
+    test {No response for single command if client output buffer hard limit is enforced} {
+        r config set client-output-buffer-limit {normal 100000 0 0}
+        # Total size of all items must be more than 100k
+        set item [string repeat "x" 1000]
+        for {set i 0} {$i < 150} {incr i} {
+            r lpush mylist $item
+        }
+        set orig_mem [s used_memory]
+        # Set client name and get all items
+        set rd [redis_deferring_client]
+        $rd client setname mybiglist
+        assert {[$rd read] eq "OK"}
+        $rd lrange mylist 0 -1
+        $rd flush
+        after 100
+
+        # Before we read reply, redis will close this client.
+        set clients [r client list]
+        assert_no_match "*name=mybiglist*" $clients
+        set cur_mem [s used_memory]
+        # 10k just is a deviation threshold
+        assert {$cur_mem < 10000 + $orig_mem}
+
+        # Read nothing
+        set fd [$rd channel]
+        assert_equal {} [read $fd]
+    }
+
+    test {No response for multi commands in pipeline if client output buffer limit is enforced} {
+        r config set client-output-buffer-limit {normal 100000 0 0}
+        set value [string repeat "x" 10000]
+        r set bigkey $value
+        set rd1 [redis_deferring_client]
+        set rd2 [redis_deferring_client]
+        $rd2 client setname multicommands
+        assert_equal "OK" [$rd2 read]
+        # Let redis sleep 2s firstly
+        $rd1 debug sleep 2
+        $rd1 flush
+        after 100
+
+        # Total size should be less than OS socket buffer, redis can
+        # execute all commands in this pipeline when it wakes up.
+        for {set i 0} {$i < 15} {incr i} {
+            $rd2 set $i $i
+            $rd2 get $i
+            $rd2 del $i
+            # One bigkey is 10k, total response size must be more than 100k
+            $rd2 get bigkey
+        }
+        $rd2 flush
+        after 100
+
+        # Reds must wake up if it can send reply
+        assert_equal "PONG" [r ping]
+        set clients [r client list]
+        assert_no_match "*name=multicommands*" $clients
+        set fd [$rd2 channel]
+        assert_equal {} [read $fd]
+    }
+
+    test {Execute transactions completely even if client output buffer limit is enforced} {
+        r config set client-output-buffer-limit {normal 100000 0 0}
+        # Total size of all items must be more than 100k
+        set item [string repeat "x" 1000]
+        for {set i 0} {$i < 150} {incr i} {
+            r lpush mylist2 $item
+        }
+
+        # Output buffer limit is enforced during executing transaction
+        r client setname transactionclient
+        r set k1 v1
+        r multi
+        r set k2 v2
+        r get k2
+        r lrange mylist2 0 -1
+        r set k3 v3
+        r del k1
+        catch {[r exec]} e
+        assert_match "*I/O error*" $e
+        reconnect
+        set clients [r client list]
+        assert_no_match "*name=transactionclient*" $clients
+
+        # Transactions should be executed completely
+        assert_equal {} [r get k1]
+        assert_equal "v2" [r get k2]
+        assert_equal "v3" [r get k3]
+    }
 }

From 323029baa6958781ffae5da331dfc918d66a7117 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Thu, 24 Sep 2020 11:17:53 -0400
Subject: [PATCH 0689/1098] rdb.c: handle fclose error case differently to
 avoid double fclose (#7307)

When fclose would fail, the previous implementation would have attempted to do fclose again
this can in theory lead to segfault.

other changes:
check for non-zero return value as failure rather than a specific error code.
this doesn't fix a real bug, just a minor cleanup.
---
 src/rdb.c | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/rdb.c b/src/rdb.c
index 6420e9924..73bc143f5 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1314,7 +1314,7 @@ werr: /* Write error. */
 int rdbSave(char *filename, rdbSaveInfo *rsi) {
     char tmpfile[256];
     char cwd[MAXPATHLEN]; /* Current working dir path for error messages. */
-    FILE *fp;
+    FILE *fp = NULL;
     rio rdb;
     int error = 0;
 
@@ -1343,10 +1343,11 @@ int rdbSave(char *filename, rdbSaveInfo *rsi) {
     }
 
     /* Make sure data will not remain on the OS's output buffers */
-    if (fflush(fp) == EOF) goto werr;
-    if (fsync(fileno(fp)) == -1) goto werr;
-    if (fclose(fp) == EOF) goto werr;
-
+    if (fflush(fp)) goto werr;
+    if (fsync(fileno(fp))) goto werr;
+    if (fclose(fp)) { fp = NULL; goto werr; }
+    fp = NULL;
+    
     /* Use RENAME to make sure the DB file is changed atomically only
      * if the generate DB file is ok. */
     if (rename(tmpfile,filename) == -1) {
@@ -1372,7 +1373,7 @@ int rdbSave(char *filename, rdbSaveInfo *rsi) {
 
 werr:
     serverLog(LL_WARNING,"Write error saving DB on disk: %s", strerror(errno));
-    fclose(fp);
+    if (fp) fclose(fp);
     unlink(tmpfile);
     stopSaving(0);
     return C_ERR;

From 0d62caab2113fc12077aadd469e80dd19ca78db2 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Fri, 25 Sep 2020 13:08:06 +0800
Subject: [PATCH 0690/1098] Add fsync to readSyncBulkPayload(). (#7839)

We should sync temp DB file before renaming as rdb_fsync_range does not use
flag `SYNC_FILE_RANGE_WAIT_AFTER`.

Refer to `Linux Programmer's Manual`:
SYNC_FILE_RANGE_WAIT_AFTER
    Wait upon write-out of all pages in the range after performing any write.
---
 src/replication.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/replication.c b/src/replication.c
index 445ee1970..9ed4dd117 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1739,6 +1739,17 @@ void readSyncBulkPayload(connection *conn) {
             killRDBChild();
         }
 
+        /* Make sure the new file (also used for persistence) is fully synced
+         * (not covered by earlier calls to rdb_fsync_range). */
+        if (fsync(server.repl_transfer_fd) == -1) {
+            serverLog(LL_WARNING,
+                "Failed trying to sync the temp DB to disk in "
+                "MASTER <-> REPLICA synchronization: %s",
+                strerror(errno));
+            cancelReplicationHandshake(1);
+            return;
+        }
+
         /* Rename rdb like renaming rewrite aof asynchronously. */
         int old_rdb_fd = open(server.rdb_filename,O_RDONLY|O_NONBLOCK);
         if (rename(server.repl_transfer_tmpfile,server.rdb_filename) == -1) {

From c30bd02c9d251c63b49ee77a6fd456fb82ff1382 Mon Sep 17 00:00:00 2001
From: Uri Shachar <ushachar@apache.org>
Date: Fri, 25 Sep 2020 12:55:45 +0300
Subject: [PATCH 0691/1098] Fix config rewrite file handling to make it really
 atomic (#7824)

Make sure we handle short writes correctly, sync to disk after writing  and use
rename to make sure the replacement is actually atomic.
In any case of failure old configuration will remain in place.

Also, add some additional logging to make it easier to diagnose rewrite problems.
---
 src/config.c | 82 +++++++++++++++++++++++++++-------------------------
 1 file changed, 42 insertions(+), 40 deletions(-)

diff --git a/src/config.c b/src/config.c
index be0494ccd..fbdef05b1 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1552,60 +1552,62 @@ void rewriteConfigRemoveOrphaned(struct rewriteConfigState *state) {
     dictReleaseIterator(di);
 }
 
-/* This function overwrites the old configuration file with the new content.
- *
- * 1) The old file length is obtained.
- * 2) If the new content is smaller, padding is added.
- * 3) A single write(2) call is used to replace the content of the file.
- * 4) Later the file is truncated to the length of the new content.
- *
- * This way we are sure the file is left in a consistent state even if the
- * process is stopped between any of the four operations.
+/* This function replaces the old configuration file with the new content
+ * in an atomic manner.
  *
  * The function returns 0 on success, otherwise -1 is returned and errno
- * set accordingly. */
+ * is set accordingly. */
 int rewriteConfigOverwriteFile(char *configfile, sds content) {
-    int retval = 0;
-    int fd = open(configfile,O_RDWR|O_CREAT,0644);
-    int content_size = sdslen(content), padding = 0;
-    struct stat sb;
-    sds content_padded;
+    int fd = -1;
+    int retval = -1;
+    char tmp_conffile[PATH_MAX];
+    const char *tmp_suffix = ".XXXXXX";
+    size_t offset = 0;
+    ssize_t written_bytes = 0;
 
-    /* 1) Open the old file (or create a new one if it does not
-     *    exist), get the size. */
-    if (fd == -1) return -1; /* errno set by open(). */
-    if (fstat(fd,&sb) == -1) {
-        close(fd);
-        return -1; /* errno set by fstat(). */
+    int tmp_path_len = snprintf(tmp_conffile, sizeof(tmp_conffile), "%s%s", configfile, tmp_suffix);
+    if (tmp_path_len <= 0 || (unsigned int)tmp_path_len >= sizeof(tmp_conffile)) {
+        serverLog(LL_WARNING, "Config file full path is too long");
+        errno = ENAMETOOLONG;
+        return retval;
     }
 
-    /* 2) Pad the content at least match the old file size. */
-    content_padded = sdsdup(content);
-    if (content_size < sb.st_size) {
-        /* If the old file was bigger, pad the content with
-         * a newline plus as many "#" chars as required. */
-        padding = sb.st_size - content_size;
-        content_padded = sdsgrowzero(content_padded,sb.st_size);
-        content_padded[content_size] = '\n';
-        memset(content_padded+content_size+1,'#',padding-1);
+#ifdef _GNU_SOURCE
+    fd = mkostemp(tmp_conffile, O_CLOEXEC);
+#else
+    /* There's a theoretical chance here to leak the FD if a module thread forks & execv in the middle */
+    fd = mkstemp(tmp_conffile);
+#endif
+
+    if (fd == -1) {
+        serverLog(LL_WARNING, "Could not create tmp config file (%s)", strerror(errno));
+        return retval;
     }
 
-    /* 3) Write the new content using a single write(2). */
-    if (write(fd,content_padded,strlen(content_padded)) == -1) {
-        retval = -1;
-        goto cleanup;
+    while (offset < sdslen(content)) {
+         written_bytes = write(fd, content + offset, sdslen(content) - offset);
+         if (written_bytes <= 0) {
+             if (errno == EINTR) continue; /* FD is blocking, no other retryable errors */
+             serverLog(LL_WARNING, "Failed after writing (%ld) bytes to tmp config file (%s)", offset, strerror(errno));
+             goto cleanup;
+         }
+         offset+=written_bytes;
     }
 
-    /* 4) Truncate the file to the right length if we used padding. */
-    if (padding) {
-        if (ftruncate(fd,content_size) == -1) {
-            /* Non critical error... */
-        }
+    if (fsync(fd))
+        serverLog(LL_WARNING, "Could not sync tmp config file to disk (%s)", strerror(errno));
+    else if (fchmod(fd, 0644) == -1)
+        serverLog(LL_WARNING, "Could not chmod config file (%s)", strerror(errno));
+    else if (rename(tmp_conffile, configfile) == -1)
+        serverLog(LL_WARNING, "Could not rename tmp config file (%s)", strerror(errno));
+    else {
+        retval = 0;
+        serverLog(LL_DEBUG, "Rewritten config file (%s) successfully", configfile);
     }
 
 cleanup:
-    sdsfree(content_padded);
     close(fd);
+    if (retval) unlink(tmp_conffile);
     return retval;
 }
 

From f1863a1fe760610cebfd1c121623a3c12a79d600 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuancode@163.com>
Date: Fri, 25 Sep 2020 21:25:47 +0800
Subject: [PATCH 0692/1098] Set 'loading' and 'shutdown_asap' to volatile
 sig_atomic_t type (#7845)

We may access and modify these two variables in signal handler function,
to guarantee them async-signal-safe, so we should set them to volatile
sig_atomic_t type.

It doesn't look like this could have caused any real issue, and it seems that
signals are handled in main thread on most platforms. But we want to follow C
and POSIX standard in signal handler function.
---
 src/server.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/server.h b/src/server.h
index 5c266f61d..b920e1f4c 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1072,7 +1072,7 @@ struct redisServer {
     dict *orig_commands;        /* Command table before command renaming. */
     aeEventLoop *el;
     redisAtomic unsigned int lruclock; /* Clock for LRU eviction */
-    int shutdown_asap;          /* SHUTDOWN needed ASAP */
+    volatile sig_atomic_t shutdown_asap; /* SHUTDOWN needed ASAP */
     int activerehashing;        /* Incremental rehash in serverCron() */
     int active_defrag_running;  /* Active defragmentation running (holds current scan aggressiveness) */
     char *pidfile;              /* PID file path */
@@ -1129,7 +1129,7 @@ struct redisServer {
     long long events_processed_while_blocked; /* processEventsWhileBlocked() */
 
     /* RDB / AOF loading information */
-    int loading;                /* We are loading data from disk if true */
+    volatile sig_atomic_t loading; /* We are loading data from disk if true */
     off_t loading_total_bytes;
     off_t loading_loaded_bytes;
     time_t loading_start_time;

From c9f00bcce2ce7a59a815642674a12c940bbe2207 Mon Sep 17 00:00:00 2001
From: Wang Yuan <duaianzhou@sina.com>
Date: Sun, 27 Sep 2020 17:35:16 +0800
Subject: [PATCH 0693/1098] Don't support Gopher if enable io threads to read
 queries (#7851)

There's currently an issue with IO threads and gopher (issuing lookupKey from within the thread).
simply fix is to just not support it for now.
---
 redis.conf       | 8 +++++---
 src/networking.c | 5 +++--
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/redis.conf b/redis.conf
index 424dfa09a..31655e92e 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1538,8 +1538,11 @@ notify-keyspace-events ""
 #
 # So use the 'requirepass' option to protect your instance.
 #
-# To enable Gopher support uncomment the following line and set
-# the option from no (the default) to yes.
+# Note that Gopher is not currently supported when 'io-threads-do-reads'
+# is enabled.
+#
+# To enable Gopher support, uncomment the following line and set the option
+# from no (the default) to yes.
 #
 # gopher-enabled no
 
@@ -1879,4 +1882,3 @@ jemalloc-bg-thread yes
 #
 # Set bgsave child process to cpu affinity 1,10,11
 # bgsave_cpulist 1,10-11
-
diff --git a/src/networking.c b/src/networking.c
index 4cbddb660..7e97e4d31 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1919,8 +1919,9 @@ void processInputBuffer(client *c) {
         if (c->reqtype == PROTO_REQ_INLINE) {
             if (processInlineBuffer(c) != C_OK) break;
             /* If the Gopher mode and we got zero or one argument, process
-             * the request in Gopher mode. */
-            if (server.gopher_enabled &&
+             * the request in Gopher mode. To avoid data race, Redis won't
+             * support Gopher if enable io threads to read queries. */
+            if (server.gopher_enabled && !server.io_threads_do_reads &&
                 ((c->argc == 1 && ((char*)(c->argv[0]->ptr))[0] == '/') ||
                   c->argc == 0))
             {

From a295770e32b4bd71ff560c5ec7bc5c0ad12bf068 Mon Sep 17 00:00:00 2001
From: caozb <1162650653@qq.com>
Date: Sun, 27 Sep 2020 20:40:07 +0800
Subject: [PATCH 0694/1098] ignore slaveof no one in redis.conf (#7842)

when slaveof config is "no one", reset any pre-existing config and resume.

also solve a memory leak if slaveof appears twice.
and fail loading if port number is out of range or not an integer.

Co-authored-by: caozhengbin <caozb@yidingyun.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/config.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/config.c b/src/config.c
index fbdef05b1..3d5a79eab 100644
--- a/src/config.c
+++ b/src/config.c
@@ -473,8 +473,17 @@ void loadServerConfigFromString(char *config) {
         } else if ((!strcasecmp(argv[0],"slaveof") ||
                     !strcasecmp(argv[0],"replicaof")) && argc == 3) {
             slaveof_linenum = linenum;
+            sdsfree(server.masterhost);
+            if (!strcasecmp(argv[1], "no") && !strcasecmp(argv[2], "one")) {
+                server.masterhost = NULL;
+                continue;
+            }
             server.masterhost = sdsnew(argv[1]);
-            server.masterport = atoi(argv[2]);
+            char *ptr;
+            server.masterport = strtol(argv[2], &ptr, 10);
+            if (server.masterport < 0 || server.masterport > 65535 || *ptr != '\0') {
+                err = "Invalid master port"; goto loaderr;
+            }
             server.repl_state = REPL_STATE_CONNECT;
         } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) {
             if (strlen(argv[1]) > CONFIG_AUTHPASS_MAX_LEN) {

From 8aa083bd283963f60c46eb6156e7aacca3c853c3 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 27 Sep 2020 17:13:33 +0300
Subject: [PATCH 0695/1098] Fix new obuf-limits tests to work with TLS (#7848)

Also stabilize new shutdown tests on slow machines (valgrind)
---
 tests/unit/obuf-limits.tcl | 20 ++++++++++++++------
 tests/unit/shutdown.tcl    | 22 +++++++++++++++-------
 2 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/tests/unit/obuf-limits.tcl b/tests/unit/obuf-limits.tcl
index 20ba32fd5..456d3ac82 100644
--- a/tests/unit/obuf-limits.tcl
+++ b/tests/unit/obuf-limits.tcl
@@ -99,6 +99,8 @@ start_server {tags {"obuf-limits"}} {
         assert_equal {} [read $fd]
     }
 
+    # Note: This test assumes that what's written with one write, will be read by redis in one read.
+    # this assumption is wrong, but seem to work empirically (for now)
     test {No response for multi commands in pipeline if client output buffer limit is enforced} {
         r config set client-output-buffer-limit {normal 100000 0 0}
         set value [string repeat "x" 10000]
@@ -107,20 +109,26 @@ start_server {tags {"obuf-limits"}} {
         set rd2 [redis_deferring_client]
         $rd2 client setname multicommands
         assert_equal "OK" [$rd2 read]
-        # Let redis sleep 2s firstly
-        $rd1 debug sleep 2
+
+        # Let redis sleep 1s firstly
+        $rd1 debug sleep 1
         $rd1 flush
         after 100
 
+        # Create a pipeline of commands that will be processed in one socket read.
+        # It is important to use one write, in TLS mode independant writes seem
+        # to wait for response from the server.
         # Total size should be less than OS socket buffer, redis can
         # execute all commands in this pipeline when it wakes up.
+        set buf ""
         for {set i 0} {$i < 15} {incr i} {
-            $rd2 set $i $i
-            $rd2 get $i
-            $rd2 del $i
+            append buf "set $i $i\r\n"
+            append buf "get $i\r\n"
+            append buf "del $i\r\n"
             # One bigkey is 10k, total response size must be more than 100k
-            $rd2 get bigkey
+            append buf "get bigkey\r\n"
         }
+        $rd2 write $buf
         $rd2 flush
         after 100
 
diff --git a/tests/unit/shutdown.tcl b/tests/unit/shutdown.tcl
index 21ea8545d..f48eadc50 100644
--- a/tests/unit/shutdown.tcl
+++ b/tests/unit/shutdown.tcl
@@ -30,20 +30,28 @@ start_server {tags {"shutdown"}} {
         for {set i 0} {$i < 20} {incr i} {
             r set $i $i
         }
-        # It will cost 2s(20 * 100ms) to dump rdb
+        # It will cost 2s (20 * 100ms) to dump rdb
         r config set rdb-key-save-delay 100000
         
         set pid [s process_id]
         set temp_rdb [file join [lindex [r config get dir] 1] temp-${pid}.rdb]
 
+        # trigger a shutdown which will save an rdb
         exec kill -SIGINT $pid
-        after 100
-        # Temp rdb must be existed
-        assert {[file exists $temp_rdb]}
+        # Wait for creation of temp rdb
+        wait_for_condition 50 10 {
+            [file exists $temp_rdb]
+        } else {
+            fail "Can't trigger rdb save on shutdown"
+        }
 
-        # Temp rdb file must be deleted
+        # Insist on immediate shutdown, temp rdb file must be deleted
         exec kill -SIGINT $pid
-        after 100
-        assert {![file exists $temp_rdb]}
+        # wait for the rdb file to be deleted
+        wait_for_condition 50 10 {
+            ![file exists $temp_rdb]
+        } else {
+            fail "Can't trigger rdb save on shutdown"
+        }
     }
 }

From c2e5546071f18b002fd7e0c222645baef1596089 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Mon, 28 Sep 2020 16:27:26 +0800
Subject: [PATCH 0696/1098] Normalize sds test mechanism together with some
 compile warnings. (#7854)

---
 src/Makefile |  4 ----
 src/sds.c    | 19 +++++++++----------
 src/server.c |  2 ++
 3 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/src/Makefile b/src/Makefile
index f6e5f3e3f..9b8083c06 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -362,10 +362,6 @@ lcov:
 	@geninfo -o redis.info .
 	@genhtml --legend -o lcov-html redis.info
 
-test-sds: sds.c sds.h
-	$(REDIS_CC) sds.c zmalloc.c -DSDS_TEST_MAIN $(FINAL_LIBS) -o /tmp/sds_test
-	/tmp/sds_test
-
 .PHONY: lcov
 
 bench: $(REDIS_BENCHMARK_NAME)
diff --git a/src/sds.c b/src/sds.c
index ce9cd7032..e30b33b6c 100644
--- a/src/sds.c
+++ b/src/sds.c
@@ -1121,13 +1121,16 @@ void *sds_malloc(size_t size) { return s_malloc(size); }
 void *sds_realloc(void *ptr, size_t size) { return s_realloc(ptr,size); }
 void sds_free(void *ptr) { s_free(ptr); }
 
-#if defined(SDS_TEST_MAIN)
+#ifdef REDIS_TEST
 #include <stdio.h>
 #include "testhelp.h"
 #include "limits.h"
 
 #define UNUSED(x) (void)(x)
-int sdsTest(void) {
+int sdsTest(int argc, char **argv) {
+    UNUSED(argc);
+    UNUSED(argv);
+
     {
         sds x = sdsnew("foo"), y;
 
@@ -1254,7 +1257,8 @@ int sdsTest(void) {
         {
             unsigned int oldfree;
             char *p;
-            int step = 10, j, i;
+            int i;
+            size_t step = 10, j;
 
             sdsfree(x);
             sdsfree(y);
@@ -1264,7 +1268,7 @@ int sdsTest(void) {
             /* Run the test a few times in order to hit the first two
              * SDS header types. */
             for (i = 0; i < 10; i++) {
-                int oldlen = sdslen(x);
+                size_t oldlen = sdslen(x);
                 x = sdsMakeRoomFor(x,step);
                 int type = x[-1]&SDS_TYPE_MASK;
 
@@ -1272,6 +1276,7 @@ int sdsTest(void) {
                 if (type != SDS_TYPE_5) {
                     test_cond("sdsMakeRoomFor() free", sdsavail(x) >= step);
                     oldfree = sdsavail(x);
+                    UNUSED(oldfree);
                 }
                 p = x+oldlen;
                 for (j = 0; j < step; j++) {
@@ -1290,9 +1295,3 @@ int sdsTest(void) {
     return 0;
 }
 #endif
-
-#ifdef SDS_TEST_MAIN
-int main(void) {
-    return sdsTest();
-}
-#endif
diff --git a/src/server.c b/src/server.c
index 24238c595..956cad587 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5247,6 +5247,8 @@ int main(int argc, char **argv) {
             return crc64Test(argc, argv);
         } else if (!strcasecmp(argv[2], "zmalloc")) {
             return zmalloc_test(argc, argv);
+        } else if (!strcasecmp(argv[2], "sds")) {
+            return sdsTest(argc, argv);
         }
 
         return -1; /* test not found */

From ce8bfc56ad002a3b276249d62038fae4488eae61 Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Tue, 29 Sep 2020 06:49:35 +0100
Subject: [PATCH 0697/1098] getting rss size implementation for netbsd (#7293)

---
 src/zmalloc.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/zmalloc.c b/src/zmalloc.c
index cefa39aa8..01cb0f917 100644
--- a/src/zmalloc.c
+++ b/src/zmalloc.c
@@ -304,6 +304,26 @@ size_t zmalloc_get_rss(void) {
 
     return 0L;
 }
+#elif defined(__NetBSD__)
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#include <unistd.h>
+
+size_t zmalloc_get_rss(void) {
+    struct kinfo_proc2 info;
+    size_t infolen = sizeof(info);
+    int mib[6];
+    mib[0] = CTL_KERN;
+    mib[1] = KERN_PROC;
+    mib[2] = KERN_PROC_PID;
+    mib[3] = getpid();
+    mib[4] = sizeof(info);
+    mib[5] = 1;
+    if (sysctl(mib, 4, &info, &infolen, NULL, 0) == 0)
+        return (size_t)info.p_vm_rssize;
+
+    return 0L;
+}
 #else
 size_t zmalloc_get_rss(void) {
     /* If we can't get the RSS in an OS-specific way for this system just

From 49383a4af74bbcca1c194fca458198412ff1820c Mon Sep 17 00:00:00 2001
From: Gavrie Philipson <gavrie@philipson.co.il>
Date: Tue, 29 Sep 2020 13:10:08 +0300
Subject: [PATCH 0698/1098] Fix typo in module API docs (#7861)

---
 src/module.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/module.c b/src/module.c
index b76b57698..2de7f60ed 100644
--- a/src/module.c
+++ b/src/module.c
@@ -2938,8 +2938,8 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) {
 }
 
 /* Get fields from an hash value. This function is called using a variable
- * number of arguments, alternating a field name (as a StringRedisModule
- * pointer) with a pointer to a StringRedisModule pointer, that is set to the
+ * number of arguments, alternating a field name (as a RedisModuleString
+ * pointer) with a pointer to a RedisModuleString pointer, that is set to the
  * value of the field if the field exists, or NULL if the field does not exist.
  * At the end of the field/value-ptr pairs, NULL must be specified as last
  * argument to signal the end of the arguments in the variadic function.

From f971a5d8ed58547f01325611c0189745a904c9cc Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Tue, 29 Sep 2020 13:52:13 +0100
Subject: [PATCH 0699/1098] Add support for Haiku OS (#7435)

---
 src/Makefile  | 7 +++++++
 src/config.h  | 4 ++++
 src/memtest.c | 5 +++++
 3 files changed, 16 insertions(+)

diff --git a/src/Makefile b/src/Makefile
index 9b8083c06..54dda1ae9 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -155,6 +155,12 @@ else
 ifeq ($(uname_S),NetBSD)
 	# NetBSD
 	FINAL_LIBS+= -lpthread -lexecinfo
+else
+ifeq ($(uname_S),Haiku)
+	# Haiku
+	FINAL_CFLAGS+= -DBSD_SOURCE
+	FINAL_LDFLAGS+= -lbsd -lnetwork
+	FINAL_LIBS+= -lpthread
 else
 	# All the other OSes (notably Linux)
 	FINAL_LDFLAGS+= -rdynamic
@@ -168,6 +174,7 @@ endif
 endif
 endif
 endif
+endif
 # Include paths to dependencies
 FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram
 
diff --git a/src/config.h b/src/config.h
index d391508fa..320837b7e 100644
--- a/src/config.h
+++ b/src/config.h
@@ -124,6 +124,10 @@
 #define USE_SETPROCTITLE
 #endif
 
+#if defined(__HAIKU__)
+#define ESOCKTNOSUPPORT 0
+#endif
+
 #if ((defined __linux && defined(__GLIBC__)) || defined __APPLE__)
 #define USE_SETPROCTITLE
 #define INIT_SETPROCTITLE_REPLACEMENT
diff --git a/src/memtest.c b/src/memtest.c
index a455430f5..cb4d35e83 100644
--- a/src/memtest.c
+++ b/src/memtest.c
@@ -347,10 +347,15 @@ void memtest_alloc_and_test(size_t megabytes, int passes) {
 }
 
 void memtest(size_t megabytes, int passes) {
+#if !defined(__HAIKU__)
     if (ioctl(1, TIOCGWINSZ, &ws) == -1) {
         ws.ws_col = 80;
         ws.ws_row = 20;
     }
+#else
+    ws.ws_col = 80;
+    ws.ws_row = 20;
+#endif
     memtest_alloc_and_test(megabytes,passes);
     printf("\nYour memory passed this test.\n");
     printf("Please if you are still in doubt use the following two tools:\n");

From ef33252c434ff25991ff75e2c7e4646bc9ad5ff7 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 29 Sep 2020 17:03:47 +0300
Subject: [PATCH 0700/1098] warning: comparison between signed and unsigned
 integer in 32bit build (#7838)

---
 src/listpack.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/listpack.c b/src/listpack.c
index 075552ccb..7e2da9b74 100644
--- a/src/listpack.c
+++ b/src/listpack.c
@@ -768,10 +768,10 @@ unsigned char *lpSeek(unsigned char *lp, long index) {
     if (numele != LP_HDR_NUMELE_UNKNOWN) {
         if (index < 0) index = (long)numele+index;
         if (index < 0) return NULL; /* Index still < 0 means out of range. */
-        if (index >= numele) return NULL; /* Out of range the other side. */
+        if ((long)index >= numele) return NULL; /* Out of range the other side. */
         /* We want to scan right-to-left if the element we are looking for
          * is past the half of the listpack. */
-        if (index > numele/2) {
+        if ((long)index > numele/2) {
             forward = 0;
             /* Right to left scanning always expects a negative index. Convert
              * our index to negative form. */

From 448c435b1b5ff747d0d84c49a23435a83f0d40ac Mon Sep 17 00:00:00 2001
From: YoongHM <yoonghm@users.noreply.github.com>
Date: Tue, 29 Sep 2020 22:10:54 +0800
Subject: [PATCH 0701/1098] Fix compilation warnings in Lua and jemalloc
 dependencies (#7785)

- The argument `u` in for `ar` is ignored (and generates warnings since `D` became the default.
  All it does is avoid updating unchanged objects (shouldn't have any impact on our build)
- Enable `LUA_USE_MKSTEMP` to force the use of `mkstemp()` instead of `tmpname()` (which is dead
  code in redis anyway).
- Remove unused variable `c` in `f_parser()`
- Removed misleadingly indented space in `luaL_loadfile()` and ``addfield()`

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 deps/Makefile              | 4 ++--
 deps/jemalloc/configure.ac | 2 +-
 deps/lua/src/lauxlib.c     | 2 +-
 deps/lua/src/ldo.c         | 2 +-
 deps/lua/src/ltablib.c     | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/deps/Makefile b/deps/Makefile
index b90bbd2b7..ff16ee9e1 100644
--- a/deps/Makefile
+++ b/deps/Makefile
@@ -69,13 +69,13 @@ ifeq ($(uname_S),SunOS)
 	LUA_CFLAGS= -D__C99FEATURES__=1
 endif
 
-LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' $(CFLAGS)
+LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' -DLUA_USE_MKSTEMP $(CFLAGS)
 LUA_LDFLAGS+= $(LDFLAGS)
 # lua's Makefile defines AR="ar rcu", which is unusual, and makes it more
 # challenging to cross-compile lua (and redis).  These defines make it easier
 # to fit redis into cross-compilation environments, which typically set AR.
 AR=ar
-ARFLAGS=rcu
+ARFLAGS=rc
 
 lua: .make-prerequisites
 	@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
diff --git a/deps/jemalloc/configure.ac b/deps/jemalloc/configure.ac
index a6a08db08..afa5df800 100644
--- a/deps/jemalloc/configure.ac
+++ b/deps/jemalloc/configure.ac
@@ -512,7 +512,7 @@ CTARGET='-o $@'
 LDTARGET='-o $@'
 TEST_LD_MODE=
 EXTRA_LDFLAGS=
-ARFLAGS='crus'
+ARFLAGS='crs'
 AROUT=' $@'
 CC_MM=1
 
diff --git a/deps/lua/src/lauxlib.c b/deps/lua/src/lauxlib.c
index 10f14e2c0..751f1e887 100644
--- a/deps/lua/src/lauxlib.c
+++ b/deps/lua/src/lauxlib.c
@@ -575,7 +575,7 @@ LUALIB_API int luaL_loadfile (lua_State *L, const char *filename) {
     if (lf.f == NULL) return errfile(L, "reopen", fnameindex);
     /* skip eventual `#!...' */
    while ((c = getc(lf.f)) != EOF && c != LUA_SIGNATURE[0]) ;
-    lf.extraline = 0;
+   lf.extraline = 0;
   }
   ungetc(c, lf.f);
   status = lua_load(L, getF, &lf, lua_tostring(L, -1));
diff --git a/deps/lua/src/ldo.c b/deps/lua/src/ldo.c
index 939940a4c..e641dfcd9 100644
--- a/deps/lua/src/ldo.c
+++ b/deps/lua/src/ldo.c
@@ -493,7 +493,7 @@ static void f_parser (lua_State *L, void *ud) {
   Proto *tf;
   Closure *cl;
   struct SParser *p = cast(struct SParser *, ud);
-  int c = luaZ_lookahead(p->z);
+  luaZ_lookahead(p->z);
   luaC_checkGC(L);
   tf = (luaY_parser)(L, p->z,
                                                              &p->buff, p->name);
diff --git a/deps/lua/src/ltablib.c b/deps/lua/src/ltablib.c
index b6d9cb4ac..0bdac7f6c 100644
--- a/deps/lua/src/ltablib.c
+++ b/deps/lua/src/ltablib.c
@@ -137,7 +137,7 @@ static void addfield (lua_State *L, luaL_Buffer *b, int i) {
   if (!lua_isstring(L, -1))
     luaL_error(L, "invalid value (%s) at index %d in table for "
                   LUA_QL("concat"), luaL_typename(L, -1), i);
-    luaL_addvalue(b);
+  luaL_addvalue(b);
 }
 
 

From 1591e3479d46e7e21a98ae685fab2ccab076d094 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 29 Sep 2020 20:48:21 +0300
Subject: [PATCH 0702/1098] TLS: Do not require CA config if not used. (#7862)

The tls-ca-cert or tls-ca-cert-dir configuration parameters are only
used when Redis needs to authenticate peer certificates, in one of these
scenarios:

1. Incoming clients or replicas, with `tls-auth-clients` enabled.
2. A replica authenticating the master's peer certificate.
3. Cluster nodes authenticating other nodes when establishing the bus
   protocol connection.
---
 src/tls.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/tls.c b/src/tls.c
index d3173fab0..0a9e07895 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -167,8 +167,9 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
         goto error;
     }
 
-    if (!ctx_config->ca_cert_file && !ctx_config->ca_cert_dir) {
-        serverLog(LL_WARNING, "Either tls-ca-cert-file or tls-ca-cert-dir must be configured!");
+    if (((server.tls_auth_clients != TLS_CLIENT_AUTH_NO) || server.tls_cluster || server.tls_replication) &&
+            !ctx_config->ca_cert_file && !ctx_config->ca_cert_dir) {
+        serverLog(LL_WARNING, "Either tls-ca-cert-file or tls-ca-cert-dir must be specified when tls-cluster, tls-replication or tls-auth-clients are enabled!");
         goto error;
     }
 
@@ -235,7 +236,8 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
         goto error;
     }
     
-    if (SSL_CTX_load_verify_locations(ctx, ctx_config->ca_cert_file, ctx_config->ca_cert_dir) <= 0) {
+    if ((ctx_config->ca_cert_file || ctx_config->ca_cert_dir) &&
+        SSL_CTX_load_verify_locations(ctx, ctx_config->ca_cert_file, ctx_config->ca_cert_dir) <= 0) {
         ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf));
         serverLog(LL_WARNING, "Failed to configure CA certificate(s) file/directory: %s", errbuf);
         goto error;

From cfccfbd6f4ad23c1a9feb44ebb91da9cd2a09734 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 18 Aug 2020 17:13:09 +0300
Subject: [PATCH 0703/1098] Module API: Fail ineffective auth calls.

The client pointed to by the module context may in some cases be a fake
client. RM_Authenticate*() calls in this case would be ineffective but
appear to succeed, and this change fails them to make it easier to catch
such cases.
---
 src/module.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/module.c b/src/module.c
index 2de7f60ed..615a5223f 100644
--- a/src/module.c
+++ b/src/module.c
@@ -5639,6 +5639,11 @@ static int authenticateClientWithUser(RedisModuleCtx *ctx, user *user, RedisModu
         return REDISMODULE_ERR;
     }
 
+    /* Avoid settings which are meaningless and will be lost */
+    if (!ctx->client || (ctx->client->flags & CLIENT_MODULE)) {
+        return REDISMODULE_ERR;
+    }
+
     moduleNotifyUserChanged(ctx->client);
 
     ctx->client->user = user;

From 67b43f75e23ba7a3915bc87e7edaa306fd1eee65 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 18 Aug 2020 17:16:08 +0300
Subject: [PATCH 0704/1098] Modules: expose real client on conn events.

When REDISMODULE_EVENT_CLIENT_CHANGE events are delivered, modules may
want to mutate the client state (e.g. perform authentication).

This change links the module context with the real client rather than a
fake client for these events.
---
 src/module.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/module.c b/src/module.c
index 615a5223f..fb8f83e9c 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7264,6 +7264,7 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) {
      * cheap if there are no registered modules. */
     if (listLength(RedisModule_EventListeners) == 0) return;
 
+    int real_client_used = 0;
     listIter li;
     listNode *ln;
     listRewind(RedisModule_EventListeners,&li);
@@ -7273,7 +7274,15 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) {
             RedisModuleCtx ctx = REDISMODULE_CTX_INIT;
             ctx.module = el->module;
 
-            if (ModulesInHooks == 0) {
+            if (eid == REDISMODULE_EVENT_CLIENT_CHANGE) {
+                /* In the case of client changes, we're pushing the real client
+                 * so the event handler can mutate it if needed. For example,
+                 * to change its authentication state in a way that does not
+                 * depend on specific commands executed later.
+                 */
+                ctx.client = (client *) data;
+                real_client_used = 1;
+            } else if (ModulesInHooks == 0) {
                 ctx.client = moduleFreeContextReusedClient;
             } else {
                 ctx.client = createClient(NULL);
@@ -7326,7 +7335,7 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) {
             el->module->in_hook--;
             ModulesInHooks--;
 
-            if (ModulesInHooks != 0) freeClient(ctx.client);
+            if (ModulesInHooks != 0 && !real_client_used) freeClient(ctx.client);
             moduleFreeContext(&ctx);
         }
     }

From 8fb89a572892e600146bd933c4f8f99d46a519c7 Mon Sep 17 00:00:00 2001
From: nitaicaro <42576749+nitaicaro@users.noreply.github.com>
Date: Wed, 30 Sep 2020 19:52:01 +0300
Subject: [PATCH 0705/1098] =?UTF-8?q?Fixed=20Tracking=20test=20=E2=80=9CTh?=
 =?UTF-8?q?e=20other=20connection=20is=20able=20to=20get=20invalidations?=
 =?UTF-8?q?=E2=80=9D=20(#7871)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

PROBLEM:

[$rd1 read] reads invalidation messages one by one, so it's never going to see the second invalidation message produced after INCR b, whether or not it exists. Adding another read will block incase no invalidation message is produced.

FIX:

We switch the order of "INCR a" and "INCR b" - now "INCR b" comes first. We still only read the first invalidation message produces. If an invalidation message is wrongly produces for b - then it will be produced before that of a, since "INCR b" comes before "INCR a".

Co-authored-by: Nitai Caro <caronita@amazon.com>
---
 tests/unit/tracking.tcl | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index 0332fa726..839b894ea 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -16,9 +16,10 @@ start_server {tags {"tracking"}} {
 
     test {The other connection is able to get invalidations} {
         r SET a 1
+        r SET b 1
         r GET a
-        r INCR a
-        r INCR b ; # This key should not be notified, since it wasn't fetched.
+        r INCR b ; # This key should not be notified, since it wasn't fetched. 
+        r INCR a 
         set keys [lindex [$rd1 read] 2]
         assert {[llength $keys] == 1}
         assert {[lindex $keys 0] eq {a}}

From b8187d39fb3eb1cbd2b0346a1e39b68f6ab3029c Mon Sep 17 00:00:00 2001
From: Rafi Einstein <rafi@redislabs.com>
Date: Thu, 1 Oct 2020 10:56:23 +0300
Subject: [PATCH 0706/1098] Makefile: enable program suffixes via PROG_SUFFIX
 (#7868)

---
 README.md    |  4 ++++
 src/Makefile | 12 ++++++------
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index e1cd3b87a..d892f4881 100644
--- a/README.md
+++ b/README.md
@@ -45,6 +45,10 @@ as libsystemd-dev on Debian/Ubuntu or systemd-devel on CentOS) and run:
 
     % make USE_SYSTEMD=yes
 
+To append a suffix to Redis program names, use:
+
+    % make PROG_SUFFIX="-alt"
+
 You can run a 32 bit Redis binary using:
 
     % make 32bit
diff --git a/src/Makefile b/src/Makefile
index 54dda1ae9..daa201d73 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -253,15 +253,15 @@ QUIET_LINK = @printf '    %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(EN
 QUIET_INSTALL = @printf '    %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) 1>&2;
 endif
 
-REDIS_SERVER_NAME=redis-server
-REDIS_SENTINEL_NAME=redis-sentinel
+REDIS_SERVER_NAME=redis-server$(PROG_SUFFIX)
+REDIS_SENTINEL_NAME=redis-sentinel$(PROG_SUFFIX)
 REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o
-REDIS_CLI_NAME=redis-cli
+REDIS_CLI_NAME=redis-cli$(PROG_SUFFIX)
 REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o
-REDIS_BENCHMARK_NAME=redis-benchmark
+REDIS_BENCHMARK_NAME=redis-benchmark$(PROG_SUFFIX)
 REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o monotonic.o
-REDIS_CHECK_RDB_NAME=redis-check-rdb
-REDIS_CHECK_AOF_NAME=redis-check-aof
+REDIS_CHECK_RDB_NAME=redis-check-rdb$(PROG_SUFFIX)
+REDIS_CHECK_AOF_NAME=redis-check-aof$(PROG_SUFFIX)
 
 all: $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME)
 	@echo ""

From dc803d25a6f5eb8c41b6afed913cbcd769e4b377 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 1 Oct 2020 11:27:45 +0300
Subject: [PATCH 0707/1098] Fix crash in script timeout during AOF loading
 (#7870)

---
 src/networking.c         | 12 ++++++++----
 tests/unit/scripting.tcl | 39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 7e97e4d31..6d9bf960f 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1547,16 +1547,20 @@ void resetClient(client *c) {
  *    path, it is not really released, but only marked for later release. */
 void protectClient(client *c) {
     c->flags |= CLIENT_PROTECTED;
-    connSetReadHandler(c->conn,NULL);
-    connSetWriteHandler(c->conn,NULL);
+    if (c->conn) {
+        connSetReadHandler(c->conn,NULL);
+        connSetWriteHandler(c->conn,NULL);
+    }
 }
 
 /* This will undo the client protection done by protectClient() */
 void unprotectClient(client *c) {
     if (c->flags & CLIENT_PROTECTED) {
         c->flags &= ~CLIENT_PROTECTED;
-        connSetReadHandler(c->conn,readQueryFromClient);
-        if (clientHasPendingReplies(c)) clientInstallWriteHandler(c);
+        if (c->conn) {
+            connSetReadHandler(c->conn,readQueryFromClient);
+            if (clientHasPendingReplies(c)) clientInstallWriteHandler(c);
+        }
     }
 }
 
diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl
index 3283edc66..6bcba4c3f 100644
--- a/tests/unit/scripting.tcl
+++ b/tests/unit/scripting.tcl
@@ -430,6 +430,45 @@ start_server {tags {"scripting"}} {
         set res
     } {102}
 
+    test {EVAL timeout from AOF} {
+        # generate a long running script that is propagated to the AOF as script
+        # make sure that the script times out during loading
+        r config set appendonly no
+        r config set aof-use-rdb-preamble no
+        r config set lua-replicate-commands no
+        r flushall
+        r config set appendonly yes
+        wait_for_condition 50 100 {
+            [s aof_rewrite_in_progress] == 0
+        } else {
+            fail "AOF rewrite can't complete after CONFIG SET appendonly yes."
+        }
+        r config set lua-time-limit 1
+        set rd [redis_deferring_client]
+        set start [clock clicks -milliseconds]
+        $rd eval {redis.call('set',KEYS[1],'y'); for i=1,1500000 do redis.call('ping') end return 'ok'} 1 x
+        $rd flush
+        after 100
+        catch {r ping} err
+        assert_match {BUSY*} $err
+        $rd read
+        set elapsed [expr [clock clicks -milliseconds]-$start]
+        if {$::verbose} { puts "script took $elapsed milliseconds" }
+        set start [clock clicks -milliseconds]
+        $rd debug loadaof
+        $rd flush
+        after 100
+        catch {r ping} err
+        assert_match {LOADING*} $err
+        $rd read
+        set elapsed [expr [clock clicks -milliseconds]-$start]
+        if {$::verbose} { puts "loading took $elapsed milliseconds" }
+        $rd close
+        r get x
+    } {y}
+    r config set aof-use-rdb-preamble yes
+    r config set lua-replicate-commands yes
+
     test {We can call scripts rewriting client->argv from Lua} {
         r del myset
         r sadd myset a b c

From eb6241a3dd3ee27610922e6bccd44603a9b845e8 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 1 Oct 2020 11:30:22 +0300
Subject: [PATCH 0708/1098] Include internal sds fragmentation in MEMORY
 reporting (#7864)

The MEMORY command is used for debugging memory usage, so it should include internal
fragmentation, same as used_memory
---
 src/object.c | 12 ++++++------
 src/server.c |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/object.c b/src/object.c
index 1380fffa5..1b8852f18 100644
--- a/src/object.c
+++ b/src/object.c
@@ -787,7 +787,7 @@ size_t objectComputeSize(robj *o, size_t sample_size) {
         if(o->encoding == OBJ_ENCODING_INT) {
             asize = sizeof(*o);
         } else if(o->encoding == OBJ_ENCODING_RAW) {
-            asize = sdsAllocSize(o->ptr)+sizeof(*o);
+            asize = sdsZmallocSize(o->ptr)+sizeof(*o);
         } else if(o->encoding == OBJ_ENCODING_EMBSTR) {
             asize = sdslen(o->ptr)+2+sizeof(*o);
         } else {
@@ -815,7 +815,7 @@ size_t objectComputeSize(robj *o, size_t sample_size) {
             asize = sizeof(*o)+sizeof(dict)+(sizeof(struct dictEntry*)*dictSlots(d));
             while((de = dictNext(di)) != NULL && samples < sample_size) {
                 ele = dictGetKey(de);
-                elesize += sizeof(struct dictEntry) + sdsAllocSize(ele);
+                elesize += sizeof(struct dictEntry) + sdsZmallocSize(ele);
                 samples++;
             }
             dictReleaseIterator(di);
@@ -837,7 +837,7 @@ size_t objectComputeSize(robj *o, size_t sample_size) {
                     (sizeof(struct dictEntry*)*dictSlots(d))+
                     zmalloc_size(zsl->header);
             while(znode != NULL && samples < sample_size) {
-                elesize += sdsAllocSize(znode->ele);
+                elesize += sdsZmallocSize(znode->ele);
                 elesize += sizeof(struct dictEntry) + zmalloc_size(znode);
                 samples++;
                 znode = znode->level[0].forward;
@@ -856,7 +856,7 @@ size_t objectComputeSize(robj *o, size_t sample_size) {
             while((de = dictNext(di)) != NULL && samples < sample_size) {
                 ele = dictGetKey(de);
                 ele2 = dictGetVal(de);
-                elesize += sdsAllocSize(ele) + sdsAllocSize(ele2);
+                elesize += sdsZmallocSize(ele) + sdsZmallocSize(ele2);
                 elesize += sizeof(struct dictEntry);
                 samples++;
             }
@@ -996,7 +996,7 @@ struct redisMemOverhead *getMemoryOverheadData(void) {
 
     mem = 0;
     if (server.aof_state != AOF_OFF) {
-        mem += sdsalloc(server.aof_buf);
+        mem += sdsZmallocSize(server.aof_buf);
         mem += aofRewriteBufferSize();
     }
     mh->aof_buffer = mem;
@@ -1312,7 +1312,7 @@ NULL
             return;
         }
         size_t usage = objectComputeSize(dictGetVal(de),samples);
-        usage += sdsAllocSize(dictGetKey(de));
+        usage += sdsZmallocSize(dictGetKey(de));
         usage += sizeof(dictEntry);
         addReplyLongLong(c,usage);
     } else if (!strcasecmp(c->argv[1]->ptr,"stats") && c->argc == 2) {
diff --git a/src/server.c b/src/server.c
index 956cad587..2f7aa8713 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1626,7 +1626,7 @@ int clientsCronTrackClientsMemUsage(client *c) {
     size_t mem = 0;
     int type = getClientType(c);
     mem += getClientOutputBufferMemoryUsage(c);
-    mem += sdsAllocSize(c->querybuf);
+    mem += sdsZmallocSize(c->querybuf);
     mem += sizeof(client);
     /* Now that we have the memory used by the client, remove the old
      * value from the old category, and add it back. */

From 3945a321779a0be52e556aa11babb3fe15883a78 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 2 Oct 2020 08:19:44 +0300
Subject: [PATCH 0709/1098] performance and memory reporting improvement - sds
 take control of it's internal frag (#7875)

This commit has two aspects:
1) improve memory reporting for all the places that use sdsAllocSize to compute
   memory used by a string, in this case it'll include the internal fragmentation.
2) reduce the need for realloc calls by making the sds implicitly take over
   the internal fragmentation of the block it allocated.
---
 src/networking.c |  6 ++--
 src/sds.c        | 38 ++++++++++++++++-----
 src/sdsalloc.h   |  3 ++
 src/zmalloc.c    | 86 +++++++++++++++++++++++++++++++++++++++++++++++-
 src/zmalloc.h    |  8 +++--
 5 files changed, 127 insertions(+), 14 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 6d9bf960f..bfb65f5a1 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -293,7 +293,7 @@ void _addReplyProtoToList(client *c, const char *s, size_t len) {
         size_t size = len < PROTO_REPLY_CHUNK_BYTES? PROTO_REPLY_CHUNK_BYTES: len;
         tail = zmalloc(size + sizeof(clientReplyBlock));
         /* take over the allocation's internal fragmentation */
-        tail->size = zmalloc_usable(tail) - sizeof(clientReplyBlock);
+        tail->size = zmalloc_usable_size(tail) - sizeof(clientReplyBlock);
         tail->used = len;
         memcpy(tail->buf, s, len);
         listAddNodeTail(c->reply, tail);
@@ -489,7 +489,7 @@ void trimReplyUnusedTailSpace(client *c) {
         tail = zrealloc(tail, tail->used + sizeof(clientReplyBlock));
         /* take over the allocation's internal fragmentation (at least for
          * memory usage tracking) */
-        tail->size = zmalloc_usable(tail) - sizeof(clientReplyBlock);
+        tail->size = zmalloc_usable_size(tail) - sizeof(clientReplyBlock);
         c->reply_bytes = c->reply_bytes + tail->size - old_size;
         listNodeValue(ln) = tail;
     }
@@ -541,7 +541,7 @@ void setDeferredAggregateLen(client *c, void *node, long length, char prefix) {
         /* Create a new node */
         clientReplyBlock *buf = zmalloc(lenstr_len + sizeof(clientReplyBlock));
         /* Take over the allocation's internal fragmentation */
-        buf->size = zmalloc_usable(buf) - sizeof(clientReplyBlock);
+        buf->size = zmalloc_usable_size(buf) - sizeof(clientReplyBlock);
         buf->used = lenstr_len;
         memcpy(buf->buf, lenstr, lenstr_len);
         listNodeValue(ln) = buf;
diff --git a/src/sds.c b/src/sds.c
index e30b33b6c..4cbbabfbb 100644
--- a/src/sds.c
+++ b/src/sds.c
@@ -73,6 +73,20 @@ static inline char sdsReqType(size_t string_size) {
 #endif
 }
 
+static inline size_t sdsTypeMaxSize(char type) {
+    if (type == SDS_TYPE_5)
+        return (1<<5) - 1;
+    if (type == SDS_TYPE_8)
+        return (1<<8) - 1;
+    if (type == SDS_TYPE_16)
+        return (1<<16) - 1;
+#if (LONG_MAX == LLONG_MAX)
+    if (type == SDS_TYPE_32)
+        return (1ll<<32) - 1;
+#endif
+    return -1; /* this is equivalent to the max SDS_TYPE_64 or SDS_TYPE_32 */
+}
+
 /* Create a new sds string with the content specified by the 'init' pointer
  * and 'initlen'.
  * If NULL is used for 'init' the string is initialized with zero bytes.
@@ -95,8 +109,9 @@ sds sdsnewlen(const void *init, size_t initlen) {
     if (type == SDS_TYPE_5 && initlen == 0) type = SDS_TYPE_8;
     int hdrlen = sdsHdrSize(type);
     unsigned char *fp; /* flags pointer. */
+    size_t usable;
 
-    sh = s_malloc(hdrlen+initlen+1);
+    sh = s_malloc_usable(hdrlen+initlen+1, &usable);
     if (sh == NULL) return NULL;
     if (init==SDS_NOINIT)
         init = NULL;
@@ -104,6 +119,9 @@ sds sdsnewlen(const void *init, size_t initlen) {
         memset(sh, 0, hdrlen+initlen+1);
     s = (char*)sh+hdrlen;
     fp = ((unsigned char*)s)-1;
+    usable = usable-hdrlen-1;
+    if (usable > sdsTypeMaxSize(type))
+        usable = sdsTypeMaxSize(type);
     switch(type) {
         case SDS_TYPE_5: {
             *fp = type | (initlen << SDS_TYPE_BITS);
@@ -112,28 +130,28 @@ sds sdsnewlen(const void *init, size_t initlen) {
         case SDS_TYPE_8: {
             SDS_HDR_VAR(8,s);
             sh->len = initlen;
-            sh->alloc = initlen;
+            sh->alloc = usable;
             *fp = type;
             break;
         }
         case SDS_TYPE_16: {
             SDS_HDR_VAR(16,s);
             sh->len = initlen;
-            sh->alloc = initlen;
+            sh->alloc = usable;
             *fp = type;
             break;
         }
         case SDS_TYPE_32: {
             SDS_HDR_VAR(32,s);
             sh->len = initlen;
-            sh->alloc = initlen;
+            sh->alloc = usable;
             *fp = type;
             break;
         }
         case SDS_TYPE_64: {
             SDS_HDR_VAR(64,s);
             sh->len = initlen;
-            sh->alloc = initlen;
+            sh->alloc = usable;
             *fp = type;
             break;
         }
@@ -207,6 +225,7 @@ sds sdsMakeRoomFor(sds s, size_t addlen) {
     size_t len, newlen;
     char type, oldtype = s[-1] & SDS_TYPE_MASK;
     int hdrlen;
+    size_t usable;
 
     /* Return ASAP if there is enough space left. */
     if (avail >= addlen) return s;
@@ -228,13 +247,13 @@ sds sdsMakeRoomFor(sds s, size_t addlen) {
 
     hdrlen = sdsHdrSize(type);
     if (oldtype==type) {
-        newsh = s_realloc(sh, hdrlen+newlen+1);
+        newsh = s_realloc_usable(sh, hdrlen+newlen+1, &usable);
         if (newsh == NULL) return NULL;
         s = (char*)newsh+hdrlen;
     } else {
         /* Since the header size changes, need to move the string forward,
          * and can't use realloc */
-        newsh = s_malloc(hdrlen+newlen+1);
+        newsh = s_malloc_usable(hdrlen+newlen+1, &usable);
         if (newsh == NULL) return NULL;
         memcpy((char*)newsh+hdrlen, s, len+1);
         s_free(sh);
@@ -242,7 +261,10 @@ sds sdsMakeRoomFor(sds s, size_t addlen) {
         s[-1] = type;
         sdssetlen(s, len);
     }
-    sdssetalloc(s, newlen);
+    usable = usable-hdrlen-1;
+    if (usable > sdsTypeMaxSize(type))
+        usable = sdsTypeMaxSize(type);
+    sdssetalloc(s, usable);
     return s;
 }
 
diff --git a/src/sdsalloc.h b/src/sdsalloc.h
index c04ff2a0a..725ad79e3 100644
--- a/src/sdsalloc.h
+++ b/src/sdsalloc.h
@@ -43,5 +43,8 @@
 #define s_malloc zmalloc
 #define s_realloc zrealloc
 #define s_free zfree
+#define s_malloc_usable zmalloc_usable
+#define s_realloc_usable zrealloc_usable
+#define s_free_usable zfree_usable
 
 #endif
diff --git a/src/zmalloc.c b/src/zmalloc.c
index 01cb0f917..4a7f2028e 100644
--- a/src/zmalloc.c
+++ b/src/zmalloc.c
@@ -99,6 +99,21 @@ void *zmalloc(size_t size) {
 #endif
 }
 
+/* Similar to zmalloc, '*usable' is set to the usable size. */
+void *zmalloc_usable(size_t size, size_t *usable) {
+    void *ptr = malloc(size+PREFIX_SIZE);
+
+    if (!ptr) zmalloc_oom_handler(size);
+#ifdef HAVE_MALLOC_SIZE
+    update_zmalloc_stat_alloc(*usable = zmalloc_size(ptr));
+    return ptr;
+#else
+    *((size_t*)ptr) = *usable = size;
+    update_zmalloc_stat_alloc(size+PREFIX_SIZE);
+    return (char*)ptr+PREFIX_SIZE;
+#endif
+}
+
 /* Allocation and free functions that bypass the thread cache
  * and go straight to the allocator arena bins.
  * Currently implemented only for jemalloc. Used for online defragmentation. */
@@ -131,6 +146,21 @@ void *zcalloc(size_t size) {
 #endif
 }
 
+/* Similar to zcalloc, '*usable' is set to the usable size. */
+void *zcalloc_usable(size_t size, size_t *usable) {
+    void *ptr = calloc(1, size+PREFIX_SIZE);
+
+    if (!ptr) zmalloc_oom_handler(size);
+#ifdef HAVE_MALLOC_SIZE
+    update_zmalloc_stat_alloc(*usable = zmalloc_size(ptr));
+    return ptr;
+#else
+    *((size_t*)ptr) = *usable = size;
+    update_zmalloc_stat_alloc(size+PREFIX_SIZE);
+    return (char*)ptr+PREFIX_SIZE;
+#endif
+}
+
 void *zrealloc(void *ptr, size_t size) {
 #ifndef HAVE_MALLOC_SIZE
     void *realptr;
@@ -164,6 +194,41 @@ void *zrealloc(void *ptr, size_t size) {
 #endif
 }
 
+/* Similar to zrealloc, '*usable' is set to the new usable size. */
+void *zrealloc_usable(void *ptr, size_t size, size_t *usable) {
+#ifndef HAVE_MALLOC_SIZE
+    void *realptr;
+#endif
+    size_t oldsize;
+    void *newptr;
+
+    if (size == 0 && ptr != NULL) {
+        zfree(ptr);
+        *usable = 0;
+        return NULL;
+    }
+    if (ptr == NULL) return zmalloc_usable(size, usable);
+#ifdef HAVE_MALLOC_SIZE
+    oldsize = zmalloc_size(ptr);
+    newptr = realloc(ptr,size);
+    if (!newptr) zmalloc_oom_handler(size);
+
+    update_zmalloc_stat_free(oldsize);
+    update_zmalloc_stat_alloc(*usable = zmalloc_size(newptr));
+    return newptr;
+#else
+    realptr = (char*)ptr-PREFIX_SIZE;
+    oldsize = *((size_t*)realptr);
+    newptr = realloc(realptr,size+PREFIX_SIZE);
+    if (!newptr) zmalloc_oom_handler(size);
+
+    *((size_t*)newptr) = *usable = size;
+    update_zmalloc_stat_free(oldsize);
+    update_zmalloc_stat_alloc(size);
+    return (char*)newptr+PREFIX_SIZE;
+#endif
+}
+
 /* Provide zmalloc_size() for systems where this function is not provided by
  * malloc itself, given that in that case we store a header with this
  * information as the first bytes of every allocation. */
@@ -176,7 +241,7 @@ size_t zmalloc_size(void *ptr) {
     if (size&(sizeof(long)-1)) size += sizeof(long)-(size&(sizeof(long)-1));
     return size+PREFIX_SIZE;
 }
-size_t zmalloc_usable(void *ptr) {
+size_t zmalloc_usable_size(void *ptr) {
     return zmalloc_size(ptr)-PREFIX_SIZE;
 }
 #endif
@@ -199,6 +264,25 @@ void zfree(void *ptr) {
 #endif
 }
 
+/* Similar to zfree, '*usable' is set to the usable size being freed. */
+void zfree_usable(void *ptr, size_t *usable) {
+#ifndef HAVE_MALLOC_SIZE
+    void *realptr;
+    size_t oldsize;
+#endif
+
+    if (ptr == NULL) return;
+#ifdef HAVE_MALLOC_SIZE
+    update_zmalloc_stat_free(*usable = zmalloc_size(ptr));
+    free(ptr);
+#else
+    realptr = (char*)ptr-PREFIX_SIZE;
+    *usable = oldsize = *((size_t*)realptr);
+    update_zmalloc_stat_free(oldsize+PREFIX_SIZE);
+    free(realptr);
+#endif
+}
+
 char *zstrdup(const char *s) {
     size_t l = strlen(s)+1;
     char *p = zmalloc(l);
diff --git a/src/zmalloc.h b/src/zmalloc.h
index b136a910d..df63253ef 100644
--- a/src/zmalloc.h
+++ b/src/zmalloc.h
@@ -81,6 +81,10 @@ void *zmalloc(size_t size);
 void *zcalloc(size_t size);
 void *zrealloc(void *ptr, size_t size);
 void zfree(void *ptr);
+void *zmalloc_usable(size_t size, size_t *usable);
+void *zcalloc_usable(size_t size, size_t *usable);
+void *zrealloc_usable(void *ptr, size_t size, size_t *usable);
+void zfree_usable(void *ptr, size_t *usable);
 char *zstrdup(const char *s);
 size_t zmalloc_used_memory(void);
 void zmalloc_set_oom_handler(void (*oom_handler)(size_t));
@@ -100,9 +104,9 @@ void *zmalloc_no_tcache(size_t size);
 
 #ifndef HAVE_MALLOC_SIZE
 size_t zmalloc_size(void *ptr);
-size_t zmalloc_usable(void *ptr);
+size_t zmalloc_usable_size(void *ptr);
 #else
-#define zmalloc_usable(p) zmalloc_size(p)
+#define zmalloc_usable_size(p) zmalloc_size(p)
 #endif
 
 #ifdef REDIS_TEST

From 66ee45b65cc8165ba86e98192920374dcd06d943 Mon Sep 17 00:00:00 2001
From: Nykolas Laurentino de Lima <nykolas.lima@gmail.com>
Date: Fri, 2 Oct 2020 09:07:19 -0300
Subject: [PATCH 0710/1098] Add GET parameter to SET command (#7852)

Add optional GET parameter to SET command in order to set a new value to
a key and retrieve the old key value. With this change we can deprecate
`GETSET` command and use only the SET command with the GET parameter.
---
 src/t_string.c             | 23 ++++++++++++++++++-----
 tests/unit/type/string.tcl | 21 +++++++++++++++++++++
 2 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index 6c54a6fe9..a881b86f6 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -44,9 +44,9 @@ static int checkStringLength(client *c, long long size) {
 
 /* The setGenericCommand() function implements the SET operation with different
  * options and variants. This function is called in order to implement the
- * following commands: SET, SETEX, PSETEX, SETNX.
+ * following commands: SET, SETEX, PSETEX, SETNX, GETSET.
  *
- * 'flags' changes the behavior of the command (NX or XX, see below).
+ * 'flags' changes the behavior of the command (NX, XX or GET, see below).
  *
  * 'expire' represents an expire to set in form of a Redis object as passed
  * by the user. It is interpreted according to the specified 'unit'.
@@ -64,6 +64,7 @@ static int checkStringLength(client *c, long long size) {
 #define OBJ_SET_EX (1<<2)          /* Set if time in seconds is given */
 #define OBJ_SET_PX (1<<3)          /* Set if time in ms in given */
 #define OBJ_SET_KEEPTTL (1<<4)     /* Set and keep the ttl */
+#define OBJ_SET_GET (1<<5)         /* Set if want to get key before set */
 
 void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire, int unit, robj *ok_reply, robj *abort_reply) {
     long long milliseconds = 0; /* initialized to avoid any harmness warning */
@@ -84,16 +85,23 @@ void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire,
         addReply(c, abort_reply ? abort_reply : shared.null[c->resp]);
         return;
     }
+
+    if (flags & OBJ_SET_GET) {
+       getGenericCommand(c);
+    }
+
     genericSetKey(c,c->db,key,val,flags & OBJ_SET_KEEPTTL,1);
     server.dirty++;
     if (expire) setExpire(c,c->db,key,mstime()+milliseconds);
     notifyKeyspaceEvent(NOTIFY_STRING,"set",key,c->db->id);
     if (expire) notifyKeyspaceEvent(NOTIFY_GENERIC,
         "expire",key,c->db->id);
-    addReply(c, ok_reply ? ok_reply : shared.ok);
+    if (!(flags & OBJ_SET_GET)) {
+        addReply(c, ok_reply ? ok_reply : shared.ok);
+    }
 }
 
-/* SET key value [NX] [XX] [KEEPTTL] [EX <seconds>] [PX <milliseconds>] */
+/* SET key value [NX] [XX] [KEEPTTL] [GET] [EX <seconds>] [PX <milliseconds>] */
 void setCommand(client *c) {
     int j;
     robj *expire = NULL;
@@ -106,7 +114,7 @@ void setCommand(client *c) {
 
         if ((a[0] == 'n' || a[0] == 'N') &&
             (a[1] == 'x' || a[1] == 'X') && a[2] == '\0' &&
-            !(flags & OBJ_SET_XX))
+            !(flags & OBJ_SET_XX) && !(flags & OBJ_SET_GET))
         {
             flags |= OBJ_SET_NX;
         } else if ((a[0] == 'x' || a[0] == 'X') &&
@@ -114,6 +122,11 @@ void setCommand(client *c) {
                    !(flags & OBJ_SET_NX))
         {
             flags |= OBJ_SET_XX;
+        } else if ((a[0] == 'g' || a[0] == 'G') &&
+                   (a[1] == 'e' || a[1] == 'E') &&
+                   (a[2] == 't' || a[2] == 'T') && a[3] == '\0' &&
+                   !(flags & OBJ_SET_NX)) {
+            flags |= OBJ_SET_GET;
         } else if (!strcasecmp(c->argv[j]->ptr,"KEEPTTL") &&
                    !(flags & OBJ_SET_EX) && !(flags & OBJ_SET_PX))
         {
diff --git a/tests/unit/type/string.tcl b/tests/unit/type/string.tcl
index 8126cdee8..5058acec4 100644
--- a/tests/unit/type/string.tcl
+++ b/tests/unit/type/string.tcl
@@ -394,6 +394,27 @@ start_server {tags {"string"}} {
         list $v1 $v2 [r get foo]
     } {{} OK 2}
 
+    test {Extended SET GET option} {
+        r del foo
+        r set foo bar
+        set old_value [r set foo bar2 GET]
+        set new_value [r get foo]
+        list $old_value $new_value
+    } {bar bar2}
+
+    test {Extended SET GET option with no previous value} {
+        r del foo
+        set old_value [r set foo bar GET]
+        set new_value [r get foo]
+        list $old_value $new_value
+    } {{} bar}
+
+    test {Extended SET GET with NX option should result in syntax err} {
+      catch {r set foo bar NX GET} err1
+      catch {r set foo bar NX GET} err2
+      list $err1 $err2
+    } {*syntax err* *syntax err*}
+
     test {Extended SET EX option} {
         r del foo
         r set foo bar ex 10

From 6418d767e86b6fd98ba79253108b2e53966a04b3 Mon Sep 17 00:00:00 2001
From: DvirDukhan <dvir@redislabs.com>
Date: Sun, 4 Oct 2020 17:18:17 +0300
Subject: [PATCH 0711/1098] redis-cli add control on raw format line delimiter
 (#7841)

Adding -D option for redis-cli to control newline between command
responses in raw mode.

Also removing cleanup code before calling exit, just in order
to avoid adding more adding more cleanup code (redis doesn't
bother to release allocations before exit anyway)

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/redis-cli.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 60bdca6de..d79fb35f8 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -239,6 +239,7 @@ static struct config {
     int output; /* output mode, see OUTPUT_* defines */
     int push_output; /* Should we display spontaneous PUSH replies */
     sds mb_delim;
+    sds cmd_delim;
     char prompt[128];
     char *eval;
     int eval_ldb;
@@ -1236,7 +1237,7 @@ static sds cliFormatReply(redisReply *reply, int mode, int verbatim) {
         out = cliFormatReplyTTY(reply, "");
     } else if (mode == OUTPUT_RAW) {
         out = cliFormatReplyRaw(reply);
-        out = sdscatlen(out, "\n", 1);
+        out = sdscatsds(out, config.cmd_delim);
     } else if (mode == OUTPUT_CSV) {
         out = cliFormatReplyCSV(reply);
         out = sdscatlen(out, "\n", 1);
@@ -1604,6 +1605,9 @@ static int parseOptions(int argc, char **argv) {
         } else if (!strcmp(argv[i],"-d") && !lastarg) {
             sdsfree(config.mb_delim);
             config.mb_delim = sdsnew(argv[++i]);
+        } else if (!strcmp(argv[i],"-D") && !lastarg) {
+            sdsfree(config.cmd_delim);
+            config.cmd_delim = sdsnew(argv[++i]);
         } else if (!strcmp(argv[i],"--verbose")) {
             config.verbose = 1;
         } else if (!strcmp(argv[i],"--cluster") && !lastarg) {
@@ -1807,7 +1811,8 @@ static void usage(void) {
 "  -n <db>            Database number.\n"
 "  -3                 Start session in RESP3 protocol mode.\n"
 "  -x                 Read last argument from STDIN.\n"
-"  -d <delimiter>     Multi-bulk delimiter in for raw formatting (default: \\n).\n"
+"  -d <delimiter>     Delimiter between response bulks for raw formatting (default: \\n).\n"
+"  -D <delimiter>     Delimiter between responses for raw formatting (default: \\n).\n"
 "  -c                 Enable cluster mode (follow -ASK and -MOVED redirections).\n"
 #ifdef USE_OPENSSL
 "  --tls              Establish a secure TLS connection.\n"
@@ -5443,8 +5448,6 @@ static void clusterManagerMode(clusterManagerCommandProc *proc) {
     exit(0);
 cluster_manager_err:
     freeClusterManager();
-    sdsfree(config.hostip);
-    sdsfree(config.mb_delim);
     exit(1);
 }
 
@@ -8204,6 +8207,7 @@ int main(int argc, char **argv) {
         config.push_output = 1;
     }
     config.mb_delim = sdsnew("\n");
+    config.cmd_delim = sdsnew("\n");
 
     firstarg = parseOptions(argc,argv);
     argc -= firstarg;
@@ -8227,8 +8231,6 @@ int main(int argc, char **argv) {
     if (CLUSTER_MANAGER_MODE()) {
         clusterManagerCommandProc *proc = validateClusterManagerCommand();
         if (!proc) {
-            sdsfree(config.hostip);
-            sdsfree(config.mb_delim);
             exit(1);
         }
         clusterManagerMode(proc);

From f87a47dc97d69efcdc14dc162f41016b986e7d22 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 4 Oct 2020 18:10:21 +0300
Subject: [PATCH 0712/1098] fix new build warning about getGenericCommand
 (#7882)

---
 src/t_string.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/t_string.c b/src/t_string.c
index a881b86f6..188f27dc0 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -30,6 +30,9 @@
 #include "server.h"
 #include <math.h> /* isnan(), isinf() */
 
+/* Forward declarations */
+int getGenericCommand(client *c);
+
 /*-----------------------------------------------------------------------------
  * String Commands
  *----------------------------------------------------------------------------*/

From bea40e6a41e31a52e9e6efee77ea5a4bd873b759 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 5 Oct 2020 11:15:36 +0300
Subject: [PATCH 0713/1098] memory reporting of clients argv (#7874)

track and report memory used by clients argv.
this is very usaful in case clients started sending a command and didn't
complete it. in which case the first args of the command are already
trimmed from the query buffer.

in an effort to avoid cache misses and overheads while keeping track of
these, i avoid calling sdsZmallocSize and instead use the sdslen /
bulk-len which can at least give some insight into the problem.

This memory is now added to the total clients memory usage, as well as
the client list.
---
 src/aof.c                    |  2 ++
 src/networking.c             | 51 +++++++++++++++++++++++++++++++++---
 src/server.c                 |  4 ++-
 src/server.h                 |  1 +
 tests/unit/introspection.tcl |  2 +-
 5 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index c51fdbeff..b1c7595b5 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -669,6 +669,7 @@ struct client *createAOFClient(void) {
     c->querybuf_peak = 0;
     c->argc = 0;
     c->argv = NULL;
+    c->argv_len_sum = 0;
     c->bufpos = 0;
     c->flags = 0;
     c->btype = BLOCKED_NONE;
@@ -694,6 +695,7 @@ void freeFakeClientArgv(struct client *c) {
     for (j = 0; j < c->argc; j++)
         decrRefCount(c->argv[j]);
     zfree(c->argv);
+    c->argv_len_sum = 0;
 }
 
 void freeFakeClient(struct client *c) {
diff --git a/src/networking.c b/src/networking.c
index bfb65f5a1..fcb612e0b 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -48,7 +48,7 @@ size_t sdsZmallocSize(sds s) {
 }
 
 /* Return the amount of memory used by the sds string at object->ptr
- * for a string object. */
+ * for a string object. This includes internal fragmentation. */
 size_t getStringObjectSdsUsedMemory(robj *o) {
     serverAssertWithInfo(NULL,o,o->type == OBJ_STRING);
     switch(o->encoding) {
@@ -58,6 +58,17 @@ size_t getStringObjectSdsUsedMemory(robj *o) {
     }
 }
 
+/* Return the length of a string object.
+ * This does NOT includes internal fragmentation or sds unused space. */
+size_t getStringObjectLen(robj *o) {
+    serverAssertWithInfo(NULL,o,o->type == OBJ_STRING);
+    switch(o->encoding) {
+    case OBJ_ENCODING_RAW: return sdslen(o->ptr);
+    case OBJ_ENCODING_EMBSTR: return sdslen(o->ptr);
+    default: return 0; /* Just integer encoding for now. */
+    }
+}
+
 /* Client.reply list dup and free methods. */
 void *dupClientReplyValue(void *o) {
     clientReplyBlock *old = o;
@@ -117,6 +128,7 @@ client *createClient(connection *conn) {
     c->reqtype = 0;
     c->argc = 0;
     c->argv = NULL;
+    c->argv_len_sum = 0;
     c->cmd = c->lastcmd = NULL;
     c->user = DefaultUser;
     c->multibulklen = 0;
@@ -1052,6 +1064,7 @@ static void freeClientArgv(client *c) {
         decrRefCount(c->argv[j]);
     c->argc = 0;
     c->cmd = NULL;
+    c->argv_len_sum = 0;
 }
 
 /* Close all the slaves connections. This is useful in chained replication
@@ -1284,6 +1297,7 @@ void freeClient(client *c) {
      * and finally release the client structure itself. */
     if (c->name) decrRefCount(c->name);
     zfree(c->argv);
+    c->argv_len_sum = 0;
     freeClientMultiState(c);
     sdsfree(c->peerid);
     zfree(c);
@@ -1630,12 +1644,14 @@ int processInlineBuffer(client *c) {
     if (argc) {
         if (c->argv) zfree(c->argv);
         c->argv = zmalloc(sizeof(robj*)*argc);
+        c->argv_len_sum = 0;
     }
 
     /* Create redis objects for all arguments. */
     for (c->argc = 0, j = 0; j < argc; j++) {
         c->argv[c->argc] = createObject(OBJ_STRING,argv[j]);
         c->argc++;
+        c->argv_len_sum += sdslen(argv[j]);
     }
     zfree(argv);
     return C_OK;
@@ -1727,6 +1743,7 @@ int processMultibulkBuffer(client *c) {
         /* Setup argv array on client structure */
         if (c->argv) zfree(c->argv);
         c->argv = zmalloc(sizeof(robj*)*c->multibulklen);
+        c->argv_len_sum = 0;
     }
 
     serverAssertWithInfo(c,NULL,c->multibulklen > 0);
@@ -1799,6 +1816,7 @@ int processMultibulkBuffer(client *c) {
                 sdslen(c->querybuf) == (size_t)(c->bulklen+2))
             {
                 c->argv[c->argc++] = createObject(OBJ_STRING,c->querybuf);
+                c->argv_len_sum += c->bulklen;
                 sdsIncrLen(c->querybuf,-2); /* remove CRLF */
                 /* Assume that if we saw a fat argument we'll see another one
                  * likely... */
@@ -1807,6 +1825,7 @@ int processMultibulkBuffer(client *c) {
             } else {
                 c->argv[c->argc++] =
                     createStringObject(c->querybuf+c->qb_pos,c->bulklen);
+                c->argv_len_sum += c->bulklen;
                 c->qb_pos += c->bulklen+2;
             }
             c->bulklen = -1;
@@ -2129,8 +2148,21 @@ sds catClientInfoString(sds s, client *client) {
         if (connHasWriteHandler(client->conn)) *p++ = 'w';
     }
     *p = '\0';
+
+    /* Compute the total memory consumed by this client. */
+    size_t obufmem = getClientOutputBufferMemoryUsage(client);
+    size_t total_mem = obufmem;
+    total_mem += zmalloc_size(client); /* includes client->buf */
+    total_mem += sdsZmallocSize(client->querybuf);
+    /* For efficiency (less work keeping track of the argv memory), it doesn't include the used memory
+     * i.e. unused sds space and internal fragmentation, just the string length. but this is enough to
+     * spot problematic clients. */
+    total_mem += client->argv_len_sum;
+    if (client->argv)
+        total_mem += zmalloc_size(client->argv);
+
     return sdscatfmt(s,
-        "id=%U addr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s user=%s",
+        "id=%U addr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U argv-mem=%U obl=%U oll=%U omem=%U tot-mem=%U events=%s cmd=%s user=%s",
         (unsigned long long) client->id,
         getClientPeerId(client),
         connGetInfo(client->conn, conninfo, sizeof(conninfo)),
@@ -2144,9 +2176,11 @@ sds catClientInfoString(sds s, client *client) {
         (client->flags & CLIENT_MULTI) ? client->mstate.count : -1,
         (unsigned long long) sdslen(client->querybuf),
         (unsigned long long) sdsavail(client->querybuf),
+        (unsigned long long) client->argv_len_sum,
         (unsigned long long) client->bufpos,
         (unsigned long long) listLength(client->reply),
-        (unsigned long long) getClientOutputBufferMemoryUsage(client),
+        (unsigned long long) obufmem, /* should not include client->buf since we want to see 0 for static clients. */
+        (unsigned long long) total_mem,
         events,
         client->lastcmd ? client->lastcmd->name : "NULL",
         client->user ? client->user->name : "(superuser)");
@@ -2684,6 +2718,10 @@ void rewriteClientCommandVector(client *c, int argc, ...) {
     /* Replace argv and argc with our new versions. */
     c->argv = argv;
     c->argc = argc;
+    c->argv_len_sum = 0;
+    for (j = 0; j < c->argc; j++)
+        if (c->argv[j])
+            c->argv_len_sum += getStringObjectLen(c->argv[j]);
     c->cmd = lookupCommandOrOriginal(c->argv[0]->ptr);
     serverAssertWithInfo(c,NULL,c->cmd != NULL);
     va_end(ap);
@@ -2691,10 +2729,15 @@ void rewriteClientCommandVector(client *c, int argc, ...) {
 
 /* Completely replace the client command vector with the provided one. */
 void replaceClientCommandVector(client *c, int argc, robj **argv) {
+    int j;
     freeClientArgv(c);
     zfree(c->argv);
     c->argv = argv;
     c->argc = argc;
+    c->argv_len_sum = 0;
+    for (j = 0; j < c->argc; j++)
+        if (c->argv[j])
+            c->argv_len_sum += getStringObjectLen(c->argv[j]);
     c->cmd = lookupCommandOrOriginal(c->argv[0]->ptr);
     serverAssertWithInfo(c,NULL,c->cmd != NULL);
 }
@@ -2719,6 +2762,8 @@ void rewriteClientCommandArgument(client *c, int i, robj *newval) {
         c->argv[i] = NULL;
     }
     oldval = c->argv[i];
+    if (oldval) c->argv_len_sum -= getStringObjectLen(oldval);
+    if (newval) c->argv_len_sum += getStringObjectLen(newval);
     c->argv[i] = newval;
     incrRefCount(newval);
     if (oldval) decrRefCount(oldval);
diff --git a/src/server.c b/src/server.c
index 2f7aa8713..d8aaef06d 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1627,7 +1627,9 @@ int clientsCronTrackClientsMemUsage(client *c) {
     int type = getClientType(c);
     mem += getClientOutputBufferMemoryUsage(c);
     mem += sdsZmallocSize(c->querybuf);
-    mem += sizeof(client);
+    mem += zmalloc_size(c);
+    mem += c->argv_len_sum;
+    if (c->argv) mem += zmalloc_size(c->argv);
     /* Now that we have the memory used by the client, remove the old
      * value from the old category, and add it back. */
     server.stat_clients_type_memory[c->client_cron_last_memory_type] -=
diff --git a/src/server.h b/src/server.h
index b920e1f4c..114034962 100644
--- a/src/server.h
+++ b/src/server.h
@@ -802,6 +802,7 @@ typedef struct client {
     size_t querybuf_peak;   /* Recent (100ms or more) peak of querybuf size. */
     int argc;               /* Num of arguments of current command. */
     robj **argv;            /* Arguments of current command. */
+    size_t argv_len_sum;    /* Sum of lengths of objects in argv list. */
     struct redisCommand *cmd, *lastcmd;  /* Last command executed. */
     user *user;             /* User associated with this connection. If the
                                user is set to NULL the connection can do
diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl
index c8bc976cf..2d50b417b 100644
--- a/tests/unit/introspection.tcl
+++ b/tests/unit/introspection.tcl
@@ -1,7 +1,7 @@
 start_server {tags {"introspection"}} {
     test {CLIENT LIST} {
         r client list
-    } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=26 qbuf-free=* obl=0 oll=0 omem=0 events=r cmd=client*}
+    } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=26 qbuf-free=* argv-mem=* obl=0 oll=0 omem=0 tot-mem=* events=r cmd=client*}
 
     test {MONITOR can log executed commands} {
         set rd [redis_deferring_client]

From 57b8e00695d037971342cbcc43734a990bfaf081 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Mon, 5 Oct 2020 06:37:20 -0400
Subject: [PATCH 0714/1098] support sentinel myid subcommand (#7858)

---
 src/sentinel.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/sentinel.c b/src/sentinel.c
index 36644a63c..a346a2547 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -3050,6 +3050,7 @@ void sentinelCommand(client *c) {
 "MASTER <master-name> -- Show the state and info of the specified master.",
 "REPLICAS <master-name> -- Show a list of replicas for this master and their state.",
 "SENTINELS <master-name> -- Show a list of Sentinel instances for this master and their state.",
+"MYID -- Show Current Sentinel Id",
 "IS-MASTER-DOWN-BY-ADDR <ip> <port> <current-epoch> <runid> -- Check if the master specified by ip:port is down from current Sentinel's point of view.",
 "GET-MASTER-ADDR-BY-NAME <master-name> -- Return the ip and port number of the master with that name.",
 "RESET <pattern> -- Reset masters for specific master name matching this pattern.",
@@ -3097,6 +3098,9 @@ NULL
         if ((ri = sentinelGetMasterByNameOrReplyError(c,c->argv[2])) == NULL)
             return;
         addReplyDictOfRedisInstances(c,ri->sentinels);
+    } else if (!strcasecmp(c->argv[1]->ptr,"myid") && c->argc == 2) {
+        /* SENTINEL MYID */
+        addReplyBulkCBuffer(c,sentinel.myid,CONFIG_RUN_ID_SIZE);
     } else if (!strcasecmp(c->argv[1]->ptr,"is-master-down-by-addr")) {
         /* SENTINEL IS-MASTER-DOWN-BY-ADDR <ip> <port> <current-epoch> <runid>
          *

From 6f3b91fe26990f4c849d3f20269d69f3e3dc96f5 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Mon, 5 Oct 2020 11:49:47 -0400
Subject: [PATCH 0715/1098] ACL : mentioning requirepass and aclfile not
 compatable in redis conf (#7883)

---
 redis.conf | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/redis.conf b/redis.conf
index 31655e92e..561aaa9db 100644
--- a/redis.conf
+++ b/redis.conf
@@ -798,6 +798,9 @@ acllog-max-len 128
 # AUTH <password> as usually, or more explicitly with AUTH default <password>
 # if they follow the new protocol: both will work.
 #
+# The requirepass is not compatable with aclfile option and the ACL LOAD
+# command, these will cause requirepass to be ignored.
+#
 # requirepass foobared
 
 # Command renaming (DEPRECATED).

From 216c1106099f7c66c991a5a811c8fbacbb0ab17c Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 6 Oct 2020 21:43:30 +0300
Subject: [PATCH 0716/1098] Allow blocked XREAD on a cluster replica (#7881)

I suppose that it was overlooked, since till recently none of the blocked commands were readonly.

other changes:
- add test for the above.
- add better support for additional (and deferring) clients for
  cluster tests
- improve a test which left the client in MULTI state.
---
 src/cluster.c                                 |  9 ++++++++
 .../tests/16-transactions-on-replica.tcl      | 21 +++++++++++++++++++
 tests/instances.tcl                           | 13 ++++++++++++
 3 files changed, 43 insertions(+)

diff --git a/src/cluster.c b/src/cluster.c
index 03235d58e..f7f033ba3 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -5855,6 +5855,15 @@ int clusterRedirectBlockedClientIfNeeded(client *c) {
             int slot = keyHashSlot((char*)key->ptr, sdslen(key->ptr));
             clusterNode *node = server.cluster->slots[slot];
 
+            /* if the client is read-only and attempting to access key that our
+             * replica can handle, allow it. */
+            if ((c->flags & CLIENT_READONLY) &&
+                (c->lastcmd->flags & CMD_READONLY) &&
+                nodeIsSlave(myself) && myself->slaveof == node)
+            {
+                node = myself;
+            }
+
             /* We send an error and unblock the client if:
              * 1) The slot is unassigned, emitting a cluster down error.
              * 2) The slot is not handled by this node, nor being imported. */
diff --git a/tests/cluster/tests/16-transactions-on-replica.tcl b/tests/cluster/tests/16-transactions-on-replica.tcl
index da9dff1ca..41083f421 100644
--- a/tests/cluster/tests/16-transactions-on-replica.tcl
+++ b/tests/cluster/tests/16-transactions-on-replica.tcl
@@ -45,4 +45,25 @@ test "MULTI-EXEC with write operations is MOVED" {
     $replica MULTI
     catch {$replica HSET h b 4} err
     assert {[string range $err 0 4] eq {MOVED}}
+    catch {$replica exec} err
+    assert {[string range $err 0 8] eq {EXECABORT}}
+}
+
+test "read-only blocking operations from replica" {
+    set rd [redis_deferring_client redis 1]
+    $rd readonly
+    $rd read
+    $rd XREAD BLOCK 0 STREAMS k 0
+
+    wait_for_condition 1000 50 {
+        [RI 1 blocked_clients] eq {1}
+    } else {
+        fail "client wasn't blocked"
+    }
+
+    $primary XADD k * foo bar
+    set res [$rd read]
+    set res [lindex [lindex [lindex [lindex $res 0] 1] 0] 1]
+    assert {$res eq {foo bar}}
+    $rd close
 }
diff --git a/tests/instances.tcl b/tests/instances.tcl
index 5c4b665db..d3b1b50cd 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -606,3 +606,16 @@ proc restart_instance {type id} {
     }
 }
 
+proc redis_deferring_client {type id} {
+    set port [get_instance_attrib $type $id port]
+    set host [get_instance_attrib $type $id host]
+    set client [redis $host $port 1 $::tls]
+    return $client
+}
+
+proc redis_client {type id} {
+    set port [get_instance_attrib $type $id port]
+    set host [get_instance_attrib $type $id host]
+    set client [redis $host $port 0 $::tls]
+    return $client
+}

From f659d236196ec57b02148031f425cf8864ed77ad Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 7 Oct 2020 20:28:57 +0300
Subject: [PATCH 0717/1098] Add some additional signal info to the crash log
 (#7891)

- si_code can be very useful info some day.
- a clear indication that redis was killed by an external user
---
 src/debug.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/debug.c b/src/debug.c
index 87b4bec71..d1a30ae1b 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1724,11 +1724,14 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
 
     bugReportStart();
     serverLog(LL_WARNING,
-        "Redis %s crashed by signal: %d", REDIS_VERSION, sig);
+        "Redis %s crashed by signal: %d, si_code: %d", REDIS_VERSION, sig, info->si_code);
     if (sig == SIGSEGV || sig == SIGBUS) {
         serverLog(LL_WARNING,
         "Accessing address: %p", (void*)info->si_addr);
     }
+    if (info->si_pid != -1) {
+        serverLog(LL_WARNING, "Killed by PID: %d, UID: %d", info->si_pid, info->si_uid);
+    }
 
 #ifdef HAVE_BACKTRACE
     ucontext_t *uc = (ucontext_t*) secret;

From 2127f7c8ebc1d8638eb0c99764f698cebcdb6513 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Wed, 7 Oct 2020 22:09:09 -0700
Subject: [PATCH 0718/1098] Fixed excessive categories being displayed from
 acls (#7889)

---
 src/acl.c          | 17 +++++++++++++++--
 tests/unit/acl.tcl | 14 ++++++++++++++
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index b4fb52c50..30fdc8e07 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -478,17 +478,30 @@ sds ACLDescribeUserCommandRules(user *u) {
     /* Try to add or subtract each category one after the other. Often a
      * single category will not perfectly match the set of commands into
      * it, so at the end we do a final pass adding/removing the single commands
-     * needed to make the bitmap exactly match. */
+     * needed to make the bitmap exactly match. A temp user is maintained to
+     * keep track of categories already applied. */
+    user tu = {0};
+    user *tempuser = &tu;
+    memcpy(tempuser->allowed_commands,
+        u->allowed_commands, 
+        sizeof(u->allowed_commands));
+
     for (int j = 0; ACLCommandCategories[j].flag != 0; j++) {
         unsigned long on, off;
-        ACLCountCategoryBitsForUser(u,&on,&off,ACLCommandCategories[j].name);
+        ACLCountCategoryBitsForUser(tempuser,&on,&off,ACLCommandCategories[j].name);
         if ((additive && on > off) || (!additive && off > on)) {
             sds op = sdsnewlen(additive ? "+@" : "-@", 2);
             op = sdscat(op,ACLCommandCategories[j].name);
             ACLSetUser(fakeuser,op,-1);
+
+            sds invop = sdsnewlen(additive ? "-@" : "+@", 2);
+            invop = sdscat(invop,ACLCommandCategories[j].name);
+            ACLSetUser(tempuser,invop,-1);
+
             rules = sdscatsds(rules,op);
             rules = sdscatlen(rules," ",1);
             sdsfree(op);
+            sdsfree(invop);
         }
     }
 
diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl
index f015f75a0..12f59e749 100644
--- a/tests/unit/acl.tcl
+++ b/tests/unit/acl.tcl
@@ -135,6 +135,20 @@ start_server {tags {"acl"}} {
         assert_match {*+acl*} $cmdstr
     }
 
+    # A regression test make sure that as long as there is a simple
+    # category defining the commands, that it will be used as is.
+    test {ACL GETUSER provides reasonable results} {
+        # Test for future commands where allowed
+        r ACL setuser additive reset +@all -@write
+        set cmdstr [dict get [r ACL getuser additive] commands]
+        assert_match {+@all -@write} $cmdstr
+
+        # Test for future commands are disallowed
+        r ACL setuser subtractive reset -@all +@read
+        set cmdstr [dict get [r ACL getuser subtractive] commands]
+        assert_match {-@all +@read} $cmdstr
+    }
+
     test {ACL #5998 regression: memory leaks adding / removing subcommands} {
         r AUTH default ""
         r ACL setuser newuser reset -debug +debug|a +debug|b +debug|c

From c3f9e017946cffdc3e7166e96e9ff98484db7963 Mon Sep 17 00:00:00 2001
From: Felipe Machado <462154+felipou@users.noreply.github.com>
Date: Thu, 8 Oct 2020 02:33:17 -0300
Subject: [PATCH 0719/1098] Adds new pop-push commands (LMOVE, BLMOVE) (#6929)

Adding [B]LMOVE <src> <dst> RIGHT|LEFT RIGHT|LEFT. deprecating [B]RPOPLPUSH.

Note that when receiving a BRPOPLPUSH we'll still propagate an RPOPLPUSH,
but on BLMOVE RIGHT LEFT we'll propagate an LMOVE

improvement to existing tests
- Replace "after 1000" with "wait_for_condition" when wait for
  clients to block/unblock.
- Add a pre-existing element to target list on basic tests so
  that we can check if the new element was added to the correct
  side of the list.
- check command stats on the replica to make sure the right
  command was replicated

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/blocked.c                     |  26 ++--
 src/module.c                      |   2 +-
 src/server.c                      |  14 ++
 src/server.h                      |  18 ++-
 src/t_list.c                      | 235 +++++++++++++++++++-----------
 src/t_stream.c                    |   2 +-
 src/t_zset.c                      |   2 +-
 tests/integration/replication.tcl |  37 +++++
 tests/support/util.tcl            |   6 +
 tests/unit/introspection-2.tcl    |   4 +-
 tests/unit/type/list.tcl          | 189 +++++++++++++++++++++---
 11 files changed, 409 insertions(+), 126 deletions(-)

diff --git a/src/blocked.c b/src/blocked.c
index 87f816faa..8641fcf8c 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -62,7 +62,8 @@
 
 #include "server.h"
 
-int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int where);
+int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int wherefrom, int whereto);
+int getListPositionFromObjectOrReply(client *c, robj *arg, int *position);
 
 /* This structure represents the blocked key information that we store
  * in the client structure. Each client blocked on keys, has a
@@ -231,10 +232,9 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) {
             }
 
             robj *dstkey = receiver->bpop.target;
-            int where = (receiver->lastcmd &&
-                         receiver->lastcmd->proc == blpopCommand) ?
-                         LIST_HEAD : LIST_TAIL;
-            robj *value = listTypePop(o,where);
+            int wherefrom = receiver->bpop.listpos.wherefrom;
+            int whereto = receiver->bpop.listpos.whereto;
+            robj *value = listTypePop(o, wherefrom);
 
             if (value) {
                 /* Protect receiver->bpop.target, that will be
@@ -245,11 +245,11 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) {
 
                 if (serveClientBlockedOnList(receiver,
                     rl->key,dstkey,rl->db,value,
-                    where) == C_ERR)
+                    wherefrom, whereto) == C_ERR)
                 {
                     /* If we failed serving the client we need
                      * to also undo the POP operation. */
-                    listTypePush(o,value,where);
+                    listTypePush(o,value,wherefrom);
                 }
 
                 if (dstkey) decrRefCount(dstkey);
@@ -466,8 +466,8 @@ void serveClientsBlockedOnKeyByModule(readyList *rl) {
  * one new element via some write operation are accumulated into
  * the server.ready_keys list. This function will run the list and will
  * serve clients accordingly. Note that the function will iterate again and
- * again as a result of serving BRPOPLPUSH we can have new blocking clients
- * to serve because of the PUSH side of BRPOPLPUSH.
+ * again as a result of serving BLMOVE we can have new blocking clients
+ * to serve because of the PUSH side of BLMOVE.
  *
  * This function is normally "fair", that is, it will server clients
  * using a FIFO behavior. However this fairness is violated in certain
@@ -485,7 +485,7 @@ void handleClientsBlockedOnKeys(void) {
         /* Point server.ready_keys to a fresh list and save the current one
          * locally. This way as we run the old list we are free to call
          * signalKeyAsReady() that may push new elements in server.ready_keys
-         * when handling clients blocked into BRPOPLPUSH. */
+         * when handling clients blocked into BLMOVE. */
         l = server.ready_keys;
         server.ready_keys = listCreate();
 
@@ -500,7 +500,7 @@ void handleClientsBlockedOnKeys(void) {
             /* Even if we are not inside call(), increment the call depth
              * in order to make sure that keys are expired against a fixed
              * reference time, and not against the wallclock time. This
-             * way we can lookup an object multiple times (BRPOPLPUSH does
+             * way we can lookup an object multiple times (BLMOVE does
              * that) without the risk of it being freed in the second
              * lookup, invalidating the first one.
              * See https://github.com/antirez/redis/pull/6554. */
@@ -560,7 +560,7 @@ void handleClientsBlockedOnKeys(void) {
  * stream keys, we also provide an array of streamID structures: clients will
  * be unblocked only when items with an ID greater or equal to the specified
  * one is appended to the stream. */
-void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids) {
+void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, struct listPos *listpos, streamID *ids) {
     dictEntry *de;
     list *l;
     int j;
@@ -568,6 +568,8 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo
     c->bpop.timeout = timeout;
     c->bpop.target = target;
 
+    if (listpos != NULL) c->bpop.listpos = *listpos;
+
     if (target != NULL) incrRefCount(target);
 
     for (j = 0; j < numkeys; j++) {
diff --git a/src/module.c b/src/module.c
index fb8f83e9c..3f3d92aea 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4492,7 +4492,7 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF
             "Blocking module command called from transaction");
     } else {
         if (keys) {
-            blockForKeys(c,BLOCKED_MODULE,keys,numkeys,timeout,NULL,NULL);
+            blockForKeys(c,BLOCKED_MODULE,keys,numkeys,timeout,NULL,NULL,NULL);
         } else {
             blockClient(c,BLOCKED_MODULE);
         }
diff --git a/src/server.c b/src/server.c
index d8aaef06d..a9a18ea19 100644
--- a/src/server.c
+++ b/src/server.c
@@ -302,6 +302,10 @@ struct redisCommand redisCommandTable[] = {
      "write use-memory no-script @list @blocking",
      0,NULL,1,2,1,0,0,0},
 
+    {"blmove",blmoveCommand,6,
+     "write use-memory no-script @list @blocking",
+     0,NULL,1,2,1,0,0,0},
+
     {"blpop",blpopCommand,-3,
      "write no-script @list @blocking",
      0,NULL,1,-2,1,0,0,0},
@@ -338,6 +342,10 @@ struct redisCommand redisCommandTable[] = {
      "write use-memory @list",
      0,NULL,1,2,1,0,0,0},
 
+    {"lmove",lmoveCommand,5,
+     "write use-memory @list",
+     0,NULL,1,2,1,0,0,0},
+
     {"sadd",saddCommand,-3,
      "write use-memory fast @set",
      0,NULL,1,1,1,0,0,0},
@@ -2385,10 +2393,15 @@ void createSharedObjects(void) {
     shared.lpop = createStringObject("LPOP",4);
     shared.lpush = createStringObject("LPUSH",5);
     shared.rpoplpush = createStringObject("RPOPLPUSH",9);
+    shared.lmove = createStringObject("LMOVE",5);
+    shared.blmove = createStringObject("BLMOVE",6);
     shared.zpopmin = createStringObject("ZPOPMIN",7);
     shared.zpopmax = createStringObject("ZPOPMAX",7);
     shared.multi = createStringObject("MULTI",5);
     shared.exec = createStringObject("EXEC",4);
+    /* Used in the LMOVE/BLMOVE commands */
+    shared.left = createStringObject("left",4);
+    shared.right = createStringObject("right",5);
     for (j = 0; j < OBJ_SHARED_INTEGERS; j++) {
         shared.integers[j] =
             makeObjectShared(createObject(OBJ_STRING,(void*)(long)j));
@@ -2523,6 +2536,7 @@ void initServerConfig(void) {
     server.xclaimCommand = lookupCommandByCString("xclaim");
     server.xgroupCommand = lookupCommandByCString("xgroup");
     server.rpoplpushCommand = lookupCommandByCString("rpoplpush");
+    server.lmoveCommand = lookupCommandByCString("lmove");
 
     /* Debugging */
     server.watchdog_period = 0;
diff --git a/src/server.h b/src/server.h
index 114034962..8ddc0b75f 100644
--- a/src/server.h
+++ b/src/server.h
@@ -703,7 +703,13 @@ typedef struct blockingState {
     dict *keys;             /* The keys we are waiting to terminate a blocking
                              * operation such as BLPOP or XREAD. Or NULL. */
     robj *target;           /* The key that should receive the element,
-                             * for BRPOPLPUSH. */
+                             * for BLMOVE. */
+    struct listPos {
+        int wherefrom;      /* Where to pop from */
+        int whereto;        /* Where to push to */
+    } listpos;              /* The positions in the src/dst lists
+                             * where we want to pop/push an element
+                             * for BLPOP, BRPOP and BLMOVE. */
 
     /* BLOCK_STREAM */
     size_t xread_count;     /* XREAD COUNT option. */
@@ -893,8 +899,8 @@ struct sharedObjectsStruct {
     *masterdownerr, *roslaveerr, *execaborterr, *noautherr, *noreplicaserr,
     *busykeyerr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk,
     *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *unlink,
-    *rpop, *lpop, *lpush, *rpoplpush, *zpopmin, *zpopmax, *emptyscan,
-    *multi, *exec,
+    *rpop, *lpop, *lpush, *rpoplpush, *lmove, *blmove, *zpopmin, *zpopmax,
+    *emptyscan, *multi, *exec, *left, *right,
     *select[PROTO_SHARED_SELECT_CMDS],
     *integers[OBJ_SHARED_INTEGERS],
     *mbulkhdr[OBJ_SHARED_BULKHDR_LEN], /* "*<value>\r\n" */
@@ -1140,7 +1146,7 @@ struct redisServer {
                         *lpopCommand, *rpopCommand, *zpopminCommand,
                         *zpopmaxCommand, *sremCommand, *execCommand,
                         *expireCommand, *pexpireCommand, *xclaimCommand,
-                        *xgroupCommand, *rpoplpushCommand;
+                        *xgroupCommand, *rpoplpushCommand, *lmoveCommand;
     /* Fields used only for stats */
     time_t stat_starttime;          /* Server start time */
     long long stat_numcommands;     /* Number of processed commands */
@@ -2226,7 +2232,7 @@ int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int
 void disconnectAllBlockedClients(void);
 void handleClientsBlockedOnKeys(void);
 void signalKeyAsReady(redisDb *db, robj *key, int type);
-void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids);
+void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, struct listPos *listpos, streamID *ids);
 
 /* timeout.c -- Blocked clients timeout and connections timeout. */
 void addClientToTimeoutTable(client *c);
@@ -2337,6 +2343,7 @@ void sortCommand(client *c);
 void lremCommand(client *c);
 void lposCommand(client *c);
 void rpoplpushCommand(client *c);
+void lmoveCommand(client *c);
 void infoCommand(client *c);
 void mgetCommand(client *c);
 void monitorCommand(client *c);
@@ -2380,6 +2387,7 @@ void discardCommand(client *c);
 void blpopCommand(client *c);
 void brpopCommand(client *c);
 void brpoplpushCommand(client *c);
+void blmoveCommand(client *c);
 void appendCommand(client *c);
 void strlenCommand(client *c);
 void zrankCommand(client *c);
diff --git a/src/t_list.c b/src/t_list.c
index 4edcc8be0..e7a69b1e9 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -647,6 +647,102 @@ void lremCommand(client *c) {
     addReplyLongLong(c,removed);
 }
 
+void lmoveHandlePush(client *c, robj *dstkey, robj *dstobj, robj *value,
+                     int where) {
+    /* Create the list if the key does not exist */
+    if (!dstobj) {
+        dstobj = createQuicklistObject();
+        quicklistSetOptions(dstobj->ptr, server.list_max_ziplist_size,
+                            server.list_compress_depth);
+        dbAdd(c->db,dstkey,dstobj);
+    }
+    signalModifiedKey(c,c->db,dstkey);
+    listTypePush(dstobj,value,where);
+    notifyKeyspaceEvent(NOTIFY_LIST,
+                        where == LIST_HEAD ? "lpush" : "rpush",
+                        dstkey,
+                        c->db->id);
+    /* Always send the pushed value to the client. */
+    addReplyBulk(c,value);
+}
+
+int getListPositionFromObjectOrReply(client *c, robj *arg, int *position) {
+    if (strcasecmp(arg->ptr,"right") == 0) {
+        *position = LIST_TAIL;
+    } else if (strcasecmp(arg->ptr,"left") == 0) {
+        *position = LIST_HEAD;
+    } else {
+        addReply(c,shared.syntaxerr);
+        return C_ERR;
+    }
+    return C_OK;
+}
+
+robj *getStringObjectFromListPosition(int position) {
+    if (position == LIST_HEAD) {
+        return shared.left;
+    } else {
+        // LIST_TAIL
+        return shared.right;
+    }
+}
+
+void lmoveGenericCommand(client *c, int wherefrom, int whereto) {
+    robj *sobj, *value;
+    if ((sobj = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp]))
+        == NULL || checkType(c,sobj,OBJ_LIST)) return;
+
+    if (listTypeLength(sobj) == 0) {
+        /* This may only happen after loading very old RDB files. Recent
+         * versions of Redis delete keys of empty lists. */
+        addReplyNull(c);
+    } else {
+        robj *dobj = lookupKeyWrite(c->db,c->argv[2]);
+        robj *touchedkey = c->argv[1];
+
+        if (checkType(c,dobj,OBJ_LIST)) return;
+        value = listTypePop(sobj,wherefrom);
+        /* We saved touched key, and protect it, since lmoveHandlePush
+         * may change the client command argument vector (it does not
+         * currently). */
+        incrRefCount(touchedkey);
+        lmoveHandlePush(c,c->argv[2],dobj,value,whereto);
+
+        /* listTypePop returns an object with its refcount incremented */
+        decrRefCount(value);
+
+        /* Delete the source list when it is empty */
+        notifyKeyspaceEvent(NOTIFY_LIST,
+                            wherefrom == LIST_HEAD ? "lpop" : "rpop",
+                            touchedkey,
+                            c->db->id);
+        if (listTypeLength(sobj) == 0) {
+            dbDelete(c->db,touchedkey);
+            notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
+                                touchedkey,c->db->id);
+        }
+        signalModifiedKey(c,c->db,touchedkey);
+        decrRefCount(touchedkey);
+        server.dirty++;
+        if (c->cmd->proc == blmoveCommand) {
+            rewriteClientCommandVector(c,5,shared.lmove,
+                                       c->argv[1],c->argv[2],c->argv[3],c->argv[4]);
+        } else if (c->cmd->proc == brpoplpushCommand) {
+            rewriteClientCommandVector(c,3,shared.rpoplpush,
+                                       c->argv[1],c->argv[2]);
+        }
+    }
+}
+
+void lmoveCommand(client *c) {
+    int wherefrom, whereto;
+    if (getListPositionFromObjectOrReply(c,c->argv[3],&wherefrom)
+        != C_OK) return;
+    if (getListPositionFromObjectOrReply(c,c->argv[4],&whereto)
+        != C_OK) return;
+    lmoveGenericCommand(c, wherefrom, whereto);
+}
+
 /* This is the semantic of this command:
  *  RPOPLPUSH srclist dstlist:
  *    IF LLEN(srclist) > 0
@@ -662,60 +758,8 @@ void lremCommand(client *c) {
  * since the element is not just returned but pushed against another list
  * as well. This command was originally proposed by Ezra Zygmuntowicz.
  */
-
-void rpoplpushHandlePush(client *c, robj *dstkey, robj *dstobj, robj *value) {
-    /* Create the list if the key does not exist */
-    if (!dstobj) {
-        dstobj = createQuicklistObject();
-        quicklistSetOptions(dstobj->ptr, server.list_max_ziplist_size,
-                            server.list_compress_depth);
-        dbAdd(c->db,dstkey,dstobj);
-    }
-    signalModifiedKey(c,c->db,dstkey);
-    listTypePush(dstobj,value,LIST_HEAD);
-    notifyKeyspaceEvent(NOTIFY_LIST,"lpush",dstkey,c->db->id);
-    /* Always send the pushed value to the client. */
-    addReplyBulk(c,value);
-}
-
 void rpoplpushCommand(client *c) {
-    robj *sobj, *value;
-    if ((sobj = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp]))
-        == NULL || checkType(c,sobj,OBJ_LIST)) return;
-
-    if (listTypeLength(sobj) == 0) {
-        /* This may only happen after loading very old RDB files. Recent
-         * versions of Redis delete keys of empty lists. */
-        addReplyNull(c);
-    } else {
-        robj *dobj = lookupKeyWrite(c->db,c->argv[2]);
-        robj *touchedkey = c->argv[1];
-
-        if (checkType(c,dobj,OBJ_LIST)) return;
-        value = listTypePop(sobj,LIST_TAIL);
-        /* We saved touched key, and protect it, since rpoplpushHandlePush
-         * may change the client command argument vector (it does not
-         * currently). */
-        incrRefCount(touchedkey);
-        rpoplpushHandlePush(c,c->argv[2],dobj,value);
-
-        /* listTypePop returns an object with its refcount incremented */
-        decrRefCount(value);
-
-        /* Delete the source list when it is empty */
-        notifyKeyspaceEvent(NOTIFY_LIST,"rpop",touchedkey,c->db->id);
-        if (listTypeLength(sobj) == 0) {
-            dbDelete(c->db,touchedkey);
-            notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
-                                touchedkey,c->db->id);
-        }
-        signalModifiedKey(c,c->db,touchedkey);
-        decrRefCount(touchedkey);
-        server.dirty++;
-        if (c->cmd->proc == brpoplpushCommand) {
-            rewriteClientCommandVector(c,3,shared.rpoplpush,c->argv[1],c->argv[2]);
-        }
-    }
+    lmoveGenericCommand(c, LIST_TAIL, LIST_HEAD);
 }
 
 /*-----------------------------------------------------------------------------
@@ -727,30 +771,34 @@ void rpoplpushCommand(client *c) {
  * in the context of the specified 'db', doing the following:
  *
  * 1) Provide the client with the 'value' element.
- * 2) If the dstkey is not NULL (we are serving a BRPOPLPUSH) also push the
- *    'value' element on the destination list (the LPUSH side of the command).
- * 3) Propagate the resulting BRPOP, BLPOP and additional LPUSH if any into
+ * 2) If the dstkey is not NULL (we are serving a BLMOVE) also push the
+ *    'value' element on the destination list (the "push" side of the command).
+ * 3) Propagate the resulting BRPOP, BLPOP and additional xPUSH if any into
  *    the AOF and replication channel.
  *
- * The argument 'where' is LIST_TAIL or LIST_HEAD, and indicates if the
+ * The argument 'wherefrom' is LIST_TAIL or LIST_HEAD, and indicates if the
  * 'value' element was popped from the head (BLPOP) or tail (BRPOP) so that
  * we can propagate the command properly.
  *
+ * The argument 'whereto' is LIST_TAIL or LIST_HEAD, and indicates if the
+ * 'value' element is to be pushed to the head or tail so that we can
+ * propagate the command properly.
+ *
  * The function returns C_OK if we are able to serve the client, otherwise
  * C_ERR is returned to signal the caller that the list POP operation
  * should be undone as the client was not served: This only happens for
- * BRPOPLPUSH that fails to push the value to the destination key as it is
+ * BLMOVE that fails to push the value to the destination key as it is
  * of the wrong type. */
-int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int where)
+int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int wherefrom, int whereto)
 {
-    robj *argv[3];
+    robj *argv[5];
 
     if (dstkey == NULL) {
         /* Propagate the [LR]POP operation. */
-        argv[0] = (where == LIST_HEAD) ? shared.lpop :
-                                          shared.rpop;
+        argv[0] = (wherefrom == LIST_HEAD) ? shared.lpop :
+                                             shared.rpop;
         argv[1] = key;
-        propagate((where == LIST_HEAD) ?
+        propagate((wherefrom == LIST_HEAD) ?
             server.lpopCommand : server.rpopCommand,
             db->id,argv,2,PROPAGATE_AOF|PROPAGATE_REPL);
 
@@ -760,30 +808,33 @@ int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb
         addReplyBulk(receiver,value);
 
         /* Notify event. */
-        char *event = (where == LIST_HEAD) ? "lpop" : "rpop";
+        char *event = (wherefrom == LIST_HEAD) ? "lpop" : "rpop";
         notifyKeyspaceEvent(NOTIFY_LIST,event,key,receiver->db->id);
     } else {
-        /* BRPOPLPUSH */
+        /* BLMOVE */
         robj *dstobj =
             lookupKeyWrite(receiver->db,dstkey);
         if (!(dstobj &&
              checkType(receiver,dstobj,OBJ_LIST)))
         {
-            rpoplpushHandlePush(receiver,dstkey,dstobj,
-                value);
-            /* Propagate the RPOPLPUSH operation. */
-            argv[0] = shared.rpoplpush;
+            lmoveHandlePush(receiver,dstkey,dstobj,value,whereto);
+            /* Propagate the LMOVE/RPOPLPUSH operation. */
+            int isbrpoplpush = (receiver->lastcmd->proc == brpoplpushCommand);
+            argv[0] = isbrpoplpush ? shared.rpoplpush : shared.lmove;
             argv[1] = key;
             argv[2] = dstkey;
-            propagate(server.rpoplpushCommand,
-                db->id,argv,3,
+            argv[3] = getStringObjectFromListPosition(wherefrom);
+            argv[4] = getStringObjectFromListPosition(whereto);
+            propagate(isbrpoplpush ? server.rpoplpushCommand : server.lmoveCommand,
+                db->id,argv,(isbrpoplpush ? 3 : 5),
                 PROPAGATE_AOF|
                 PROPAGATE_REPL);
 
-            /* Notify event ("lpush" was notified by rpoplpushHandlePush). */
-            notifyKeyspaceEvent(NOTIFY_LIST,"rpop",key,receiver->db->id);
+            /* Notify event ("lpush" or "rpush" was notified by lmoveHandlePush). */
+            notifyKeyspaceEvent(NOTIFY_LIST,wherefrom == LIST_TAIL ? "rpop" : "lpop",
+                                key,receiver->db->id);
         } else {
-            /* BRPOPLPUSH failed because of wrong
+            /* BLMOVE failed because of wrong
              * destination type. */
             return C_ERR;
         }
@@ -844,7 +895,8 @@ void blockingPopGenericCommand(client *c, int where) {
     }
 
     /* If the keys do not exist we must block */
-    blockForKeys(c,BLOCKED_LIST,c->argv + 1,c->argc - 2,timeout,NULL,NULL);
+    struct listPos pos = {where};
+    blockForKeys(c,BLOCKED_LIST,c->argv + 1,c->argc - 2,timeout,NULL,&pos,NULL);
 }
 
 void blpopCommand(client *c) {
@@ -855,12 +907,7 @@ void brpopCommand(client *c) {
     blockingPopGenericCommand(c,LIST_TAIL);
 }
 
-void brpoplpushCommand(client *c) {
-    mstime_t timeout;
-
-    if (getTimeoutFromObjectOrReply(c,c->argv[3],&timeout,UNIT_SECONDS)
-        != C_OK) return;
-
+void blmoveGenericCommand(client *c, int wherefrom, int whereto, mstime_t timeout) {
     robj *key = lookupKeyWrite(c->db, c->argv[1]);
     if (checkType(c,key,OBJ_LIST)) return;
 
@@ -871,12 +918,32 @@ void brpoplpushCommand(client *c) {
             addReplyNull(c);
         } else {
             /* The list is empty and the client blocks. */
-            blockForKeys(c,BLOCKED_LIST,c->argv + 1,1,timeout,c->argv[2],NULL);
+            struct listPos pos = {wherefrom, whereto};
+            blockForKeys(c,BLOCKED_LIST,c->argv + 1,1,timeout,c->argv[2],&pos,NULL);
         }
     } else {
         /* The list exists and has elements, so
-         * the regular rpoplpushCommand is executed. */
+         * the regular lmoveCommand is executed. */
         serverAssertWithInfo(c,key,listTypeLength(key) > 0);
-        rpoplpushCommand(c);
+        lmoveGenericCommand(c,wherefrom,whereto);
     }
 }
+
+void blmoveCommand(client *c) {
+    mstime_t timeout;
+    int wherefrom, whereto;
+    if (getListPositionFromObjectOrReply(c,c->argv[3],&wherefrom)
+        != C_OK) return;
+    if (getListPositionFromObjectOrReply(c,c->argv[4],&whereto)
+        != C_OK) return;
+    if (getTimeoutFromObjectOrReply(c,c->argv[5],&timeout,UNIT_SECONDS)
+        != C_OK) return;
+    blmoveGenericCommand(c,wherefrom,whereto,timeout);
+}
+
+void brpoplpushCommand(client *c) {
+    mstime_t timeout;
+    if (getTimeoutFromObjectOrReply(c,c->argv[3],&timeout,UNIT_SECONDS)
+        != C_OK) return;
+    blmoveGenericCommand(c, LIST_TAIL, LIST_HEAD, timeout);
+}
diff --git a/src/t_stream.c b/src/t_stream.c
index 285684987..6eb6363fc 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1628,7 +1628,7 @@ void xreadCommand(client *c) {
             goto cleanup;
         }
         blockForKeys(c, BLOCKED_STREAM, c->argv+streams_arg, streams_count,
-                     timeout, NULL, ids);
+                     timeout, NULL, NULL, ids);
         /* If no COUNT is given and we block, set a relatively small count:
          * in case the ID provided is too low, we do not want the server to
          * block just to serve this client a huge stream of messages. */
diff --git a/src/t_zset.c b/src/t_zset.c
index 9fddf37d1..5fc7643c3 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -3364,7 +3364,7 @@ void blockingGenericZpopCommand(client *c, int where) {
     }
 
     /* If the keys do not exist we must block */
-    blockForKeys(c,BLOCKED_ZSET,c->argv + 1,c->argc - 2,timeout,NULL,NULL);
+    blockForKeys(c,BLOCKED_ZSET,c->argv + 1,c->argc - 2,timeout,NULL,NULL,NULL);
 }
 
 // BZPOPMIN key [key ...] timeout
diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl
index d2c8b8e56..06414dd8a 100644
--- a/tests/integration/replication.tcl
+++ b/tests/integration/replication.tcl
@@ -78,6 +78,7 @@ start_server {tags {"repl"}} {
         }
 
         test {BRPOPLPUSH replication, when blocking against empty list} {
+            $A config resetstat
             set rd [redis_deferring_client]
             $rd brpoplpush a b 5
             r lpush a foo
@@ -86,9 +87,12 @@ start_server {tags {"repl"}} {
             } else {
                 fail "Master and replica have different digest: [$A debug digest] VS [$B debug digest]"
             }
+            assert_match {*calls=1,*} [cmdrstat rpoplpush $A]
+            assert_match {} [cmdrstat lmove $A]
         }
 
         test {BRPOPLPUSH replication, list exists} {
+            $A config resetstat
             set rd [redis_deferring_client]
             r lpush c 1
             r lpush c 2
@@ -96,6 +100,39 @@ start_server {tags {"repl"}} {
             $rd brpoplpush c d 5
             after 1000
             assert_equal [$A debug digest] [$B debug digest]
+            assert_match {*calls=1,*} [cmdrstat rpoplpush $A]
+            assert_match {} [cmdrstat lmove $A]
+        }
+
+        foreach wherefrom {left right} {
+            foreach whereto {left right} {
+                test "BLMOVE ($wherefrom, $whereto) replication, when blocking against empty list" {
+                    $A config resetstat
+                    set rd [redis_deferring_client]
+                    $rd blmove a b $wherefrom $whereto 5
+                    r lpush a foo
+                    wait_for_condition 50 100 {
+                        [$A debug digest] eq [$B debug digest]
+                    } else {
+                        fail "Master and replica have different digest: [$A debug digest] VS [$B debug digest]"
+                    }
+                    assert_match {*calls=1,*} [cmdrstat lmove $A]
+                    assert_match {} [cmdrstat rpoplpush $A]
+                }
+
+                test "BLMOVE ($wherefrom, $whereto) replication, list exists" {
+                    $A config resetstat
+                    set rd [redis_deferring_client]
+                    r lpush c 1
+                    r lpush c 2
+                    r lpush c 3
+                    $rd blmove c d $wherefrom $whereto 5
+                    after 1000
+                    assert_equal [$A debug digest] [$B debug digest]
+                    assert_match {*calls=1,*} [cmdrstat lmove $A]
+                    assert_match {} [cmdrstat rpoplpush $A]
+                }
+            }
         }
 
         test {BLPOP followed by role change, issue #2473} {
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index b9a65358f..bc1394ce4 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -519,3 +519,9 @@ proc get_child_pid {idx} {
 
     return $child_pid
 }
+
+proc cmdrstat {cmd r} {
+    if {[regexp "\r\ncmdstat_$cmd:(.*?)\r\n" [$r info commandstats] _ value]} {
+        set _ $value
+    }
+}
diff --git a/tests/unit/introspection-2.tcl b/tests/unit/introspection-2.tcl
index 55f656956..0968c2933 100644
--- a/tests/unit/introspection-2.tcl
+++ b/tests/unit/introspection-2.tcl
@@ -1,7 +1,5 @@
 proc cmdstat {cmd} {
-    if {[regexp "\r\ncmdstat_$cmd:(.*?)\r\n" [r info commandstats] _ value]} {
-        set _ $value
-    }
+    return [cmdrstat $cmd r]
 }
 
 start_server {tags {"introspection"}} {
diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl
index 95f18b086..600a7ce70 100644
--- a/tests/unit/type/list.tcl
+++ b/tests/unit/type/list.tcl
@@ -194,6 +194,7 @@ start_server {
 
         test "BRPOPLPUSH - $type" {
             r del target
+            r rpush target bar
 
             set rd [redis_deferring_client]
             create_list blist "a b $large c d"
@@ -201,9 +202,38 @@ start_server {
             $rd brpoplpush blist target 1
             assert_equal d [$rd read]
 
-            assert_equal d [r rpop target]
+            assert_equal d [r lpop target]
             assert_equal "a b $large c" [r lrange blist 0 -1]
         }
+
+        foreach wherefrom {left right} {
+            foreach whereto {left right} {
+                test "BLMOVE $wherefrom $whereto - $type" {
+                    r del target
+                    r rpush target bar
+
+                    set rd [redis_deferring_client]
+                    create_list blist "a b $large c d"
+
+                    $rd blmove blist target $wherefrom $whereto 1
+                    set poppedelement [$rd read]
+
+                    if {$wherefrom eq "right"} {
+                        assert_equal d $poppedelement
+                        assert_equal "a b $large c" [r lrange blist 0 -1]
+                    } else {
+                        assert_equal a $poppedelement
+                        assert_equal "b $large c d" [r lrange blist 0 -1]
+                    }
+
+                    if {$whereto eq "right"} {
+                        assert_equal $poppedelement [r rpop target]
+                    } else {
+                        assert_equal $poppedelement [r lpop target]
+                    }
+                }
+            }
+        }
     }
 
     test "BLPOP, LPUSH + DEL should not awake blocked client" {
@@ -285,24 +315,60 @@ start_server {
     test "BRPOPLPUSH with zero timeout should block indefinitely" {
         set rd [redis_deferring_client]
         r del blist target
+        r rpush target bar
         $rd brpoplpush blist target 0
-        after 1000
+        wait_for_condition 100 10 {
+            [s blocked_clients] == 1
+        } else {
+            fail "Timeout waiting for blocked clients"
+        }
         r rpush blist foo
         assert_equal foo [$rd read]
-        assert_equal {foo} [r lrange target 0 -1]
+        assert_equal {foo bar} [r lrange target 0 -1]
     }
 
-    test "BRPOPLPUSH with a client BLPOPing the target list" {
-        set rd [redis_deferring_client]
-        set rd2 [redis_deferring_client]
-        r del blist target
-        $rd2 blpop target 0
-        $rd brpoplpush blist target 0
-        after 1000
-        r rpush blist foo
-        assert_equal foo [$rd read]
-        assert_equal {target foo} [$rd2 read]
-        assert_equal 0 [r exists target]
+    foreach wherefrom {left right} {
+        foreach whereto {left right} {
+            test "BLMOVE $wherefrom $whereto with zero timeout should block indefinitely" {
+                set rd [redis_deferring_client]
+                r del blist target
+                r rpush target bar
+                $rd blmove blist target $wherefrom $whereto 0
+                wait_for_condition 100 10 {
+                    [s blocked_clients] == 1
+                } else {
+                    fail "Timeout waiting for blocked clients"
+                }
+                r rpush blist foo
+                assert_equal foo [$rd read]
+                if {$whereto eq "right"} {
+                    assert_equal {bar foo} [r lrange target 0 -1]
+                } else {
+                    assert_equal {foo bar} [r lrange target 0 -1]
+                }
+            }
+        }
+    }
+
+    foreach wherefrom {left right} {
+        foreach whereto {left right} {
+            test "BLMOVE ($wherefrom, $whereto) with a client BLPOPing the target list" {
+                set rd [redis_deferring_client]
+                set rd2 [redis_deferring_client]
+                r del blist target
+                $rd2 blpop target 0
+                $rd blmove blist target $wherefrom $whereto 0
+                wait_for_condition 100 10 {
+                    [s blocked_clients] == 2
+                } else {
+                    fail "Timeout waiting for blocked clients"
+                }
+                r rpush blist foo
+                assert_equal foo [$rd read]
+                assert_equal {target foo} [$rd2 read]
+                assert_equal 0 [r exists target]
+            }
+        }
     }
 
     test "BRPOPLPUSH with wrong source type" {
@@ -325,7 +391,11 @@ start_server {
         r del blist target
         r set target nolist
         $rd brpoplpush blist target 0
-        after 1000
+        wait_for_condition 100 10 {
+            [s blocked_clients] == 1
+        } else {
+            fail "Timeout waiting for blocked clients"
+        }
         r rpush blist foo
         assert_error "WRONGTYPE*" {$rd read}
         assert_equal {foo} [r lrange blist 0 -1]
@@ -355,14 +425,14 @@ start_server {
         assert_equal {foo} [r lrange target2 0 -1]
     }
 
-    test "Linked BRPOPLPUSH" {
+    test "Linked LMOVEs" {
       set rd1 [redis_deferring_client]
       set rd2 [redis_deferring_client]
 
       r del list1 list2 list3
 
-      $rd1 brpoplpush list1 list2 0
-      $rd2 brpoplpush list2 list3 0
+      $rd1 blmove list1 list2 right left 0
+      $rd2 blmove list2 list3 left right 0
 
       r rpush list1 foo
 
@@ -451,7 +521,16 @@ start_server {
       set rd [redis_deferring_client]
 
       $rd brpoplpush foo_list bar_list 1
-      after 2000
+      wait_for_condition 100 10 {
+          [s blocked_clients] == 1
+      } else {
+          fail "Timeout waiting for blocked client"
+      }
+      wait_for_condition 500 10 {
+          [s blocked_clients] == 0
+      } else {
+          fail "Timeout waiting for client to unblock"
+      }
       $rd read
     } {}
 
@@ -676,6 +755,29 @@ start_server {
             assert_encoding quicklist mylist2
         }
 
+        foreach wherefrom {left right} {
+            foreach whereto {left right} {
+                test "LMOVE $wherefrom $whereto base case - $type" {
+                    r del mylist1 mylist2
+
+                    if {$wherefrom eq "right"} {
+                        create_list mylist1 "c d $large a"
+                    } else {
+                        create_list mylist1 "a $large c d"
+                    }
+                    assert_equal a [r lmove mylist1 mylist2 $wherefrom $whereto]
+                    assert_equal $large [r lmove mylist1 mylist2 $wherefrom $whereto]
+                    assert_equal "c d" [r lrange mylist1 0 -1]
+                    if {$whereto eq "right"} {
+                        assert_equal "a $large" [r lrange mylist2 0 -1]
+                    } else {
+                        assert_equal "$large a" [r lrange mylist2 0 -1]
+                    }
+                    assert_encoding quicklist mylist2
+                }
+            }
+        }
+
         test "RPOPLPUSH with the same list as src and dst - $type" {
             create_list mylist "a $large c"
             assert_equal "a $large c" [r lrange mylist 0 -1]
@@ -683,6 +785,26 @@ start_server {
             assert_equal "c a $large" [r lrange mylist 0 -1]
         }
 
+        foreach wherefrom {left right} {
+            foreach whereto {left right} {
+                test "LMOVE $wherefrom $whereto with the same list as src and dst - $type" {
+                    if {$wherefrom eq "right"} {
+                        create_list mylist "a $large c"
+                        assert_equal "a $large c" [r lrange mylist 0 -1]
+                    } else {
+                        create_list mylist "c a $large"
+                        assert_equal "c a $large" [r lrange mylist 0 -1]
+                    }
+                    assert_equal c [r lmove mylist mylist $wherefrom $whereto]
+                    if {$whereto eq "right"} {
+                        assert_equal "a $large c" [r lrange mylist 0 -1]
+                    } else {
+                        assert_equal "c a $large" [r lrange mylist 0 -1]
+                    }
+                }
+            }
+        }
+
         foreach {othertype otherlarge} [array get largevalue] {
             test "RPOPLPUSH with $type source and existing target $othertype" {
                 create_list srclist "a b c $large"
@@ -698,6 +820,35 @@ start_server {
                     assert_encoding quicklist dstlist
                 }
             }
+
+            foreach wherefrom {left right} {
+                foreach whereto {left right} {
+                    test "LMOVE $wherefrom $whereto with $type source and existing target $othertype" {
+                        create_list dstlist "$otherlarge"
+
+                        if {$wherefrom eq "right"} {
+                            create_list srclist "a b c $large"
+                        } else {
+                            create_list srclist "$large c a b"
+                        }
+                        assert_equal $large [r lmove srclist dstlist $wherefrom $whereto]
+                        assert_equal c [r lmove srclist dstlist $wherefrom $whereto]
+                        assert_equal "a b" [r lrange srclist 0 -1]
+
+                        if {$whereto eq "right"} {
+                            assert_equal "$otherlarge $large c" [r lrange dstlist 0 -1]
+                        } else {
+                            assert_equal "c $large $otherlarge" [r lrange dstlist 0 -1]
+                        }
+
+                        # When we lmoved a large value, dstlist should be
+                        # converted to the same encoding as srclist.
+                        if {$type eq "linkedlist"} {
+                            assert_encoding quicklist dstlist
+                        }
+                    }
+                }
+            }
         }
     }
 

From dab5ec9b8d5c6f7a99a557d65cc95816b981835b Mon Sep 17 00:00:00 2001
From: Uri Shachar <ushachar@apache.org>
Date: Sun, 11 Oct 2020 13:43:23 +0300
Subject: [PATCH 0720/1098] Support getting configuration from both stdin and
 file at the same time (#7893)

This allows supplying secret configuration (for example - masterauth) via a secure channel
instead of having it in a plaintext file / command line param, while still allowing for most
of the configuration to reside there.

Also, remove 'special' case handling for --check-rdb which hasn't been relevant
since 4.0.0.
---
 src/config.c | 27 ++++++++++++++------------
 src/server.c | 53 ++++++++++++++++++++++++++--------------------------
 src/server.h |  2 +-
 3 files changed, 42 insertions(+), 40 deletions(-)

diff --git a/src/config.c b/src/config.c
index 3d5a79eab..350469e52 100644
--- a/src/config.c
+++ b/src/config.c
@@ -467,7 +467,7 @@ void loadServerConfigFromString(char *config) {
                 fclose(logfp);
             }
         } else if (!strcasecmp(argv[0],"include") && argc == 2) {
-            loadServerConfig(argv[1],NULL);
+            loadServerConfig(argv[1], 0, NULL);
         } else if ((!strcasecmp(argv[0],"client-query-buffer-limit")) && argc == 2) {
              server.client_max_querybuf_len = memtoll(argv[1],NULL);
         } else if ((!strcasecmp(argv[0],"slaveof") ||
@@ -619,28 +619,31 @@ loaderr:
  * Both filename and options can be NULL, in such a case are considered
  * empty. This way loadServerConfig can be used to just load a file or
  * just load a string. */
-void loadServerConfig(char *filename, char *options) {
+void loadServerConfig(char *filename, char config_from_stdin, char *options) {
     sds config = sdsempty();
     char buf[CONFIG_MAX_LINE+1];
+    FILE *fp;
 
     /* Load the file content */
     if (filename) {
-        FILE *fp;
-
-        if (filename[0] == '-' && filename[1] == '\0') {
-            fp = stdin;
-        } else {
-            if ((fp = fopen(filename,"r")) == NULL) {
-                serverLog(LL_WARNING,
+        if ((fp = fopen(filename,"r")) == NULL) {
+            serverLog(LL_WARNING,
                     "Fatal error, can't open config file '%s': %s",
                     filename, strerror(errno));
-                exit(1);
-            }
+            exit(1);
         }
         while(fgets(buf,CONFIG_MAX_LINE+1,fp) != NULL)
             config = sdscat(config,buf);
-        if (fp != stdin) fclose(fp);
+        fclose(fp);
     }
+    /* Append content from stdin */
+    if (config_from_stdin) {
+        serverLog(LL_WARNING,"Reading config from stdin");
+        fp = stdin;
+        while(fgets(buf,CONFIG_MAX_LINE+1,fp) != NULL)
+            config = sdscat(config,buf);
+    }
+
     /* Append the additional options */
     if (options) {
         config = sdscat(config,"\n");
diff --git a/src/server.c b/src/server.c
index a9a18ea19..ef40b4fff 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4907,7 +4907,7 @@ void version(void) {
 }
 
 void usage(void) {
-    fprintf(stderr,"Usage: ./redis-server [/path/to/redis.conf] [options]\n");
+    fprintf(stderr,"Usage: ./redis-server [/path/to/redis.conf] [options] [-]\n");
     fprintf(stderr,"       ./redis-server - (read config from stdin)\n");
     fprintf(stderr,"       ./redis-server -v or --version\n");
     fprintf(stderr,"       ./redis-server -h or --help\n");
@@ -4917,6 +4917,7 @@ void usage(void) {
     fprintf(stderr,"       ./redis-server /etc/redis/6379.conf\n");
     fprintf(stderr,"       ./redis-server --port 7777\n");
     fprintf(stderr,"       ./redis-server --port 7777 --replicaof 127.0.0.1 8888\n");
+    fprintf(stderr,"       ./redis-server /etc/myredis.conf --loglevel verbose -\n");
     fprintf(stderr,"       ./redis-server /etc/myredis.conf --loglevel verbose\n\n");
     fprintf(stderr,"Sentinel mode:\n");
     fprintf(stderr,"       ./redis-server /etc/sentinel.conf --sentinel\n");
@@ -5242,6 +5243,7 @@ int iAmMaster(void) {
 int main(int argc, char **argv) {
     struct timeval tv;
     int j;
+    char config_from_stdin = 0;
 
 #ifdef REDIS_TEST
     if (argc == 3 && !strcasecmp(argv[1], "test")) {
@@ -5319,7 +5321,6 @@ int main(int argc, char **argv) {
     if (argc >= 2) {
         j = 1; /* First option to parse in argv[] */
         sds options = sdsempty();
-        char *configfile = NULL;
 
         /* Handle special options --help and --version */
         if (strcmp(argv[1], "-v") == 0 ||
@@ -5336,30 +5337,29 @@ int main(int argc, char **argv) {
                 exit(1);
             }
         }
-
-        /* First argument is the config file name? */
-        if (argv[j][0] != '-' || argv[j][1] != '-') {
-            configfile = argv[j];
-            server.configfile = getAbsolutePath(configfile);
-            /* Replace the config file in server.exec_argv with
-             * its absolute path. */
-            zfree(server.exec_argv[j]);
-            server.exec_argv[j] = zstrdup(server.configfile);
-            j++;
+        /* Parse command line options
+         * Precedence wise, File, stdin, explicit options -- last config is the one that matters.
+         *
+         * First argument is the config file name? */
+        if (argv[1][0] != '-') {
+            /* Replace the config file in server.exec_argv with its absolute path. */
+            server.configfile = getAbsolutePath(argv[1]);
+            zfree(server.exec_argv[1]);
+            server.exec_argv[1] = zstrdup(server.configfile);
+            j = 2; // Skip this arg when parsing options
         }
 
-        /* All the other options are parsed and conceptually appended to the
-         * configuration file. For instance --port 6380 will generate the
-         * string "port 6380\n" to be parsed after the actual file name
-         * is parsed, if any. */
-        while(j != argc) {
-            if (argv[j][0] == '-' && argv[j][1] == '-') {
+        while(j < argc) {
+            /* Either first or last argument - Should we read config from stdin? */
+            if (argv[j][0] == '-' && argv[j][1] == '\0' && (j == 1 || j == argc-1)) {
+                config_from_stdin = 1;
+            }
+            /* All the other options are parsed and conceptually appended to the
+             * configuration file. For instance --port 6380 will generate the
+             * string "port 6380\n" to be parsed after the actual config file
+             * and stdin input are parsed (if they exist). */
+            else if (argv[j][0] == '-' && argv[j][1] == '-') {
                 /* Option name */
-                if (!strcmp(argv[j], "--check-rdb")) {
-                    /* Argument has no options, need to skip for parsing. */
-                    j++;
-                    continue;
-                }
                 if (sdslen(options)) options = sdscat(options,"\n");
                 options = sdscat(options,argv[j]+2);
                 options = sdscat(options," ");
@@ -5370,14 +5370,13 @@ int main(int argc, char **argv) {
             }
             j++;
         }
-        if (server.sentinel_mode && configfile && *configfile == '-') {
-            serverLog(LL_WARNING,
-                "Sentinel config from STDIN not allowed.");
+
+        if (server.sentinel_mode && ! server.configfile) {
             serverLog(LL_WARNING,
                 "Sentinel needs config file on disk to save state.  Exiting...");
             exit(1);
         }
-        loadServerConfig(configfile,options);
+        loadServerConfig(server.configfile, config_from_stdin, options);
         sdsfree(options);
     }
 
diff --git a/src/server.h b/src/server.h
index 8ddc0b75f..50083652e 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2115,7 +2115,7 @@ int keyspaceEventsStringToFlags(char *classes);
 sds keyspaceEventsFlagsToString(int flags);
 
 /* Configuration */
-void loadServerConfig(char *filename, char *options);
+void loadServerConfig(char *filename, char config_from_stdin, char *options);
 void appendServerSaveParams(time_t seconds, int changes);
 void resetServerSaveParams(void);
 struct rewriteConfigState; /* Forward declaration to export API. */

From 9b7f8ba84b9de1ec846badd45e4004e3cd23f178 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 5 Oct 2020 17:03:17 +0300
Subject: [PATCH 0721/1098] Introduce getKeysResult for getKeysFromCommand.

Avoid using a static buffer for short key index responses, and make it
caller's responsibility to stack-allocate a result type. Responses that
don't fit are still allocated on the heap.
---
 src/acl.c      |   9 +--
 src/cluster.c  |  11 ++--
 src/db.c       | 169 +++++++++++++++++++++++++++----------------------
 src/module.c   |  33 +++++++---
 src/server.c   |  14 ++--
 src/server.h   |  42 ++++++++----
 src/tracking.c |  13 ++--
 7 files changed, 171 insertions(+), 120 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 30fdc8e07..aa3ed197f 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1115,8 +1115,9 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) {
     if (!(c->user->flags & USER_FLAG_ALLKEYS) &&
         (c->cmd->getkeys_proc || c->cmd->firstkey))
     {
-        int numkeys;
-        int *keyidx = getKeysFromCommand(c->cmd,c->argv,c->argc,&numkeys);
+        getKeysResult result = GETKEYS_RESULT_INIT;
+        int numkeys = getKeysFromCommand(c->cmd,c->argv,c->argc,&result);
+        int *keyidx = result.keys;
         for (int j = 0; j < numkeys; j++) {
             listIter li;
             listNode *ln;
@@ -1137,11 +1138,11 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) {
             }
             if (!match) {
                 if (keyidxptr) *keyidxptr = keyidx[j];
-                getKeysFreeResult(keyidx);
+                getKeysFreeResult(&result);
                 return ACL_DENIED_KEY;
             }
         }
-        getKeysFreeResult(keyidx);
+        getKeysFreeResult(&result);
     }
 
     /* If we survived all the above checks, the user can execute the
diff --git a/src/cluster.c b/src/cluster.c
index f7f033ba3..248d5c17d 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -5644,7 +5644,10 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
         margc = ms->commands[i].argc;
         margv = ms->commands[i].argv;
 
-        keyindex = getKeysFromCommand(mcmd,margv,margc,&numkeys);
+        getKeysResult result = GETKEYS_RESULT_INIT;
+        numkeys = getKeysFromCommand(mcmd,margv,margc,&result);
+        keyindex = result.keys;
+
         for (j = 0; j < numkeys; j++) {
             robj *thiskey = margv[keyindex[j]];
             int thisslot = keyHashSlot((char*)thiskey->ptr,
@@ -5662,7 +5665,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
                  * not trapped earlier in processCommand(). Report the same
                  * error to the client. */
                 if (n == NULL) {
-                    getKeysFreeResult(keyindex);
+                    getKeysFreeResult(&result);
                     if (error_code)
                         *error_code = CLUSTER_REDIR_DOWN_UNBOUND;
                     return NULL;
@@ -5686,7 +5689,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
                 if (!equalStringObjects(firstkey,thiskey)) {
                     if (slot != thisslot) {
                         /* Error: multiple keys from different slots. */
-                        getKeysFreeResult(keyindex);
+                        getKeysFreeResult(&result);
                         if (error_code)
                             *error_code = CLUSTER_REDIR_CROSS_SLOT;
                         return NULL;
@@ -5705,7 +5708,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
                 missing_keys++;
             }
         }
-        getKeysFreeResult(keyindex);
+        getKeysFreeResult(&result);
     }
 
     /* No key at all in command? then we can serve the request
diff --git a/src/db.c b/src/db.c
index c004468d8..d30956c9a 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1314,27 +1314,54 @@ int expireIfNeeded(redisDb *db, robj *key) {
 /* -----------------------------------------------------------------------------
  * API to get key arguments from commands
  * ---------------------------------------------------------------------------*/
-#define MAX_KEYS_BUFFER 256
-static int getKeysTempBuffer[MAX_KEYS_BUFFER];
+
+/* Prepare the getKeysResult struct to hold numkeys, either by using the
+ * pre-allocated keysbuf or by allocating a new array on the heap.
+ *
+ * This function must be called at least once before starting to populate
+ * the result, and can be called repeatedly to enlarge the result array.
+ */
+int *getKeysPrepareResult(getKeysResult *result, int numkeys) {
+    /* GETKEYS_RESULT_INIT initializes keys to NULL, point it to the pre-allocated stack
+     * buffer here. */
+    if (!result->keys) {
+        serverAssert(!result->numkeys);
+        result->keys = result->keysbuf;
+    }
+
+    /* Resize if necessary */
+    if (numkeys > result->size) {
+        if (result->keys != result->keysbuf) {
+            /* We're not using a static buffer, just (re)alloc */
+            result->keys = zrealloc(result->keys, numkeys * sizeof(int));
+        } else {
+            /* We are using a static buffer, copy its contents */
+            result->keys = zmalloc(numkeys * sizeof(int));
+            if (result->numkeys)
+                memcpy(result->keys, result->keysbuf, result->numkeys * sizeof(int));
+        }
+        result->size = numkeys;
+    }
+
+    return result->keys;
+}
 
 /* The base case is to use the keys position as given in the command table
  * (firstkey, lastkey, step). */
-int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, int *numkeys) {
+int getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, getKeysResult *result) {
     int j, i = 0, last, *keys;
     UNUSED(argv);
 
     if (cmd->firstkey == 0) {
-        *numkeys = 0;
-        return NULL;
+        result->numkeys = 0;
+        return 0;
     }
 
     last = cmd->lastkey;
     if (last < 0) last = argc+last;
 
     int count = ((last - cmd->firstkey)+1);
-    keys = getKeysTempBuffer;
-    if (count > MAX_KEYS_BUFFER)
-        keys = zmalloc(sizeof(int)*count);
+    keys = getKeysPrepareResult(result, count);
 
     for (j = cmd->firstkey; j <= last; j += cmd->keystep) {
         if (j >= argc) {
@@ -1345,17 +1372,17 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in
              * return no keys and expect the command implementation to report
              * an arity or syntax error. */
             if (cmd->flags & CMD_MODULE || cmd->arity < 0) {
-                getKeysFreeResult(keys);
-                *numkeys = 0;
-                return NULL;
+                getKeysFreeResult(result);
+                result->numkeys = 0;
+                return 0;
             } else {
                 serverPanic("Redis built-in command declared keys positions not matching the arity requirements.");
             }
         }
         keys[i++] = j;
     }
-    *numkeys = i;
-    return keys;
+    result->numkeys = i;
+    return i;
 }
 
 /* Return all the arguments that are keys in the command passed via argc / argv.
@@ -1369,20 +1396,20 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in
  *
  * This function uses the command table if a command-specific helper function
  * is not required, otherwise it calls the command-specific function. */
-int *getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
+int getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
     if (cmd->flags & CMD_MODULE_GETKEYS) {
-        return moduleGetCommandKeysViaAPI(cmd,argv,argc,numkeys);
+        return moduleGetCommandKeysViaAPI(cmd,argv,argc,result);
     } else if (!(cmd->flags & CMD_MODULE) && cmd->getkeys_proc) {
-        return cmd->getkeys_proc(cmd,argv,argc,numkeys);
+        return cmd->getkeys_proc(cmd,argv,argc,result);
     } else {
-        return getKeysUsingCommandTable(cmd,argv,argc,numkeys);
+        return getKeysUsingCommandTable(cmd,argv,argc,result);
     }
 }
 
 /* Free the result of getKeysFromCommand. */
-void getKeysFreeResult(int *result) {
-    if (result != getKeysTempBuffer)
-        zfree(result);
+void getKeysFreeResult(getKeysResult *result) {
+    if (result && result->keys != result->keysbuf)
+        zfree(result->keys);
 }
 
 /* Helper function to extract keys from following commands:
@@ -1397,43 +1424,42 @@ void getKeysFreeResult(int *result) {
  * 'firstKeyOfs': firstkey index.
  * 'keyStep': the interval of each key, usually this value is 1.
  * */
-int *genericGetKeys(int storeKeyOfs, int keyCountOfs, int firstKeyOfs, int keyStep,
-                    robj **argv, int argc, int *numkeys) {
+int genericGetKeys(int storeKeyOfs, int keyCountOfs, int firstKeyOfs, int keyStep,
+                    robj **argv, int argc, getKeysResult *result) {
     int i, num, *keys;
 
     num = atoi(argv[keyCountOfs]->ptr);
     /* Sanity check. Don't return any key if the command is going to
      * reply with syntax error. (no input keys). */
     if (num < 1 || num > (argc - firstKeyOfs)/keyStep) {
-        *numkeys = 0;
-        return NULL;
+        result->numkeys = 0;
+        return 0;
     }
 
-    keys = getKeysTempBuffer;
-    *numkeys = storeKeyOfs ? num + 1 : num;
-    if (*numkeys > MAX_KEYS_BUFFER)
-        keys = zmalloc(sizeof(int)*(*numkeys));
+    int numkeys = storeKeyOfs ? num + 1 : num;
+    keys = getKeysPrepareResult(result, numkeys);
+    result->numkeys = numkeys;
 
     /* Add all key positions for argv[firstKeyOfs...n] to keys[] */
     for (i = 0; i < num; i++) keys[i] = firstKeyOfs+(i*keyStep);
 
     if (storeKeyOfs) keys[num] = storeKeyOfs;
-    return keys;
+    return result->numkeys;
 }
 
-int *zunionInterStoreGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
+int zunionInterStoreGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
     UNUSED(cmd);
-    return genericGetKeys(1, 2, 3, 1, argv, argc, numkeys);
+    return genericGetKeys(1, 2, 3, 1, argv, argc, result);
 }
 
-int *zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
+int zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
     UNUSED(cmd);
-    return genericGetKeys(0, 1, 2, 1, argv, argc, numkeys);
+    return genericGetKeys(0, 1, 2, 1, argv, argc, result);
 }
 
-int *evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
+int evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
     UNUSED(cmd);
-    return genericGetKeys(0, 2, 3, 1, argv, argc, numkeys);
+    return genericGetKeys(0, 2, 3, 1, argv, argc, result);
 }
 
 /* Helper function to extract keys from the SORT command.
@@ -1443,13 +1469,12 @@ int *evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys)
  * The first argument of SORT is always a key, however a list of options
  * follow in SQL-alike style. Here we parse just the minimum in order to
  * correctly identify keys in the "STORE" option. */
-int *sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
+int sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
     int i, j, num, *keys, found_store = 0;
     UNUSED(cmd);
 
     num = 0;
-    keys = getKeysTempBuffer; /* Alloc 2 places for the worst case. */
-
+    keys = getKeysPrepareResult(result, 2); /* Alloc 2 places for the worst case. */
     keys[num++] = 1; /* <sort-key> is always present. */
 
     /* Search for STORE option. By default we consider options to don't
@@ -1481,11 +1506,11 @@ int *sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys)
             }
         }
     }
-    *numkeys = num + found_store;
-    return keys;
+    result->numkeys = num + found_store;
+    return result->numkeys;
 }
 
-int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
+int migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
     int i, num, first, *keys;
     UNUSED(cmd);
 
@@ -1506,20 +1531,17 @@ int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkey
         }
     }
 
-    keys = getKeysTempBuffer;
-    if (num>MAX_KEYS_BUFFER)
-        keys = zmalloc(sizeof(int)*num);
-
+    keys = getKeysPrepareResult(result, num);
     for (i = 0; i < num; i++) keys[i] = first+i;
-    *numkeys = num;
-    return keys;
+    result->numkeys = num;
+    return num;
 }
 
 /* Helper function to extract keys from following commands:
  * GEORADIUS key x y radius unit [WITHDIST] [WITHHASH] [WITHCOORD] [ASC|DESC]
  *                             [COUNT count] [STORE key] [STOREDIST key]
  * GEORADIUSBYMEMBER key member radius unit ... options ... */
-int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
+int georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
     int i, num, *keys;
     UNUSED(cmd);
 
@@ -1542,24 +1564,21 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk
      * argv[1] = key,
      * argv[5...n] = stored key if present
      */
-    keys = getKeysTempBuffer;
-    if (num>MAX_KEYS_BUFFER)
-        keys = zmalloc(sizeof(int) * num);
+    keys = getKeysPrepareResult(result, num);
 
     /* Add all key positions to keys[] */
     keys[0] = 1;
     if(num > 1) {
          keys[1] = stored_key;
     }
-    *numkeys = num;
-    return keys;
+    result->numkeys = num;
+    return num;
 }
 
 /* LCS ... [KEYS <key1> <key2>] ... */
-int *lcsGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys)
-{
+int lcsGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
     int i;
-    int *keys = getKeysTempBuffer;
+    int *keys = getKeysPrepareResult(result, 2);
     UNUSED(cmd);
 
     /* We need to parse the options of the command in order to check for the
@@ -1573,33 +1592,32 @@ int *lcsGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys)
         } else if (!strcasecmp(arg, "keys") && moreargs >= 2) {
             keys[0] = i+1;
             keys[1] = i+2;
-            *numkeys = 2;
-            return keys;
+            result->numkeys = 2;
+            return result->numkeys;
         }
     }
-    *numkeys = 0;
-    return keys;
+    result->numkeys = 0;
+    return result->numkeys;
 }
 
 /* Helper function to extract keys from memory command.
  * MEMORY USAGE <key> */
-int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
-    int *keys;
+int memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
     UNUSED(cmd);
 
+    getKeysPrepareResult(result, 1);
     if (argc >= 3 && !strcasecmp(argv[1]->ptr,"usage")) {
-        keys = getKeysTempBuffer;
-        keys[0] = 2;
-        *numkeys = 1;
-        return keys;
+        result->keys[0] = 2;
+        result->numkeys = 1;
+        return result->numkeys;
     }
-    *numkeys = 0;
-    return NULL;
+    result->numkeys = 0;
+    return 0;
 }
 
 /* XREAD [BLOCK <milliseconds>] [COUNT <count>] [GROUP <groupname> <ttl>]
  *       STREAMS key_1 key_2 ... key_N ID_1 ID_2 ... ID_N */
-int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
+int xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
     int i, num = 0, *keys;
     UNUSED(cmd);
 
@@ -1629,19 +1647,16 @@ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys)
 
     /* Syntax error. */
     if (streams_pos == -1 || num == 0 || num % 2 != 0) {
-        *numkeys = 0;
-        return NULL;
+        result->numkeys = 0;
+        return 0;
     }
     num /= 2; /* We have half the keys as there are arguments because
                  there are also the IDs, one per key. */
 
-    keys = getKeysTempBuffer;
-    if (num>MAX_KEYS_BUFFER)
-        keys = zmalloc(sizeof(int) * num);
-
+    keys = getKeysPrepareResult(result, num);
     for (i = streams_pos+1; i < argc-num; i++) keys[i-streams_pos-1] = i;
-    *numkeys = num;
-    return keys;
+    result->numkeys = num;
+    return num;
 }
 
 /* Slot to Key API. This is used by Redis Cluster in order to obtain in
diff --git a/src/module.c b/src/module.c
index 3f3d92aea..a48780be9 100644
--- a/src/module.c
+++ b/src/module.c
@@ -147,8 +147,7 @@ struct RedisModuleCtx {
                                              on keys. */
 
     /* Used if there is the REDISMODULE_CTX_KEYS_POS_REQUEST flag set. */
-    int *keys_pos;
-    int keys_count;
+    getKeysResult *keys_result;
 
     struct RedisModulePoolAllocBlock *pa_head;
     redisOpArray saved_oparray;    /* When propagating commands in a callback
@@ -158,7 +157,7 @@ struct RedisModuleCtx {
 };
 typedef struct RedisModuleCtx RedisModuleCtx;
 
-#define REDISMODULE_CTX_INIT {(void*)(unsigned long)&RM_GetApi, NULL, NULL, NULL, NULL, 0, 0, 0, NULL, 0, NULL, NULL, NULL, 0, NULL, {0}}
+#define REDISMODULE_CTX_INIT {(void*)(unsigned long)&RM_GetApi, NULL, NULL, NULL, NULL, 0, 0, 0, NULL, 0, NULL, NULL, NULL, NULL, {0}}
 #define REDISMODULE_CTX_MULTI_EMITTED (1<<0)
 #define REDISMODULE_CTX_AUTO_MEMORY (1<<1)
 #define REDISMODULE_CTX_KEYS_POS_REQUEST (1<<2)
@@ -665,18 +664,24 @@ void RedisModuleCommandDispatcher(client *c) {
  * In order to accomplish its work, the module command is called, flagging
  * the context in a way that the command can recognize this is a special
  * "get keys" call by calling RedisModule_IsKeysPositionRequest(ctx). */
-int *moduleGetCommandKeysViaAPI(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
+int moduleGetCommandKeysViaAPI(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
     RedisModuleCommandProxy *cp = (void*)(unsigned long)cmd->getkeys_proc;
     RedisModuleCtx ctx = REDISMODULE_CTX_INIT;
 
     ctx.module = cp->module;
     ctx.client = NULL;
     ctx.flags |= REDISMODULE_CTX_KEYS_POS_REQUEST;
+
+    /* Initialize getKeysResult */
+    getKeysPrepareResult(result, MAX_KEYS_BUFFER);
+    ctx.keys_result = result;
+
     cp->func(&ctx,(void**)argv,argc);
-    int *res = ctx.keys_pos;
-    if (numkeys) *numkeys = ctx.keys_count;
+    /* We currently always use the array allocated by RM_KeyAtPos() and don't try
+     * to optimize for the pre-allocated buffer.
+     */
     moduleFreeContext(&ctx);
-    return res;
+    return result->numkeys;
 }
 
 /* Return non-zero if a module command, that was declared with the
@@ -701,10 +706,18 @@ int RM_IsKeysPositionRequest(RedisModuleCtx *ctx) {
  *  keys are at fixed positions. This interface is only used for commands
  *  with a more complex structure. */
 void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) {
-    if (!(ctx->flags & REDISMODULE_CTX_KEYS_POS_REQUEST)) return;
+    if (!(ctx->flags & REDISMODULE_CTX_KEYS_POS_REQUEST) || !ctx->keys_result) return;
     if (pos <= 0) return;
-    ctx->keys_pos = zrealloc(ctx->keys_pos,sizeof(int)*(ctx->keys_count+1));
-    ctx->keys_pos[ctx->keys_count++] = pos;
+
+    getKeysResult *res = ctx->keys_result;
+
+    /* Check overflow */
+    if (res->numkeys == res->size) {
+        int newsize = res->size + (res->size > 8192 ? 8192 : res->size);
+        getKeysPrepareResult(res, newsize);
+    }
+
+    res->keys[res->numkeys++] = pos;
 }
 
 /* Helper for RM_CreateCommand(). Turns a string representing command
diff --git a/src/server.c b/src/server.c
index ef40b4fff..6ae7037af 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3273,7 +3273,7 @@ struct redisCommand *lookupCommand(sds name) {
     return dictFetchValue(server.commands, name);
 }
 
-struct redisCommand *lookupCommandByCString(char *s) {
+struct redisCommand *lookupCommandByCString(const char *s) {
     struct redisCommand *cmd;
     sds name = sdsnew(s);
 
@@ -4142,7 +4142,8 @@ NULL
         addReplyLongLong(c, dictSize(server.commands));
     } else if (!strcasecmp(c->argv[1]->ptr,"getkeys") && c->argc >= 3) {
         struct redisCommand *cmd = lookupCommand(c->argv[2]->ptr);
-        int *keys, numkeys, j;
+        getKeysResult result = GETKEYS_RESULT_INIT;
+        int j;
 
         if (!cmd) {
             addReplyError(c,"Invalid command specified");
@@ -4157,14 +4158,13 @@ NULL
             return;
         }
 
-        keys = getKeysFromCommand(cmd,c->argv+2,c->argc-2,&numkeys);
-        if (!keys) {
+        if (!getKeysFromCommand(cmd,c->argv+2,c->argc-2,&result)) {
             addReplyError(c,"Invalid arguments specified for command");
         } else {
-            addReplyArrayLen(c,numkeys);
-            for (j = 0; j < numkeys; j++) addReplyBulk(c,c->argv[keys[j]+2]);
-            getKeysFreeResult(keys);
+            addReplyArrayLen(c,result.numkeys);
+            for (j = 0; j < result.numkeys; j++) addReplyBulk(c,c->argv[result.keys[j]+2]);
         }
+        getKeysFreeResult(&result);
     } else {
         addReplySubcommandSyntaxError(c);
     }
diff --git a/src/server.h b/src/server.h
index 50083652e..a5881e9a2 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1497,8 +1497,21 @@ typedef struct pubsubPattern {
     robj *pattern;
 } pubsubPattern;
 
+#define MAX_KEYS_BUFFER 256
+
+/* A result structure for the various getkeys function calls. It lists the
+ * keys as indices to the provided argv.
+ */
+typedef struct {
+    int keysbuf[MAX_KEYS_BUFFER];       /* Pre-allocated buffer, to save heap allocations */
+    int *keys;                          /* Key indices array, points to keysbuf or heap */
+    int numkeys;                        /* Number of key indices return */
+    int size;                           /* Available array size */
+} getKeysResult;
+#define GETKEYS_RESULT_INIT { {0}, NULL, 0, MAX_KEYS_BUFFER }
+
 typedef void redisCommandProc(client *c);
-typedef int *redisGetKeysProc(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
+typedef int redisGetKeysProc(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
 struct redisCommand {
     char *name;
     redisCommandProc *proc;
@@ -1608,7 +1621,7 @@ void moduleInitModulesSystem(void);
 void moduleInitModulesSystemLast(void);
 int moduleLoad(const char *path, void **argv, int argc);
 void moduleLoadFromQueue(void);
-int *moduleGetCommandKeysViaAPI(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
+int moduleGetCommandKeysViaAPI(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
 moduleType *moduleTypeLookupModuleByID(uint64_t id);
 void moduleTypeNameByID(char *name, uint64_t moduleid);
 void moduleFreeContext(struct RedisModuleCtx *ctx);
@@ -2020,7 +2033,7 @@ int processCommand(client *c);
 void setupSignalHandlers(void);
 void removeSignalHandlers(void);
 struct redisCommand *lookupCommand(sds name);
-struct redisCommand *lookupCommandByCString(char *s);
+struct redisCommand *lookupCommandByCString(const char *s);
 struct redisCommand *lookupCommandOrOriginal(sds name);
 void call(client *c, int flags);
 void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, int flags);
@@ -2181,17 +2194,18 @@ size_t lazyfreeGetPendingObjectsCount(void);
 void freeObjAsync(robj *o);
 
 /* API to get key arguments from commands */
-int *getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
-void getKeysFreeResult(int *result);
-int *zunionInterGetKeys(struct redisCommand *cmd,robj **argv, int argc, int *numkeys);
-int *zunionInterStoreGetKeys(struct redisCommand *cmd,robj **argv, int argc, int *numkeys);
-int *evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
-int *sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
-int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
-int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
-int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
-int *memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
-int *lcsGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys);
+int *getKeysPrepareResult(getKeysResult *result, int numkeys);
+int getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
+void getKeysFreeResult(getKeysResult *result);
+int zunionInterGetKeys(struct redisCommand *cmd,robj **argv, int argc, getKeysResult *result);
+int zunionInterStoreGetKeys(struct redisCommand *cmd,robj **argv, int argc, getKeysResult *result);
+int evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
+int sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
+int migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
+int georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
+int xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
+int memoryGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
+int lcsGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
 
 /* Cluster */
 void clusterInit(void);
diff --git a/src/tracking.c b/src/tracking.c
index 3737f6859..913577eab 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -171,9 +171,14 @@ void trackingRememberKeys(client *c) {
     uint64_t caching_given = c->flags & CLIENT_TRACKING_CACHING;
     if ((optin && !caching_given) || (optout && caching_given)) return;
 
-    int numkeys;
-    int *keys = getKeysFromCommand(c->cmd,c->argv,c->argc,&numkeys);
-    if (keys == NULL) return;
+    getKeysResult result = GETKEYS_RESULT_INIT;
+    int numkeys = getKeysFromCommand(c->cmd,c->argv,c->argc,&result);
+    if (!numkeys) {
+        getKeysFreeResult(&result);
+        return;
+    }
+
+    int *keys = result.keys;
 
     for(int j = 0; j < numkeys; j++) {
         int idx = keys[j];
@@ -188,7 +193,7 @@ void trackingRememberKeys(client *c) {
         if (raxTryInsert(ids,(unsigned char*)&c->id,sizeof(c->id),NULL,NULL))
             TrackingTableTotalItems++;
     }
-    getKeysFreeResult(keys);
+    getKeysFreeResult(&result);
 }
 
 /* Given a key name, this function sends an invalidation message in the

From 7d117d7591656e947f526f5d5f8a022b88b38ad9 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 5 Oct 2020 17:06:35 +0300
Subject: [PATCH 0722/1098] Modules: add RM_GetCommandKeys().

This is essentially the same as calling COMMAND GETKEYS but provides a
more efficient interface that can be used in every context (i.e. not a
Redis command).
---
 runtest-moduleapi                |   1 +
 src/module.c                     |  64 ++++++++++++++++
 src/redismodule.h                |   2 +
 tests/modules/Makefile           |   3 +-
 tests/modules/getkeys.c          | 125 +++++++++++++++++++++++++++++++
 tests/unit/moduleapi/getkeys.tcl |  44 +++++++++++
 6 files changed, 238 insertions(+), 1 deletion(-)
 create mode 100644 tests/modules/getkeys.c
 create mode 100644 tests/unit/moduleapi/getkeys.tcl

diff --git a/runtest-moduleapi b/runtest-moduleapi
index e32147d5d..f881dfd3f 100755
--- a/runtest-moduleapi
+++ b/runtest-moduleapi
@@ -27,4 +27,5 @@ $TCLSH tests/test_helper.tcl \
 --single unit/moduleapi/auth \
 --single unit/moduleapi/keyspace_events \
 --single unit/moduleapi/blockedclient \
+--single unit/moduleapi/getkeys \
 "${@}"
diff --git a/src/module.c b/src/module.c
index a48780be9..1e18f3c46 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7887,6 +7887,69 @@ int RM_ModuleTypeReplaceValue(RedisModuleKey *key, moduleType *mt, void *new_val
     return REDISMODULE_OK;
 }
 
+/* For a specified command, parse its arguments and return an array that
+ * contains the indexes of all key name arguments. This function is
+ * essnetially a more efficient way to do COMMAND GETKEYS.
+ *
+ * A NULL return value indicates the specified command has no keys, or
+ * an error condition. Error conditions are indicated by setting errno
+ * as folllows:
+ *
+ *  ENOENT: Specified command does not exist.
+ *  EINVAL: Invalid command arity specified.
+ *
+ * NOTE: The returned array is not a Redis Module object so it does not
+ * get automatically freed even when auto-memory is used. The caller
+ * must explicitly call RM_Free() to free it.
+ */
+int *RM_GetCommandKeys(RedisModuleCtx *ctx, const char *cmdname, RedisModuleString **argv, int argc, int *num_keys) {
+    UNUSED(ctx);
+    struct redisCommand *cmd;
+    int *res = NULL;
+
+    /* Find command */
+    if ((cmd = lookupCommandByCString(cmdname)) == NULL) {
+        errno = ENOENT;
+        return NULL;
+    }
+
+    /* Bail out if command has no keys */
+    if (cmd->getkeys_proc == NULL && cmd->firstkey == 0) {
+        errno = 0;
+        return NULL;
+    }
+
+    if ((cmd->arity > 0 && cmd->arity != argc) || (argc < -cmd->arity)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    getKeysResult result = GETKEYS_RESULT_INIT;
+    getKeysFromCommand(cmd, argv, argc, &result);
+
+    *num_keys = result.numkeys;
+    if (!result.numkeys) {
+        errno = 0;
+        getKeysFreeResult(&result);
+        return NULL;
+    }
+
+    if (result.keys == result.keysbuf) {
+        /* If the result is using a stack based array, copy it. */
+        unsigned long int size = sizeof(int) * result.numkeys;
+        res = zmalloc(size);
+        memcpy(res, result.keys, size);
+    } else {
+        /* We return the heap based array and intentionally avoid calling
+         * getKeysFreeResult() here, as it is the caller's responsibility
+         * to free this array.
+         */
+        res = result.keys;
+    }
+
+    return res;
+}
+
 /* Register all the APIs we export. Keep this function at the end of the
  * file so that's easy to seek it to add new entries. */
 void moduleRegisterCoreAPI(void) {
@@ -8122,4 +8185,5 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(DeauthenticateAndCloseClient);
     REGISTER_API(AuthenticateClientWithACLUser);
     REGISTER_API(AuthenticateClientWithUser);
+    REGISTER_API(GetCommandKeys);
 }
diff --git a/src/redismodule.h b/src/redismodule.h
index 00ca1f578..e05ce65ee 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -726,6 +726,7 @@ REDISMODULE_API int (*RedisModule_SetModuleUserACL)(RedisModuleUser *user, const
 REDISMODULE_API int (*RedisModule_AuthenticateClientWithACLUser)(RedisModuleCtx *ctx, const char *name, size_t len, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_AuthenticateClientWithUser)(RedisModuleCtx *ctx, RedisModuleUser *user, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id) REDISMODULE_ATTR;
+REDISMODULE_API int *(*RedisModule_GetCommandKeys)(RedisModuleCtx *ctx, const char *cmdname, RedisModuleString **argv, int argc, int *num_keys) REDISMODULE_ATTR;
 #endif
 
 #define RedisModule_IsAOFClient(id) ((id) == CLIENT_ID_AOF)
@@ -967,6 +968,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(DeauthenticateAndCloseClient);
     REDISMODULE_GET_API(AuthenticateClientWithACLUser);
     REDISMODULE_GET_API(AuthenticateClientWithUser);
+    REDISMODULE_GET_API(GetCommandKeys);
 #endif
 
     if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR;
diff --git a/tests/modules/Makefile b/tests/modules/Makefile
index fad6e55d8..a8b08ecf0 100644
--- a/tests/modules/Makefile
+++ b/tests/modules/Makefile
@@ -24,7 +24,8 @@ TEST_MODULES = \
     datatype.so \
     auth.so \
     keyspace_events.so \
-    blockedclient.so
+    blockedclient.so \
+    getkeys.so
 
 
 .PHONY: all
diff --git a/tests/modules/getkeys.c b/tests/modules/getkeys.c
new file mode 100644
index 000000000..9cde606ab
--- /dev/null
+++ b/tests/modules/getkeys.c
@@ -0,0 +1,125 @@
+#define REDISMODULE_EXPERIMENTAL_API
+
+#include "redismodule.h"
+#include <strings.h>
+#include <assert.h>
+#include <unistd.h>
+#include <errno.h>
+
+#define UNUSED(V) ((void) V)
+
+/* A sample movable keys command that returns a list of all
+ * arguments that follow a KEY argument, i.e.
+ */
+int getkeys_command(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    int i;
+    int count = 0;
+
+    /* Handle getkeys-api introspection */
+    if (RedisModule_IsKeysPositionRequest(ctx)) {
+        for (i = 0; i < argc; i++) {
+            size_t len;
+            const char *str = RedisModule_StringPtrLen(argv[i], &len);
+
+            if (len == 3 && !strncasecmp(str, "key", 3) && i + 1 < argc)
+                RedisModule_KeyAtPos(ctx, i + 1);
+        }
+
+        return REDISMODULE_OK;
+    }
+
+    /* Handle real command invocation */
+    RedisModule_ReplyWithArray(ctx, REDISMODULE_POSTPONED_ARRAY_LEN);
+    for (i = 0; i < argc; i++) {
+        size_t len;
+        const char *str = RedisModule_StringPtrLen(argv[i], &len);
+
+        if (len == 3 && !strncasecmp(str, "key", 3) && i + 1 < argc) {
+            RedisModule_ReplyWithString(ctx, argv[i+1]);
+            count++;
+        }
+    }
+    RedisModule_ReplySetArrayLength(ctx, count);
+
+    return REDISMODULE_OK;
+}
+
+int getkeys_fixed(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    int i;
+
+    RedisModule_ReplyWithArray(ctx, argc - 1);
+    for (i = 1; i < argc; i++) {
+        RedisModule_ReplyWithString(ctx, argv[i]);
+    }
+    return REDISMODULE_OK;
+}
+
+/* Introspect a command using RM_GetCommandKeys() and returns the list
+ * of keys. Essentially this is COMMAND GETKEYS implemented in a module.
+ */
+int getkeys_introspect(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    UNUSED(argv);
+    UNUSED(argc);
+
+    if (argc < 3) {
+        RedisModule_WrongArity(ctx);
+        return REDISMODULE_OK;
+    }
+
+    size_t cmd_len;
+    const char *cmd = RedisModule_StringPtrLen(argv[1], &cmd_len);
+
+    int num_keys;
+    int *keyidx = RedisModule_GetCommandKeys(ctx, cmd, &argv[1], argc - 1, &num_keys);
+
+    if (!keyidx) {
+        if (!errno)
+            RedisModule_ReplyWithEmptyArray(ctx);
+        else {
+            char err[100];
+            switch (errno) {
+                case ENOENT:
+                    RedisModule_ReplyWithError(ctx, "ERR ENOENT");
+                    break;
+                case EINVAL:
+                    RedisModule_ReplyWithError(ctx, "ERR EINVAL");
+                    break;
+                default:
+                    snprintf(err, sizeof(err) - 1, "ERR errno=%d", errno);
+                    RedisModule_ReplyWithError(ctx, err);
+                    break;
+            }
+        }
+    } else {
+        int i;
+
+        RedisModule_ReplyWithArray(ctx, num_keys);
+        for (i = 0; i < num_keys; i++)
+            RedisModule_ReplyWithString(ctx, argv[1 + keyidx[i]]);
+
+        RedisModule_Free(keyidx);
+    }
+
+    return REDISMODULE_OK;
+}
+
+int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    UNUSED(argv);
+    UNUSED(argc);
+    if (RedisModule_Init(ctx,"getkeys",1,REDISMODULE_APIVER_1)== REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx,"getkeys.command", getkeys_command,"getkeys-api",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx,"getkeys.fixed", getkeys_fixed,"",2,4,1) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx,"getkeys.introspect", getkeys_introspect,"",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    return REDISMODULE_OK;
+}
diff --git a/tests/unit/moduleapi/getkeys.tcl b/tests/unit/moduleapi/getkeys.tcl
new file mode 100644
index 000000000..a24a65bae
--- /dev/null
+++ b/tests/unit/moduleapi/getkeys.tcl
@@ -0,0 +1,44 @@
+set testmodule [file normalize tests/modules/getkeys.so]
+
+start_server {tags {"modules"}} {
+    r module load $testmodule
+
+    test {COMMAND INFO correctly reports a movable keys module command} {
+        set info [lindex [r command info getkeys.command] 0]
+
+        assert_equal {movablekeys} [lindex $info 2]
+        assert_equal {0} [lindex $info 3]
+        assert_equal {0} [lindex $info 4]
+        assert_equal {0} [lindex $info 5]
+    }
+
+    test {COMMAND GETKEYS correctly reports a movable keys module command} {
+        r command getkeys getkeys.command arg1 arg2 key key1 arg3 key key2 key key3
+    } {key1 key2 key3}
+
+    test {RM_GetCommandKeys on non-existing command} {
+        catch {r getkeys.introspect non-command key1 key2} e
+        set _ $e
+    } {*ENOENT*}
+
+    test {RM_GetCommandKeys on built-in fixed keys command} {
+        r getkeys.introspect set key1 value1
+    } {key1}
+
+    test {RM_GetCommandKeys on EVAL} {
+        r getkeys.introspect eval "" 4 key1 key2 key3 key4 arg1 arg2
+    } {key1 key2 key3 key4}
+
+    test {RM_GetCommandKeys on a movable keys module command} {
+        r getkeys.introspect getkeys.command arg1 arg2 key key1 arg3 key key2 key key3
+    } {key1 key2 key3}
+
+    test {RM_GetCommandKeys on a non-movable module command} {
+        r getkeys.introspect getkeys.fixed arg1 key1 key2 key3 arg2
+    } {key1 key2 key3}
+
+    test {RM_GetCommandKeys with bad arity} {
+        catch {r getkeys.introspect set key} e
+        set _ $e
+    } {*EINVAL*}
+}

From 907da0580b57013c9e5c38b27f351597c72e4e25 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 11 Oct 2020 16:11:31 +0300
Subject: [PATCH 0723/1098] Modules: Add RM_GetDetachedThreadSafeContext().
 (#7886)

The main motivation here is to provide a way for modules to create a
single, global context that can be used for logging.

Currently, it is possible to obtain a thread-safe context that is not
attached to any blocked client by using `RM_GetThreadSafeContext`.
However, the attached context is not linked to the module identity so
log messages produced are not tagged with the module name.

Ideally we'd fix this in `RM_GetThreadSafeContext` itself but as it
doesn't accept the current context as an argument there's no way to do
that in a backwards compatible manner.
---
 src/module.c                  | 21 +++++++++++++++++++--
 src/redismodule.h             |  2 ++
 tests/modules/misc.c          | 26 ++++++++++++++++++++++++++
 tests/unit/moduleapi/misc.tcl |  5 +++++
 4 files changed, 52 insertions(+), 2 deletions(-)

diff --git a/src/module.c b/src/module.c
index 1e18f3c46..ef39d5d3a 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4901,8 +4901,9 @@ int RM_BlockedClientDisconnected(RedisModuleCtx *ctx) {
  * that a blocked client was used when the context was created, otherwise
  * no RedisModule_Reply* call should be made at all.
  *
- * TODO: thread safe contexts do not inherit the blocked client
- * selected database. */
+ * NOTE: If you're creating a detached thread safe context (bc is NULL),
+ * consider using `RM_GetDetachedThreadSafeContext` which will also retain
+ * the module ID and thus be more useful for logging. */
 RedisModuleCtx *RM_GetThreadSafeContext(RedisModuleBlockedClient *bc) {
     RedisModuleCtx *ctx = zmalloc(sizeof(*ctx));
     RedisModuleCtx empty = REDISMODULE_CTX_INIT;
@@ -4924,6 +4925,21 @@ RedisModuleCtx *RM_GetThreadSafeContext(RedisModuleBlockedClient *bc) {
     return ctx;
 }
 
+/* Return a detached thread safe context that is not associated with any
+ * specific blocked client, but is associated with the module's context.
+ *
+ * This is useful for modules that wish to hold a global context over
+ * a long term, for purposes such as logging. */
+RedisModuleCtx *RM_GetDetachedThreadSafeContext(RedisModuleCtx *ctx) {
+    RedisModuleCtx *new_ctx = zmalloc(sizeof(*new_ctx));
+    RedisModuleCtx empty = REDISMODULE_CTX_INIT;
+    memcpy(new_ctx,&empty,sizeof(empty));
+    new_ctx->module = ctx->module;
+    new_ctx->flags |= REDISMODULE_CTX_THREAD_SAFE;
+    new_ctx->client = createClient(NULL);
+    return new_ctx;
+}
+
 /* Release a thread safe context. */
 void RM_FreeThreadSafeContext(RedisModuleCtx *ctx) {
     moduleFreeContext(ctx);
@@ -8086,6 +8102,7 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(AbortBlock);
     REGISTER_API(Milliseconds);
     REGISTER_API(GetThreadSafeContext);
+    REGISTER_API(GetDetachedThreadSafeContext);
     REGISTER_API(FreeThreadSafeContext);
     REGISTER_API(ThreadSafeContextLock);
     REGISTER_API(ThreadSafeContextTryLock);
diff --git a/src/redismodule.h b/src/redismodule.h
index e05ce65ee..092ad3914 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -684,6 +684,7 @@ REDISMODULE_API void * (*RedisModule_GetBlockedClientPrivateData)(RedisModuleCtx
 REDISMODULE_API RedisModuleBlockedClient * (*RedisModule_GetBlockedClientHandle)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_AbortBlock)(RedisModuleBlockedClient *bc) REDISMODULE_ATTR;
 REDISMODULE_API RedisModuleCtx * (*RedisModule_GetThreadSafeContext)(RedisModuleBlockedClient *bc) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleCtx * (*RedisModule_GetDetachedThreadSafeContext)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
 REDISMODULE_API void (*RedisModule_FreeThreadSafeContext)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
 REDISMODULE_API void (*RedisModule_ThreadSafeContextLock)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_ThreadSafeContextTryLock)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
@@ -919,6 +920,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
 
 #ifdef REDISMODULE_EXPERIMENTAL_API
     REDISMODULE_GET_API(GetThreadSafeContext);
+    REDISMODULE_GET_API(GetDetachedThreadSafeContext);
     REDISMODULE_GET_API(FreeThreadSafeContext);
     REDISMODULE_GET_API(ThreadSafeContextLock);
     REDISMODULE_GET_API(ThreadSafeContextTryLock);
diff --git a/tests/modules/misc.c b/tests/modules/misc.c
index 1f9cb1932..a43bb84ac 100644
--- a/tests/modules/misc.c
+++ b/tests/modules/misc.c
@@ -231,6 +231,30 @@ int test_clientinfo(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     return REDISMODULE_OK;
 }
 
+int test_log_tsctx(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    RedisModuleCtx *tsctx = RedisModule_GetDetachedThreadSafeContext(ctx);
+
+    if (argc != 3) {
+        RedisModule_WrongArity(ctx);
+        return REDISMODULE_OK;
+    }
+
+    char level[50];
+    size_t level_len;
+    const char *level_str = RedisModule_StringPtrLen(argv[1], &level_len);
+    snprintf(level, sizeof(level) - 1, "%.*s", (int) level_len, level_str);
+
+    size_t msg_len;
+    const char *msg_str = RedisModule_StringPtrLen(argv[2], &msg_len);
+
+    RedisModule_Log(tsctx, level, "%.*s", (int) msg_len, msg_str);
+    RedisModule_FreeThreadSafeContext(tsctx);
+
+    RedisModule_ReplyWithSimpleString(ctx, "OK");
+    return REDISMODULE_OK;
+}
+
 int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
@@ -259,6 +283,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
         return REDISMODULE_ERR;
     if (RedisModule_CreateCommand(ctx,"test.clientinfo", test_clientinfo,"",0,0,0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
+    if (RedisModule_CreateCommand(ctx,"test.log_tsctx", test_log_tsctx,"",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
 
     return REDISMODULE_OK;
 }
diff --git a/tests/unit/moduleapi/misc.tcl b/tests/unit/moduleapi/misc.tcl
index b57a94f6a..2c9ac4fd0 100644
--- a/tests/unit/moduleapi/misc.tcl
+++ b/tests/unit/moduleapi/misc.tcl
@@ -86,4 +86,9 @@ start_server {tags {"modules"}} {
         set info [r test.clientinfo]
         assert { [dict get $info flags] == "${ssl_flag}::tracking::" }
     }
+
+    test {test detached thread safe cnotext} {
+        r test.log_tsctx "info" "Test message"
+        verify_log_message 0 "*<misc> Test message*" 0
+    }
 }

From 0aec98dce2acbd280ad8ff4feac631e8afa833b1 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 11 Oct 2020 17:11:42 +0300
Subject: [PATCH 0724/1098] Module API: Add RM_GetClientCertificate(). (#7866)

This API function makes it possible to retrieve the X.509 certificate
used by clients to authenticate TLS connections.
---
 src/connection.h              |  1 +
 src/module.c                  | 26 ++++++++++++++++++++++++++
 src/redismodule.h             |  2 ++
 src/tls.c                     | 30 ++++++++++++++++++++++++++++++
 tests/modules/misc.c          | 19 +++++++++++++++++++
 tests/unit/moduleapi/misc.tcl | 10 ++++++++++
 6 files changed, 88 insertions(+)

diff --git a/src/connection.h b/src/connection.h
index e00d2ea17..03281a3d9 100644
--- a/src/connection.h
+++ b/src/connection.h
@@ -230,6 +230,7 @@ int connSockName(connection *conn, char *ip, size_t ip_len, int *port);
 const char *connGetInfo(connection *conn, char *buf, size_t buf_len);
 
 /* Helpers for tls special considerations */
+sds connTLSGetPeerCert(connection *conn);
 int tlsHasPendingData();
 int tlsProcessPendingData();
 
diff --git a/src/module.c b/src/module.c
index ef39d5d3a..7b4b761fe 100644
--- a/src/module.c
+++ b/src/module.c
@@ -5737,6 +5737,31 @@ int RM_DeauthenticateAndCloseClient(RedisModuleCtx *ctx, uint64_t client_id) {
     return REDISMODULE_OK;
 }
 
+/* Return the X.509 client-side certificate used by the client to authenticate
+ * this connection.
+ *
+ * The return value is an allocated RedisModuleString that is a X.509 certificate
+ * encoded in PEM (Base64) format. It should be freed (or auto-freed) by the caller.
+ *
+ * A NULL value is returned in the following conditions:
+ *
+ * - Connection ID does not exist
+ * - Connection is not a TLS connection
+ * - Connection is a TLS connection but no client ceritifcate was used
+ */
+RedisModuleString *RM_GetClientCertificate(RedisModuleCtx *ctx, uint64_t client_id) {
+    client *c = lookupClientByID(client_id);
+    if (c == NULL) return NULL;
+
+    sds cert = connTLSGetPeerCert(c->conn);
+    if (!cert) return NULL;
+
+    RedisModuleString *s = createObject(OBJ_STRING, cert);
+    if (ctx != NULL) autoMemoryAdd(ctx, REDISMODULE_AM_STRING, s);
+
+    return s;
+}
+
 /* --------------------------------------------------------------------------
  * Modules Dictionary API
  *
@@ -8202,5 +8227,6 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(DeauthenticateAndCloseClient);
     REGISTER_API(AuthenticateClientWithACLUser);
     REGISTER_API(AuthenticateClientWithUser);
+    REGISTER_API(GetClientCertificate);
     REGISTER_API(GetCommandKeys);
 }
diff --git a/src/redismodule.h b/src/redismodule.h
index 092ad3914..7b3545585 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -727,6 +727,7 @@ REDISMODULE_API int (*RedisModule_SetModuleUserACL)(RedisModuleUser *user, const
 REDISMODULE_API int (*RedisModule_AuthenticateClientWithACLUser)(RedisModuleCtx *ctx, const char *name, size_t len, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_AuthenticateClientWithUser)(RedisModuleCtx *ctx, RedisModuleUser *user, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_GetClientCertificate)(RedisModuleCtx *ctx, uint64_t id) REDISMODULE_ATTR;
 REDISMODULE_API int *(*RedisModule_GetCommandKeys)(RedisModuleCtx *ctx, const char *cmdname, RedisModuleString **argv, int argc, int *num_keys) REDISMODULE_ATTR;
 #endif
 
@@ -970,6 +971,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(DeauthenticateAndCloseClient);
     REDISMODULE_GET_API(AuthenticateClientWithACLUser);
     REDISMODULE_GET_API(AuthenticateClientWithUser);
+    REDISMODULE_GET_API(GetClientCertificate);
     REDISMODULE_GET_API(GetCommandKeys);
 #endif
 
diff --git a/src/tls.c b/src/tls.c
index 0a9e07895..25580a4f1 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -37,6 +37,7 @@
 #include <openssl/ssl.h>
 #include <openssl/err.h>
 #include <openssl/rand.h>
+#include <openssl/pem.h>
 
 #define REDIS_TLS_PROTO_TLSv1       (1<<0)
 #define REDIS_TLS_PROTO_TLSv1_1     (1<<1)
@@ -868,6 +869,30 @@ int tlsProcessPendingData() {
     return processed;
 }
 
+/* Fetch the peer certificate used for authentication on the specified
+ * connection and return it as a PEM-encoded sds.
+ */
+sds connTLSGetPeerCert(connection *conn_) {
+    tls_connection *conn = (tls_connection *) conn_;
+    if (conn_->type->get_type(conn_) != CONN_TYPE_TLS || !conn->ssl) return NULL;
+
+    X509 *cert = SSL_get_peer_certificate(conn->ssl);
+    if (!cert) return NULL;
+
+    BIO *bio = BIO_new(BIO_s_mem());
+    if (bio == NULL || !PEM_write_bio_X509(bio, cert)) {
+        if (bio != NULL) BIO_free(bio);
+        return NULL;
+    }
+
+    const char *bio_ptr;
+    long long bio_len = BIO_get_mem_data(bio, &bio_ptr);
+    sds cert_pem = sdsnewlen(bio_ptr, bio_len);
+    BIO_free(bio);
+
+    return cert_pem;
+}
+
 #else   /* USE_OPENSSL */
 
 void tlsInit(void) {
@@ -897,4 +922,9 @@ int tlsProcessPendingData() {
     return 0;
 }
 
+sds connTLSGetPeerCert(connection *conn_) {
+    (void) conn_;
+    return NULL;
+}
+
 #endif
diff --git a/tests/modules/misc.c b/tests/modules/misc.c
index a43bb84ac..621b92b2d 100644
--- a/tests/modules/misc.c
+++ b/tests/modules/misc.c
@@ -195,6 +195,23 @@ int test_setlfu(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     return REDISMODULE_OK;
 }
 
+int test_getclientcert(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    (void) argv;
+    (void) argc;
+
+    RedisModuleString *cert = RedisModule_GetClientCertificate(ctx,
+            RedisModule_GetClientId(ctx));
+    if (!cert) {
+        RedisModule_ReplyWithNull(ctx);
+    } else {
+        RedisModule_ReplyWithString(ctx, cert);
+        RedisModule_FreeString(ctx, cert);
+    }
+
+    return REDISMODULE_OK;
+}
+
 int test_clientinfo(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
 {
     (void) argv;
@@ -283,6 +300,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
         return REDISMODULE_ERR;
     if (RedisModule_CreateCommand(ctx,"test.clientinfo", test_clientinfo,"",0,0,0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
+    if (RedisModule_CreateCommand(ctx,"test.getclientcert", test_getclientcert,"",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
     if (RedisModule_CreateCommand(ctx,"test.log_tsctx", test_log_tsctx,"",0,0,0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
 
diff --git a/tests/unit/moduleapi/misc.tcl b/tests/unit/moduleapi/misc.tcl
index 2c9ac4fd0..e825f33ff 100644
--- a/tests/unit/moduleapi/misc.tcl
+++ b/tests/unit/moduleapi/misc.tcl
@@ -87,6 +87,16 @@ start_server {tags {"modules"}} {
         assert { [dict get $info flags] == "${ssl_flag}::tracking::" }
     }
 
+    test {test module getclientcert api} {
+        set cert [r test.getclientcert]
+
+        if {$::tls} {
+            assert {$cert != ""}
+        } else {
+            assert {$cert == ""}
+        }
+    }
+
     test {test detached thread safe cnotext} {
         r test.log_tsctx "info" "Test message"
         verify_log_message 0 "*<misc> Test message*" 0

From adc3183cd2b54238424895e4298548df4526f8c3 Mon Sep 17 00:00:00 2001
From: "Meir Shpilraien (Spielrein)" <meir@redislabs.com>
Date: Sun, 11 Oct 2020 17:21:58 +0300
Subject: [PATCH 0725/1098] Add Module API for version and compatibility checks
 (#7865)

* Introduce a new API's: RM_GetContextFlagsAll, and
RM_GetKeyspaceNotificationFlagsAll that will return the
full flags mask of each feature. The module writer can
check base on this value if the Flags he needs are
supported or not.

* For each flag, introduce a new value on redismodule.h,
this value represents the LAST value and should be there
as a reminder to update it when a new value is added,
also it will be used in the code to calculate the full
flags mask (assuming flags are incrementally increasing).
In addition, stated that the module writer should not use
the LAST flag directly and he should use the GetFlagAll API's.

* Introduce a new API: RM_IsSubEventSupported, that returns for a given
event and subevent, whether or not the subevent supported.

* Introduce a new macro RMAPI_FUNC_SUPPORTED(func) that returns whether
or not a function API is supported by comparing it to NULL.

* Introduce a new API: int RM_GetServerVersion();, that will return the
current Redis version in the format 0x00MMmmpp; e.g. 0x00060008;

* Changed unstable version from 999.999.999 to 255.255.255

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
---
 src/module.c                           | 85 +++++++++++++++++++++++++-
 src/redismodule.h                      | 42 ++++++++++++-
 src/version.h                          |  3 +-
 tests/modules/blockedclient.c          |  8 +++
 tests/modules/hooks.c                  | 11 ++++
 tests/modules/keyspace_events.c        |  7 +++
 tests/modules/misc.c                   | 18 ++++++
 tests/unit/moduleapi/blockedclient.tcl |  6 ++
 tests/unit/moduleapi/misc.tcl          |  5 ++
 9 files changed, 181 insertions(+), 4 deletions(-)

diff --git a/src/module.c b/src/module.c
index 7b4b761fe..ca99d1951 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7267,13 +7267,14 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) {
  *
  *
  * The function returns REDISMODULE_OK if the module was successfully subscribed
- * for the specified event. If the API is called from a wrong context then
- * REDISMODULE_ERR is returned. */
+ * for the specified event. If the API is called from a wrong context or unsupported event
+ * is given then REDISMODULE_ERR is returned. */
 int RM_SubscribeToServerEvent(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback) {
     RedisModuleEventListener *el;
 
     /* Protect in case of calls from contexts without a module reference. */
     if (ctx->module == NULL) return REDISMODULE_ERR;
+    if (event.id >= _REDISMODULE_EVENT_NEXT) return REDISMODULE_ERR;
 
     /* Search an event matching this module and event ID. */
     listIter li;
@@ -7305,6 +7306,42 @@ int RM_SubscribeToServerEvent(RedisModuleCtx *ctx, RedisModuleEvent event, Redis
     return REDISMODULE_OK;
 }
 
+/**
+ * For a given server event and subevent, return zero if the
+ * subevent is not supported and non-zero otherwise.
+ */
+int RM_IsSubEventSupported(RedisModuleEvent event, int64_t subevent) {
+    switch (event.id) {
+    case REDISMODULE_EVENT_REPLICATION_ROLE_CHANGED:
+        return subevent < _REDISMODULE_EVENT_REPLROLECHANGED_NEXT;
+    case REDISMODULE_EVENT_PERSISTENCE:
+        return subevent < _REDISMODULE_SUBEVENT_PERSISTENCE_NEXT;
+    case REDISMODULE_EVENT_FLUSHDB:
+        return subevent < _REDISMODULE_SUBEVENT_FLUSHDB_NEXT;
+    case REDISMODULE_EVENT_LOADING:
+        return subevent < _REDISMODULE_SUBEVENT_LOADING_NEXT;
+    case REDISMODULE_EVENT_CLIENT_CHANGE:
+        return subevent < _REDISMODULE_SUBEVENT_CLIENT_CHANGE_NEXT;
+    case REDISMODULE_EVENT_SHUTDOWN:
+        return subevent < _REDISMODULE_SUBEVENT_SHUTDOWN_NEXT;
+    case REDISMODULE_EVENT_REPLICA_CHANGE:
+        return subevent < _REDISMODULE_EVENT_REPLROLECHANGED_NEXT;
+    case REDISMODULE_EVENT_MASTER_LINK_CHANGE:
+        return subevent < _REDISMODULE_SUBEVENT_MASTER_NEXT;
+    case REDISMODULE_EVENT_CRON_LOOP:
+        return subevent < _REDISMODULE_SUBEVENT_CRON_LOOP_NEXT;
+    case REDISMODULE_EVENT_MODULE_CHANGE:
+        return subevent < _REDISMODULE_SUBEVENT_MODULE_NEXT;
+    case REDISMODULE_EVENT_LOADING_PROGRESS:
+        return subevent < _REDISMODULE_SUBEVENT_LOADING_PROGRESS_NEXT;
+    case REDISMODULE_EVENT_SWAPDB:
+        return subevent < _REDISMODULE_SUBEVENT_SWAPDB_NEXT;
+    default:
+        break;
+    }
+    return 0;
+}
+
 /* This is called by the Redis internals every time we want to fire an
  * event that can be interceppted by some module. The pointer 'data' is useful
  * in order to populate the event-specific structure when needed, in order
@@ -7894,6 +7931,46 @@ int RM_GetLFU(RedisModuleKey *key, long long *lfu_freq) {
     return REDISMODULE_OK;
 }
 
+/**
+ * Returns the full ContextFlags mask, using the return value
+ * the module can check if a certain set of flags are supported
+ * by the redis server version in use.
+ * Example:
+ *        int supportedFlags = RM_GetContextFlagsAll()
+ *        if (supportedFlags & REDISMODULE_CTX_FLAGS_MULTI) {
+ *              // REDISMODULE_CTX_FLAGS_MULTI is supported
+ *        } else{
+ *              // REDISMODULE_CTX_FLAGS_MULTI is not supported
+ *        }
+ */
+int RM_GetContextFlagsAll() {
+    return _REDISMODULE_CTX_FLAGS_NEXT - 1;
+}
+
+/**
+ * Returns the full KeyspaceNotification mask, using the return value
+ * the module can check if a certain set of flags are supported
+ * by the redis server version in use.
+ * Example:
+ *        int supportedFlags = RM_GetKeyspaceNotificationFlagsAll()
+ *        if (supportedFlags & REDISMODULE_NOTIFY_LOADED) {
+ *              // REDISMODULE_NOTIFY_LOADED is supported
+ *        } else{
+ *              // REDISMODULE_NOTIFY_LOADED is not supported
+ *        }
+ */
+int RM_GetKeyspaceNotificationFlagsAll() {
+    return _REDISMODULE_NOTIFY_NEXT - 1;
+}
+
+/**
+ * Return the redis version in format of 0x00MMmmpp.
+ * Example for 6.0.7 the return value will be 0x00060007.
+ */
+int RM_GetServerVersion() {
+    return REDIS_VERSION_NUM;
+}
+
 /* Replace the value assigned to a module type.
  *
  * The key must be open for writing, have an existing value, and have a moduleType
@@ -8227,6 +8304,10 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(DeauthenticateAndCloseClient);
     REGISTER_API(AuthenticateClientWithACLUser);
     REGISTER_API(AuthenticateClientWithUser);
+    REGISTER_API(GetContextFlagsAll);
+    REGISTER_API(GetKeyspaceNotificationFlagsAll);
+    REGISTER_API(IsSubEventSupported);
+    REGISTER_API(GetServerVersion);
     REGISTER_API(GetClientCertificate);
     REGISTER_API(GetCommandKeys);
 }
diff --git a/src/redismodule.h b/src/redismodule.h
index 7b3545585..ed9a73172 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -117,6 +117,11 @@
 /* Redis is currently running inside background child process. */
 #define REDISMODULE_CTX_FLAGS_IS_CHILD (1<<20)
 
+/* Next context flag, must be updated when adding new flags above!
+This flag should not be used directly by the module.
+ * Use RedisModule_GetContextFlagsAll instead. */
+#define _REDISMODULE_CTX_FLAGS_NEXT (1<<21)
+
 /* Keyspace changes notification classes. Every class is associated with a
  * character for configuration purposes.
  * NOTE: These have to be in sync with NOTIFY_* in server.h */
@@ -133,6 +138,12 @@
 #define REDISMODULE_NOTIFY_STREAM (1<<10)     /* t */
 #define REDISMODULE_NOTIFY_KEY_MISS (1<<11)   /* m (Note: This one is excluded from REDISMODULE_NOTIFY_ALL on purpose) */
 #define REDISMODULE_NOTIFY_LOADED (1<<12)     /* module only key space notification, indicate a key loaded from rdb */
+
+/* Next notification flag, must be updated when adding new flags above!
+This flag should not be used directly by the module.
+ * Use RedisModule_GetKeyspaceNotificationFlagsAll instead. */
+#define _REDISMODULE_NOTIFY_NEXT (1<<13)
+
 #define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM)      /* A */
 
 /* A special pointer that we can use between the core and the module to signal
@@ -182,7 +193,9 @@ typedef uint64_t RedisModuleTimerID;
  * are modified from the user's sperspective, to invalidate WATCH. */
 #define REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED (1<<1)
 
-/* Server events definitions. */
+/* Server events definitions.
+ * Those flags should not be used directly by the module, instead
+ * the module should use RedisModuleEvent_* variables */
 #define REDISMODULE_EVENT_REPLICATION_ROLE_CHANGED 0
 #define REDISMODULE_EVENT_PERSISTENCE 1
 #define REDISMODULE_EVENT_FLUSHDB 2
@@ -196,6 +209,9 @@ typedef uint64_t RedisModuleTimerID;
 #define REDISMODULE_EVENT_LOADING_PROGRESS 10
 #define REDISMODULE_EVENT_SWAPDB 11
 
+/* Next event flag, should be updated if a new event added. */
+#define _REDISMODULE_EVENT_NEXT 12
+
 typedef struct RedisModuleEvent {
     uint64_t id;        /* REDISMODULE_EVENT_... defines. */
     uint64_t dataver;   /* Version of the structure we pass as 'data'. */
@@ -260,33 +276,47 @@ static const RedisModuleEvent
 #define REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START 2
 #define REDISMODULE_SUBEVENT_PERSISTENCE_ENDED 3
 #define REDISMODULE_SUBEVENT_PERSISTENCE_FAILED 4
+#define _REDISMODULE_SUBEVENT_PERSISTENCE_NEXT 5
 
 #define REDISMODULE_SUBEVENT_LOADING_RDB_START 0
 #define REDISMODULE_SUBEVENT_LOADING_AOF_START 1
 #define REDISMODULE_SUBEVENT_LOADING_REPL_START 2
 #define REDISMODULE_SUBEVENT_LOADING_ENDED 3
 #define REDISMODULE_SUBEVENT_LOADING_FAILED 4
+#define _REDISMODULE_SUBEVENT_LOADING_NEXT 5
 
 #define REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED 0
 #define REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED 1
+#define _REDISMODULE_SUBEVENT_CLIENT_CHANGE_NEXT 2
 
 #define REDISMODULE_SUBEVENT_MASTER_LINK_UP 0
 #define REDISMODULE_SUBEVENT_MASTER_LINK_DOWN 1
+#define _REDISMODULE_SUBEVENT_MASTER_NEXT 2
 
 #define REDISMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE 0
 #define REDISMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE 1
+#define _REDISMODULE_SUBEVENT_REPLICA_CHANGE_NEXT 2
 
 #define REDISMODULE_EVENT_REPLROLECHANGED_NOW_MASTER 0
 #define REDISMODULE_EVENT_REPLROLECHANGED_NOW_REPLICA 1
+#define _REDISMODULE_EVENT_REPLROLECHANGED_NEXT 2
 
 #define REDISMODULE_SUBEVENT_FLUSHDB_START 0
 #define REDISMODULE_SUBEVENT_FLUSHDB_END 1
+#define _REDISMODULE_SUBEVENT_FLUSHDB_NEXT 2
 
 #define REDISMODULE_SUBEVENT_MODULE_LOADED 0
 #define REDISMODULE_SUBEVENT_MODULE_UNLOADED 1
+#define _REDISMODULE_SUBEVENT_MODULE_NEXT 2
+
 
 #define REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB 0
 #define REDISMODULE_SUBEVENT_LOADING_PROGRESS_AOF 1
+#define _REDISMODULE_SUBEVENT_LOADING_PROGRESS_NEXT 2
+
+#define _REDISMODULE_SUBEVENT_SHUTDOWN_NEXT 0
+#define _REDISMODULE_SUBEVENT_CRON_LOOP_NEXT 0
+#define _REDISMODULE_SUBEVENT_SWAPDB_NEXT 0
 
 /* RedisModuleClientInfo flags. */
 #define REDISMODULE_CLIENTINFO_FLAG_SSL (1<<0)
@@ -672,6 +702,10 @@ REDISMODULE_API void (*RedisModule_ScanCursorRestart)(RedisModuleScanCursor *cur
 REDISMODULE_API void (*RedisModule_ScanCursorDestroy)(RedisModuleScanCursor *cursor) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_Scan)(RedisModuleCtx *ctx, RedisModuleScanCursor *cursor, RedisModuleScanCB fn, void *privdata) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_ScanKey)(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleScanKeyCB fn, void *privdata) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_GetContextFlagsAll)() REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_GetKeyspaceNotificationFlagsAll)() REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_IsSubEventSupported)(RedisModuleEvent event, uint64_t subevent) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_GetServerVersion)() REDISMODULE_ATTR;
 
 /* Experimental APIs */
 #ifdef REDISMODULE_EXPERIMENTAL_API
@@ -918,6 +952,10 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(ScanCursorDestroy);
     REDISMODULE_GET_API(Scan);
     REDISMODULE_GET_API(ScanKey);
+    REDISMODULE_GET_API(GetContextFlagsAll);
+    REDISMODULE_GET_API(GetKeyspaceNotificationFlagsAll);
+    REDISMODULE_GET_API(IsSubEventSupported);
+    REDISMODULE_GET_API(GetServerVersion);
 
 #ifdef REDISMODULE_EXPERIMENTAL_API
     REDISMODULE_GET_API(GetThreadSafeContext);
@@ -988,5 +1026,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
  * including this file. */
 #define RedisModuleString robj
 
+#define RMAPI_FUNC_SUPPORTED(func) (func != NULL)
+
 #endif /* REDISMODULE_CORE */
 #endif /* REDISMODULE_H */
diff --git a/src/version.h b/src/version.h
index eb65e9bbd..89aef53fc 100644
--- a/src/version.h
+++ b/src/version.h
@@ -1 +1,2 @@
-#define REDIS_VERSION "999.999.999"
+#define REDIS_VERSION "255.255.255"
+#define REDIS_VERSION_NUM 0x00ffffff
diff --git a/tests/modules/blockedclient.c b/tests/modules/blockedclient.c
index ca98281a4..558e06502 100644
--- a/tests/modules/blockedclient.c
+++ b/tests/modules/blockedclient.c
@@ -57,6 +57,14 @@ int acquire_gil(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     UNUSED(argv);
     UNUSED(argc);
 
+    int flags = RedisModule_GetContextFlags(ctx);
+    int allFlags = RedisModule_GetContextFlagsAll();
+    if ((allFlags & REDISMODULE_CTX_FLAGS_MULTI) &&
+        (flags & REDISMODULE_CTX_FLAGS_MULTI)) {
+        RedisModule_ReplyWithSimpleString(ctx, "Blocked client is not supported inside multi");
+        return REDISMODULE_OK;
+    }
+
     /* This command handler tries to acquire the GIL twice
      * once in the worker thread using "RedisModule_ThreadSafeContextLock"
      * second in the sub-worker thread
diff --git a/tests/modules/hooks.c b/tests/modules/hooks.c
index 54f84aa23..6fb3d2325 100644
--- a/tests/modules/hooks.c
+++ b/tests/modules/hooks.c
@@ -266,12 +266,22 @@ void swapDbCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void
 /* This function must be present on each Redis module. It is used in order to
  * register the commands into the Redis server. */
 int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+#define VerifySubEventSupported(e, s) \
+    if (!RedisModule_IsSubEventSupported(e, s)) { \
+        return REDISMODULE_ERR; \
+    }
+
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
 
     if (RedisModule_Init(ctx,"testhook",1,REDISMODULE_APIVER_1)
         == REDISMODULE_ERR) return REDISMODULE_ERR;
 
+    /* Example on how to check if a server sub event is supported */
+    if (!RedisModule_IsSubEventSupported(RedisModuleEvent_ReplicationRoleChanged, REDISMODULE_EVENT_REPLROLECHANGED_NOW_MASTER)) {
+        return REDISMODULE_ERR;
+    }
+
     /* replication related hooks */
     RedisModule_SubscribeToServerEvent(ctx,
         RedisModuleEvent_ReplicationRoleChanged, roleChangeCallback);
@@ -297,6 +307,7 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
         RedisModuleEvent_Shutdown, shutdownCallback);
     RedisModule_SubscribeToServerEvent(ctx,
         RedisModuleEvent_CronLoop, cronLoopCallback);
+
     RedisModule_SubscribeToServerEvent(ctx,
         RedisModuleEvent_ModuleChange, moduleChangeCallback);
     RedisModule_SubscribeToServerEvent(ctx,
diff --git a/tests/modules/keyspace_events.c b/tests/modules/keyspace_events.c
index db3977be1..fac7edd1f 100644
--- a/tests/modules/keyspace_events.c
+++ b/tests/modules/keyspace_events.c
@@ -87,6 +87,13 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
 
     loaded_event_log = RedisModule_CreateDict(ctx);
 
+    int keySpaceAll = RedisModule_GetKeyspaceNotificationFlagsAll();
+
+    if (!(keySpaceAll & REDISMODULE_NOTIFY_LOADED)) {
+        // REDISMODULE_NOTIFY_LOADED event are not supported we can not start
+        return REDISMODULE_ERR;
+    }
+
     if(RedisModule_SubscribeToKeyspaceEvents(ctx, REDISMODULE_NOTIFY_LOADED, KeySpace_Notification) != REDISMODULE_OK){
         return REDISMODULE_ERR;
     }
diff --git a/tests/modules/misc.c b/tests/modules/misc.c
index 621b92b2d..1c7c47a90 100644
--- a/tests/modules/misc.c
+++ b/tests/modules/misc.c
@@ -195,6 +195,22 @@ int test_setlfu(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     return REDISMODULE_OK;
 }
 
+int test_redisversion(RedisModuleCtx *ctx, RedisModuleString **argv, int argc){
+    (void) argv;
+    (void) argc;
+
+    int version = RedisModule_GetServerVersion();
+    int patch = version & 0x000000ff;
+    int minor = (version & 0x0000ff00) >> 8;
+    int major = (version & 0x00ff0000) >> 16;
+
+    RedisModuleString* vStr = RedisModule_CreateStringPrintf(ctx, "%d.%d.%d", major, minor, patch);
+    RedisModule_ReplyWithString(ctx, vStr);
+    RedisModule_FreeString(ctx, vStr);
+  
+    return REDISMODULE_OK;
+}
+
 int test_getclientcert(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
 {
     (void) argv;
@@ -300,6 +316,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
         return REDISMODULE_ERR;
     if (RedisModule_CreateCommand(ctx,"test.clientinfo", test_clientinfo,"",0,0,0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
+    if (RedisModule_CreateCommand(ctx,"test.redisversion", test_redisversion,"",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
     if (RedisModule_CreateCommand(ctx,"test.getclientcert", test_getclientcert,"",0,0,0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
     if (RedisModule_CreateCommand(ctx,"test.log_tsctx", test_log_tsctx,"",0,0,0) == REDISMODULE_ERR)
diff --git a/tests/unit/moduleapi/blockedclient.tcl b/tests/unit/moduleapi/blockedclient.tcl
index d093a0297..5a541d138 100644
--- a/tests/unit/moduleapi/blockedclient.tcl
+++ b/tests/unit/moduleapi/blockedclient.tcl
@@ -8,4 +8,10 @@ start_server {tags {"modules"}} {
     test {Locked GIL acquisition} {
         assert_match "OK" [r acquire_gil]
     }
+
+    test {Locked GIL acquisition during multi} {
+    	r multi
+    	r acquire_gil
+    	assert_equal {{Blocked client is not supported inside multi}} [r exec]
+    }
 }
diff --git a/tests/unit/moduleapi/misc.tcl b/tests/unit/moduleapi/misc.tcl
index e825f33ff..d60ccca8f 100644
--- a/tests/unit/moduleapi/misc.tcl
+++ b/tests/unit/moduleapi/misc.tcl
@@ -16,6 +16,11 @@ start_server {tags {"modules"}} {
         assert { [string match "*cmdstat_module*" $info] }
     }
 
+    test {test redis version} {
+        set version [s redis_version]
+        assert_equal $version [r test.redisversion]
+    }
+
     test {test long double conversions} {
         set ld [r test.ld_conversion]
         assert {[string match $ld "0.00000000000000001"]}

From 056a43e1a615080cf9ba405be0538a6a979c821e Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 11 Oct 2020 18:10:55 +0300
Subject: [PATCH 0726/1098] Modules: fix RM_GetCommandKeys API. (#7901)

This cleans up and simplifies the API by passing the command name as the
first argument. Previously the command name was specified explicitly,
but was still included in the argv.
---
 src/module.c            | 4 ++--
 src/redismodule.h       | 2 +-
 tests/modules/getkeys.c | 5 +----
 3 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/module.c b/src/module.c
index ca99d1951..4fde987a2 100644
--- a/src/module.c
+++ b/src/module.c
@@ -8020,13 +8020,13 @@ int RM_ModuleTypeReplaceValue(RedisModuleKey *key, moduleType *mt, void *new_val
  * get automatically freed even when auto-memory is used. The caller
  * must explicitly call RM_Free() to free it.
  */
-int *RM_GetCommandKeys(RedisModuleCtx *ctx, const char *cmdname, RedisModuleString **argv, int argc, int *num_keys) {
+int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys) {
     UNUSED(ctx);
     struct redisCommand *cmd;
     int *res = NULL;
 
     /* Find command */
-    if ((cmd = lookupCommandByCString(cmdname)) == NULL) {
+    if ((cmd = lookupCommand(argv[0]->ptr)) == NULL) {
         errno = ENOENT;
         return NULL;
     }
diff --git a/src/redismodule.h b/src/redismodule.h
index ed9a73172..cb52bd79e 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -762,7 +762,7 @@ REDISMODULE_API int (*RedisModule_AuthenticateClientWithACLUser)(RedisModuleCtx
 REDISMODULE_API int (*RedisModule_AuthenticateClientWithUser)(RedisModuleCtx *ctx, RedisModuleUser *user, RedisModuleUserChangedFunc callback, void *privdata, uint64_t *client_id) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id) REDISMODULE_ATTR;
 REDISMODULE_API RedisModuleString * (*RedisModule_GetClientCertificate)(RedisModuleCtx *ctx, uint64_t id) REDISMODULE_ATTR;
-REDISMODULE_API int *(*RedisModule_GetCommandKeys)(RedisModuleCtx *ctx, const char *cmdname, RedisModuleString **argv, int argc, int *num_keys) REDISMODULE_ATTR;
+REDISMODULE_API int *(*RedisModule_GetCommandKeys)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys) REDISMODULE_ATTR;
 #endif
 
 #define RedisModule_IsAOFClient(id) ((id) == CLIENT_ID_AOF)
diff --git a/tests/modules/getkeys.c b/tests/modules/getkeys.c
index 9cde606ab..706054174 100644
--- a/tests/modules/getkeys.c
+++ b/tests/modules/getkeys.c
@@ -69,11 +69,8 @@ int getkeys_introspect(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
         return REDISMODULE_OK;
     }
 
-    size_t cmd_len;
-    const char *cmd = RedisModule_StringPtrLen(argv[1], &cmd_len);
-
     int num_keys;
-    int *keyidx = RedisModule_GetCommandKeys(ctx, cmd, &argv[1], argc - 1, &num_keys);
+    int *keyidx = RedisModule_GetCommandKeys(ctx, &argv[1], argc - 1, &num_keys);
 
     if (!keyidx) {
         if (!errno)

From 8b497881f2eda5b028cc2ba0aa17ef204f6a083b Mon Sep 17 00:00:00 2001
From: Andreas Lind <andreaslindpetersen@gmail.com>
Date: Sun, 11 Oct 2020 17:14:02 +0200
Subject: [PATCH 0727/1098] Support redis-cli -u rediss://... (#7900)

---
 src/redis-cli.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index d79fb35f8..4c9e46926 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -388,12 +388,20 @@ static sds percentDecode(const char *pe, size_t len) {
 static void parseRedisUri(const char *uri) {
 
     const char *scheme = "redis://";
+    const char *tlsscheme = "rediss://";
     const char *curr = uri;
     const char *end = uri + strlen(uri);
     const char *userinfo, *username, *port, *host, *path;
 
     /* URI must start with a valid scheme. */
-    if (strncasecmp(scheme, curr, strlen(scheme))) {
+    if (!strncasecmp(tlsscheme, curr, strlen(tlsscheme))) {
+#ifdef USE_OPENSSL
+        config.tls = 1;
+#else
+        fprintf(stderr,"rediss:// is only supported when redis-cli is compiled with OpenSSL\n");
+        exit(1);
+#endif
+    } else if (strncasecmp(scheme, curr, strlen(scheme))) {
         fprintf(stderr,"Invalid URI scheme\n");
         exit(1);
     }

From addf47dcacec596eb7f6e1500846331d3e13fff4 Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Mon, 12 Oct 2020 16:13:38 +0200
Subject: [PATCH 0728/1098] Minor improvements to module blocked on keys
 (#7903)

- Clarify some documentation comments
- Make sure blocked-on-keys client privdata is accessible
  from withing the timeout callback
- Handle blocked clients in beforeSleep - In case a key
  becomes "ready" outside of processCommand

See #7879 #7880
---
 src/blocked.c |  2 +-
 src/module.c  | 17 +++++++++--------
 src/server.c  |  5 +++++
 3 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/src/blocked.c b/src/blocked.c
index 8641fcf8c..d85723458 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -507,7 +507,7 @@ void handleClientsBlockedOnKeys(void) {
             server.fixed_time_expire++;
             updateCachedTime(0);
 
-            /* Serve clients blocked on list key. */
+            /* Serve clients blocked on the key. */
             robj *o = lookupKeyWrite(rl->db,rl->key);
 
             if (o != NULL) {
diff --git a/src/module.c b/src/module.c
index 4fde987a2..a1bbbf47b 100644
--- a/src/module.c
+++ b/src/module.c
@@ -4638,8 +4638,9 @@ RedisModuleBlockedClient *RM_BlockClientOnKeys(RedisModuleCtx *ctx, RedisModuleC
 
 /* This function is used in order to potentially unblock a client blocked
  * on keys with RedisModule_BlockClientOnKeys(). When this function is called,
- * all the clients blocked for this key will get their reply callback called,
- * and if the callback returns REDISMODULE_OK the client will be unblocked. */
+ * all the clients blocked for this key will get their reply_callback called.
+ *
+ * Note: The function has no effect if the signaled key doesn't exist. */
 void RM_SignalKeyAsReady(RedisModuleCtx *ctx, RedisModuleString *key) {
     signalKeyAsReady(ctx->client->db, key, OBJ_MODULE);
 }
@@ -4683,14 +4684,13 @@ int moduleClientIsBlockedOnKeys(client *c) {
  *
  * Note 1: this function can be called from threads spawned by the module.
  *
- * Note 2: when we unblock a client that is blocked for keys using
- * the API RedisModule_BlockClientOnKeys(), the privdata argument here is
- * not used, and the reply callback is called with the privdata pointer that
- * was passed when blocking the client.
- *
+ * Note 2: when we unblock a client that is blocked for keys using the API
+ * RedisModule_BlockClientOnKeys(), the privdata argument here is not used.
  * Unblocking a client that was blocked for keys using this API will still
  * require the client to get some reply, so the function will use the
- * "timeout" handler in order to do so. */
+ * "timeout" handler in order to do so (The privdata provided in
+ * RedisModule_BlockClientOnKeys() is accessible from the timeout
+ * callback via RM_GetBlockedClientPrivateData). */
 int RM_UnblockClient(RedisModuleBlockedClient *bc, void *privdata) {
     if (bc->blocked_on_keys) {
         /* In theory the user should always pass the timeout handler as an
@@ -4834,6 +4834,7 @@ void moduleBlockedClientTimedOut(client *c) {
     ctx.module = bc->module;
     ctx.client = bc->client;
     ctx.blocked_client = bc;
+    ctx.blocked_privdata = bc->privdata;
     bc->timeout_callback(&ctx,(void**)c->argv,c->argc);
     moduleFreeContext(&ctx);
     /* For timeout events, we do not want to call the disconnect callback,
diff --git a/src/server.c b/src/server.c
index 6ae7037af..6abb368a8 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2286,6 +2286,11 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
      * releasing the GIL. Redis main thread will not touch anything at this
      * time. */
     if (moduleCount()) moduleReleaseGIL();
+
+    /* Try to process blocked clients every once in while. Example: A module
+     * calls RM_SignalKeyAsReady from within a timer callback (So we don't
+     * visit processCommand() at all). */
+    handleClientsBlockedOnKeys();
 }
 
 /* This function is called immediately after the event loop multiplexing

From aaacb8c955779a76faaa15b7097eef4787a4339f Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuancode@163.com>
Date: Tue, 13 Oct 2020 14:34:07 +0800
Subject: [PATCH 0729/1098] Remove temporary aof and rdb files in a background
 thread (#7905)

If we fail or stop to rewrite aof, we need to remove temporary aof.
We also remove temporary rdb when replicas abort to receive rdb.
But currently we delete them in main thread, to avoid blocking,
we should use bg_unlink to remove them in a background thread.

Btw, we have already used this way to removed child process temporary rdb.
---
 src/aof.c         | 4 ++--
 src/replication.c | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index b1c7595b5..9cd487cc4 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1703,10 +1703,10 @@ void aofRemoveTempFile(pid_t childpid) {
     char tmpfile[256];
 
     snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) childpid);
-    unlink(tmpfile);
+    bg_unlink(tmpfile);
 
     snprintf(tmpfile,256,"temp-rewriteaof-%d.aof", (int) childpid);
-    unlink(tmpfile);
+    bg_unlink(tmpfile);
 }
 
 /* Update the server.aof_current_size field explicitly using stat(2)
diff --git a/src/replication.c b/src/replication.c
index 9ed4dd117..da70c4082 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -2444,7 +2444,7 @@ void replicationAbortSyncTransfer(void) {
     undoConnectWithMaster();
     if (server.repl_transfer_fd!=-1) {
         close(server.repl_transfer_fd);
-        unlink(server.repl_transfer_tmpfile);
+        bg_unlink(server.repl_transfer_tmpfile);
         zfree(server.repl_transfer_tmpfile);
         server.repl_transfer_tmpfile = NULL;
         server.repl_transfer_fd = -1;

From 24092eea062519c99f36ee2f1348345820b9a5dd Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Tue, 13 Oct 2020 15:05:05 +0800
Subject: [PATCH 0730/1098] Delete dbExists() which is redundant. (#7906)

---
 src/db.c     | 6 ------
 src/server.h | 1 -
 2 files changed, 7 deletions(-)

diff --git a/src/db.c b/src/db.c
index d30956c9a..a26280719 100644
--- a/src/db.c
+++ b/src/db.c
@@ -254,12 +254,6 @@ void setKey(client *c, redisDb *db, robj *key, robj *val) {
     genericSetKey(c,db,key,val,0,1);
 }
 
-/* Return true if the specified key exists in the specified database.
- * LRU/LFU info is not updated in any way. */
-int dbExists(redisDb *db, robj *key) {
-    return dictFind(db->dict,key->ptr) != NULL;
-}
-
 /* Return a random key, in form of a Redis object.
  * If there are no keys, NULL is returned.
  *
diff --git a/src/server.h b/src/server.h
index a5881e9a2..63581b5ce 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2161,7 +2161,6 @@ int dbAddRDBLoad(redisDb *db, sds key, robj *val);
 void dbOverwrite(redisDb *db, robj *key, robj *val);
 void genericSetKey(client *c, redisDb *db, robj *key, robj *val, int keepttl, int signal);
 void setKey(client *c, redisDb *db, robj *key, robj *val);
-int dbExists(redisDb *db, robj *key);
 robj *dbRandomKey(redisDb *db);
 int dbSyncDelete(redisDb *db, robj *key);
 int dbDelete(redisDb *db, robj *key);

From f559a572061ac1d33bce4d778e5a2f3c164469bd Mon Sep 17 00:00:00 2001
From: Hanif Ariffin <hanif.ariffin.4326@gmail.com>
Date: Tue, 13 Oct 2020 17:42:52 +0800
Subject: [PATCH 0731/1098] Fix printf format specifier for unsigned in
 ziplistRepr (#7907)

Only used by DEBUG and testing code.

Signed-off-by: Hanif Bin Ariffin <hanif.ariffin.4326@gmail.com>
---
 src/ziplist.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/ziplist.c b/src/ziplist.c
index ac5cdafb9..dd7452006 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -1230,7 +1230,7 @@ void ziplistRepr(unsigned char *zl) {
     zlentry entry;
 
     printf(
-        "{total bytes %d} "
+        "{total bytes %u} "
         "{num entries %u}\n"
         "{tail offset %u}\n",
         intrev32ifbe(ZIPLIST_BYTES(zl)),
@@ -1243,7 +1243,7 @@ void ziplistRepr(unsigned char *zl) {
             "{\n"
                 "\taddr 0x%08lx,\n"
                 "\tindex %2d,\n"
-                "\toffset %5ld,\n"
+                "\toffset %5lu,\n"
                 "\thdr+entry len: %5u,\n"
                 "\thdr len%2u,\n"
                 "\tprevrawlen: %5u,\n"

From 37fd3d40ae5168eba5bfb5e5e966e1ac9aab990c Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Tue, 13 Oct 2020 18:50:57 +0200
Subject: [PATCH 0732/1098] performEvictions: mem_freed may be negative (#7908)

If 'delta' is negative 'mem_freed' may underflow and cause
the while loop to exit prematurely (And not evicting enough
memory)

mem_freed can be negative when:
1. We use lazy free (consuming memory by appending to a list)
2. Thread doing an allocation between the two calls to zmalloc_used_memory.
---
 src/evict.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/evict.c b/src/evict.c
index 0e8f818ff..d14603aef 100644
--- a/src/evict.c
+++ b/src/evict.c
@@ -499,7 +499,8 @@ int performEvictions(void) {
     if (!isSafeToPerformEvictions()) return EVICT_OK;
 
     int keys_freed = 0;
-    size_t mem_reported, mem_tofree, mem_freed;
+    size_t mem_reported, mem_tofree;
+    long long mem_freed; /* May be negative */
     mstime_t latency, eviction_latency;
     long long delta;
     int slaves = listLength(server.slaves);
@@ -520,7 +521,7 @@ int performEvictions(void) {
     monotime evictionTimer;
     elapsedStart(&evictionTimer);
 
-    while (mem_freed < mem_tofree) {
+    while (mem_freed < (long long)mem_tofree) {
         int j, k, i;
         static unsigned int next_db = 0;
         sds bestkey = NULL;

From 19418b6b28b70818e67c50bbdfad8568da7c3458 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 14 Oct 2020 09:38:47 +0300
Subject: [PATCH 0733/1098] Allow requirepass config to clear the password
 (#7899)

This is a compatibility issue with redis 5.0 that was introduced by ACL.
Before this commit, setting requirepass to an empty string will result
in a server that needs an empty AUTH, unlike redis 5.0 which would
accept connections without an AUTH.
---
 src/config.c | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/src/config.c b/src/config.c
index 350469e52..2a9f49d80 100644
--- a/src/config.c
+++ b/src/config.c
@@ -495,11 +495,16 @@ void loadServerConfigFromString(char *config) {
              * additionally is to remember the cleartext password in this
              * case, for backward compatibility with Redis <= 5. */
             ACLSetUser(DefaultUser,"resetpass",-1);
-            sds aclop = sdscatprintf(sdsempty(),">%s",argv[1]);
-            ACLSetUser(DefaultUser,aclop,sdslen(aclop));
-            sdsfree(aclop);
             sdsfree(server.requirepass);
-            server.requirepass = sdsnew(argv[1]);
+            server.requirepass = NULL;
+            if (sdslen(argv[1])) {
+                sds aclop = sdscatprintf(sdsempty(),">%s",argv[1]);
+                ACLSetUser(DefaultUser,aclop,sdslen(aclop));
+                sdsfree(aclop);
+                server.requirepass = sdsnew(argv[1]);
+            } else {
+                ACLSetUser(DefaultUser,"nopass",-1);
+            }
         } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){
             /* DEAD OPTION */
         } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) {
@@ -717,11 +722,16 @@ void configSetCommand(client *c) {
          * additionally is to remember the cleartext password in this
          * case, for backward compatibility with Redis <= 5. */
         ACLSetUser(DefaultUser,"resetpass",-1);
-        sds aclop = sdscatprintf(sdsempty(),">%s",(char*)o->ptr);
-        ACLSetUser(DefaultUser,aclop,sdslen(aclop));
-        sdsfree(aclop);
         sdsfree(server.requirepass);
-        server.requirepass = sdsnew(o->ptr);
+        server.requirepass = NULL;
+        if (sdslen(o->ptr)) {
+            sds aclop = sdscatprintf(sdsempty(),">%s",(char*)o->ptr);
+            ACLSetUser(DefaultUser,aclop,sdslen(aclop));
+            sdsfree(aclop);
+            server.requirepass = sdsnew(o->ptr);
+        } else {
+            ACLSetUser(DefaultUser,"nopass",-1);
+        }
     } config_set_special_field("save") {
         int vlen, j;
         sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen);

From 786d6d55cf5ebfdce4c3ce28b0a7c4460df111f0 Mon Sep 17 00:00:00 2001
From: Tommy Joe Lund <tommyjl@users.noreply.github.com>
Date: Sun, 18 Oct 2020 07:11:18 +0200
Subject: [PATCH 0734/1098] Fix typo in server.h (#7921)

---
 src/server.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/server.h b/src/server.h
index 63581b5ce..33ce2b89c 100644
--- a/src/server.h
+++ b/src/server.h
@@ -214,7 +214,7 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
 #define AOF_WAIT_REWRITE 2    /* AOF waits rewrite to start appending */
 
 /* Client flags */
-#define CLIENT_SLAVE (1<<0)   /* This client is a repliaca */
+#define CLIENT_SLAVE (1<<0)   /* This client is a replica */
 #define CLIENT_MASTER (1<<1)  /* This client is a master */
 #define CLIENT_MONITOR (1<<2) /* This client is a slave monitor, see MONITOR */
 #define CLIENT_MULTI (1<<3)   /* This client is in a MULTI context */
@@ -1967,7 +1967,7 @@ void addACLLogEntry(client *c, int reason, int keypos, sds username);
 #define ZADD_INCR (1<<0)    /* Increment the score instead of setting it. */
 #define ZADD_NX (1<<1)      /* Don't touch elements not already existing. */
 #define ZADD_XX (1<<2)      /* Only touch elements already existing. */
-#define ZADD_GT (1<<7)      /* Only update existing when new scores are higher. */ 
+#define ZADD_GT (1<<7)      /* Only update existing when new scores are higher. */
 #define ZADD_LT (1<<8)      /* Only update existing when new scores are lower. */
 
 /* Output flags. */

From f6010e106b47c7a72026a10edc6b9b7fb79c3f04 Mon Sep 17 00:00:00 2001
From: Pierre Jambet <pierre.jambet@gmail.com>
Date: Sun, 18 Oct 2020 01:14:45 -0400
Subject: [PATCH 0735/1098] t_set.c comment update for
 srandmemberWithCountCommand (#7922)

Reference the correct "case", case 4, in the comment explaining the need
for case 3, when the number of request items is too close to the
cardinality of the set. Case 4 is indeed the "natural approach"
referenced earlier in that sentence.
---
 src/t_set.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_set.c b/src/t_set.c
index b4f308963..c1e7fddef 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -694,7 +694,7 @@ void srandmemberWithCountCommand(client *c) {
      *
      * This is done because if the number of requested elements is just
      * a bit less than the number of elements in the set, the natural approach
-     * used into CASE 3 is highly inefficient. */
+     * used into CASE 4 is highly inefficient. */
     if (count*SRANDMEMBER_SUB_STRATEGY_MUL > size) {
         setTypeIterator *si;
 

From f328194d12e06a748aac7a35b1dda179bfc6cb0f Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Sun, 18 Oct 2020 03:15:43 -0400
Subject: [PATCH 0736/1098] support NOMKSTREAM option in xadd command (#7910)

introduces a NOMKSTREAM option for xadd command, this would be useful for some
use cases when we do not want to create new stream by default:

XADD key [MAXLEN [~|=] <count>] [NOMKSTREAM] <ID or *> [field value] [field value]
---
 src/t_stream.c             | 13 ++++++++++---
 tests/unit/type/stream.tcl | 12 ++++++++++++
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 6eb6363fc..d6f6a3011 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1130,10 +1130,14 @@ size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start
 
 /* Look the stream at 'key' and return the corresponding stream object.
  * The function creates a key setting it to an empty stream if needed. */
-robj *streamTypeLookupWriteOrCreate(client *c, robj *key) {
+robj *streamTypeLookupWriteOrCreate(client *c, robj *key, int no_create) {
     robj *o = lookupKeyWrite(c->db,key);
     if (checkType(c,o,OBJ_STREAM)) return NULL;
     if (o == NULL) {
+        if (no_create) {
+            addReplyNull(c);
+            return NULL;
+        }
         o = createStreamObject();
         dbAdd(c->db,key,o);
     }
@@ -1214,7 +1218,7 @@ void streamRewriteApproxMaxlen(client *c, stream *s, int maxlen_arg_idx) {
     decrRefCount(maxlen_obj);
 }
 
-/* XADD key [MAXLEN [~|=] <count>] <ID or *> [field value] [field value] ... */
+/* XADD key [MAXLEN [~|=] <count>] [NOMKSTREAM] <ID or *> [field value] [field value] ... */
 void xaddCommand(client *c) {
     streamID id;
     int id_given = 0; /* Was an ID different than "*" specified? */
@@ -1222,6 +1226,7 @@ void xaddCommand(client *c) {
     int approx_maxlen = 0;  /* If 1 only delete whole radix tree nodes, so
                                the maximum length is not applied verbatim. */
     int maxlen_arg_idx = 0; /* Index of the count in MAXLEN, for rewriting. */
+    int no_mkstream = 0; /* if set to 1 do not create new stream */
 
     /* Parse options. */
     int i = 2; /* This is the first argument position where we could
@@ -1252,6 +1257,8 @@ void xaddCommand(client *c) {
             }
             i++;
             maxlen_arg_idx = i;
+        } else if (!strcasecmp(opt,"nomkstream")) {
+            no_mkstream = 1;
         } else {
             /* If we are here is a syntax error or a valid ID. */
             if (streamParseStrictIDOrReply(c,c->argv[i],&id,0) != C_OK) return;
@@ -1278,7 +1285,7 @@ void xaddCommand(client *c) {
     /* Lookup the stream at key. */
     robj *o;
     stream *s;
-    if ((o = streamTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return;
+    if ((o = streamTypeLookupWriteOrCreate(c,c->argv[1],no_mkstream)) == NULL) return;
     s = o->ptr;
 
     /* Return ASAP if the stream has reached the last possible ID */
diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl
index 0ff570cab..b02d6a0fb 100644
--- a/tests/unit/type/stream.tcl
+++ b/tests/unit/type/stream.tcl
@@ -102,6 +102,18 @@ start_server {
         }
     }
 
+    test {XADD with NOMKSTREAM option} {
+        r DEL mystream
+        assert_equal "" [r XADD mystream NOMKSTREAM * item 1 value a]
+        assert_equal 0 [r EXISTS mystream]
+        r XADD mystream * item 1 value a
+        r XADD mystream NOMKSTREAM * item 2 value b
+        assert_equal 2 [r XLEN mystream]
+        set items [r XRANGE mystream - +]
+        assert_equal [lindex $items 0 1] {item 1 value a}
+        assert_equal [lindex $items 1 1] {item 2 value b}
+    }
+
     test {XADD mass insertion and XLEN} {
         r DEL mystream
         r multi

From ef92f507dd0c402a916c16435e7f3f92598b7242 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 18 Oct 2020 14:50:29 +0300
Subject: [PATCH 0737/1098] Fix tests failure on busybox systems. (#7916)

---
 tests/support/server.tcl | 2 +-
 tests/support/util.tcl   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index 30d0d4045..ae93ad007 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -94,7 +94,7 @@ proc kill_server config {
 
 proc is_alive config {
     set pid [dict get $config pid]
-    if {[catch {exec ps -p $pid} err]} {
+    if {[catch {exec kill -0 $pid} err]} {
         return 0
     } else {
         return 1
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index bc1394ce4..ac4fca501 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -508,7 +508,7 @@ proc populate {num prefix size} {
 
 proc get_child_pid {idx} {
     set pid [srv $idx pid]
-    if {[string match {*Darwin*} [exec uname -a]]} {
+    if {[file exists "/usr/bin/pgrep"]} {
         set fd [open "|pgrep -P $pid" "r"]
         set child_pid [string trim [lindex [split [read $fd] \n] 0]]
     } else {

From 457b7073b5d512c10aaeeb0dbf1763f0cb10462c Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 18 Oct 2020 16:56:43 +0300
Subject: [PATCH 0738/1098] INFO report peak memory before eviction (#7894)

In some cases one command added a very big bulk of memory, and this
would be "resolved" by the eviction before the next command.

Seeing an unexplained mass eviction we would wish to
know the highest momentary usage too.

Tracking it in call() and beforeSleep() adds some hooks in AOF and RDB
loading.

The fix in clientsCronTrackExpansiveClients is related to #7874
---
 src/server.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index 6abb368a8..43d7c337c 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1599,7 +1599,7 @@ size_t ClientsPeakMemInput[CLIENTS_PEAK_MEM_USAGE_SLOTS];
 size_t ClientsPeakMemOutput[CLIENTS_PEAK_MEM_USAGE_SLOTS];
 
 int clientsCronTrackExpansiveClients(client *c) {
-    size_t in_usage = sdsAllocSize(c->querybuf);
+    size_t in_usage = sdsZmallocSize(c->querybuf) + c->argv_len_sum;
     size_t out_usage = getClientOutputBufferMemoryUsage(c);
     int i = server.unixtime % CLIENTS_PEAK_MEM_USAGE_SLOTS;
     int zeroidx = (i+1) % CLIENTS_PEAK_MEM_USAGE_SLOTS;
@@ -2200,6 +2200,10 @@ extern int ProcessingEventsWhileBlocked;
 void beforeSleep(struct aeEventLoop *eventLoop) {
     UNUSED(eventLoop);
 
+    size_t zmalloc_used = zmalloc_used_memory();
+    if (zmalloc_used > server.stat_peak_memory)
+        server.stat_peak_memory = zmalloc_used;
+
     /* Just call a subset of vital functions in case we are re-entering
      * the event loop from processEventsWhileBlocked(). Note that in this
      * case we keep track of the number of events we are processing, since
@@ -3581,6 +3585,12 @@ void call(client *c, int flags) {
 
     server.fixed_time_expire--;
     server.stat_numcommands++;
+
+    /* Record peak memory after each command and before the eviction that runs
+     * before the next command. */
+    size_t zmalloc_used = zmalloc_used_memory();
+    if (zmalloc_used > server.stat_peak_memory)
+        server.stat_peak_memory = zmalloc_used;
 }
 
 /* Used when a command that is ready for execution needs to be rejected, due to

From 0047702aabdf53651e65cda8f1e7f7ed432353e7 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Mon, 19 Oct 2020 00:33:55 -0400
Subject: [PATCH 0739/1098] Support ACL for Sentinel Mode (#7888)

This commit implements ACL for Sentinel mode, main work of this PR includes:

- Update Sentinel command table in order to better support ACLs.
- Fix couple of things which currently blocks the support for ACL on sentinel mode.
- Provide "sentinel sentinel-user" and "sentinel sentinel-pass " configuration in order to let sentinel authenticate with a specific user in other sentinels.
- requirepass is kept just for compatibility with old config files

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 sentinel.conf  | 23 ++++++++++++++++
 src/acl.c      | 19 ++++++++++----
 src/sentinel.c | 71 ++++++++++++++++++++++++++++++++++++++------------
 src/server.c   |  1 +
 src/server.h   |  1 +
 5 files changed, 94 insertions(+), 21 deletions(-)

diff --git a/sentinel.conf b/sentinel.conf
index b6ff05f25..50a36a314 100644
--- a/sentinel.conf
+++ b/sentinel.conf
@@ -131,6 +131,29 @@ sentinel down-after-milliseconds mymaster 30000
 # other Sentinels. So you need to configure all your Sentinels in a given
 # group with the same "requirepass" password. Check the following documentation
 # for more info: https://redis.io/topics/sentinel
+#
+# IMPORTANT NOTE: starting with Redis 6.2 "requirepass" is a compatibility
+# layer on top of the ACL system. The option effect will be just setting
+# the password for the default user. Clients will still authenticate using
+# AUTH <password> as usually, or more explicitly with AUTH default <password>
+# if they follow the new protocol: both will work.
+#
+# New config files are advised to use separate authentication control for
+# incoming connections (via ACL), and for outgoing connections (via
+# sentinel-user and sentinel-pass) 
+#
+# The requirepass is not compatable with aclfile option and the ACL LOAD
+# command, these will cause requirepass to be ignored.
+
+# sentinel sentinel-user <username>
+#
+# You can configure Sentinel to authenticate with other Sentinels with specific
+# user name. 
+
+# sentinel sentinel-pass <password>
+#
+# The password for Sentinel to authenticate with other Sentinels. If sentinel-user
+# is not configured, Sentinel will use 'default' user with sentinel-pass to authenticate.
 
 # sentinel parallel-syncs <master-name> <numreplicas>
 #
diff --git a/src/acl.c b/src/acl.c
index aa3ed197f..d9ee774c8 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -53,6 +53,10 @@ list *UsersToLoad;  /* This is a list of users found in the configuration file
 list *ACLLog;       /* Our security log, the user is able to inspect that
                        using the ACL LOG command .*/
 
+static rax *commandId = NULL; /* Command name to id mapping */
+
+static unsigned long nextid = 0; /* Next command id that has not been assigned */
+
 struct ACLCategoryItem {
     const char *name;
     uint64_t flag;
@@ -1031,18 +1035,16 @@ int ACLAuthenticateUser(client *c, robj *username, robj *password) {
  * command name, so that a command retains the same ID in case of modules that
  * are unloaded and later reloaded. */
 unsigned long ACLGetCommandID(const char *cmdname) {
-    static rax *map = NULL;
-    static unsigned long nextid = 0;
 
     sds lowername = sdsnew(cmdname);
     sdstolower(lowername);
-    if (map == NULL) map = raxNew();
-    void *id = raxFind(map,(unsigned char*)lowername,sdslen(lowername));
+    if (commandId == NULL) commandId = raxNew();
+    void *id = raxFind(commandId,(unsigned char*)lowername,sdslen(lowername));
     if (id != raxNotFound) {
         sdsfree(lowername);
         return (unsigned long)id;
     }
-    raxInsert(map,(unsigned char*)lowername,strlen(lowername),
+    raxInsert(commandId,(unsigned char*)lowername,strlen(lowername),
               (void*)nextid,NULL);
     sdsfree(lowername);
     unsigned long thisid = nextid;
@@ -1060,6 +1062,13 @@ unsigned long ACLGetCommandID(const char *cmdname) {
     return thisid;
 }
 
+/* Clear command id table and reset nextid to 0. */
+void ACLClearCommandID(void) {
+    if (commandId) raxFree(commandId);
+    commandId = NULL;
+    nextid = 0;
+}
+
 /* Return an username by its name, or NULL if the user does not exist. */
 user *ACLGetUserByName(const char *name, size_t namelen) {
     void *myuser = raxFind(Users,(unsigned char*)name,namelen);
diff --git a/src/sentinel.c b/src/sentinel.c
index a346a2547..31a66a6ff 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -257,6 +257,8 @@ struct sentinelState {
     unsigned long simfailure_flags; /* Failures simulation. */
     int deny_scripts_reconfig; /* Allow SENTINEL SET ... to change script
                                   paths at runtime? */
+    char *sentinel_auth_pass;    /* Password to use for AUTH against other sentinel */
+    char *sentinel_auth_user;    /* Username for ACLs AUTH against other sentinel. */
 } sentinel;
 
 /* A script execution job. */
@@ -451,19 +453,20 @@ void sentinelPublishCommand(client *c);
 void sentinelRoleCommand(client *c);
 
 struct redisCommand sentinelcmds[] = {
-    {"ping",pingCommand,1,"",0,NULL,0,0,0,0,0},
-    {"sentinel",sentinelCommand,-2,"",0,NULL,0,0,0,0,0},
-    {"subscribe",subscribeCommand,-2,"",0,NULL,0,0,0,0,0},
-    {"unsubscribe",unsubscribeCommand,-1,"",0,NULL,0,0,0,0,0},
-    {"psubscribe",psubscribeCommand,-2,"",0,NULL,0,0,0,0,0},
-    {"punsubscribe",punsubscribeCommand,-1,"",0,NULL,0,0,0,0,0},
-    {"publish",sentinelPublishCommand,3,"",0,NULL,0,0,0,0,0},
-    {"info",sentinelInfoCommand,-1,"",0,NULL,0,0,0,0,0},
-    {"role",sentinelRoleCommand,1,"ok-loading",0,NULL,0,0,0,0,0},
-    {"client",clientCommand,-2,"read-only no-script",0,NULL,0,0,0,0,0},
-    {"shutdown",shutdownCommand,-1,"",0,NULL,0,0,0,0,0},
-    {"auth",authCommand,2,"no-auth no-script ok-loading ok-stale fast",0,NULL,0,0,0,0,0},
-    {"hello",helloCommand,-2,"no-auth no-script fast",0,NULL,0,0,0,0,0}
+    {"ping",pingCommand,1,"fast @connection",0,NULL,0,0,0,0,0},
+    {"sentinel",sentinelCommand,-2,"admin",0,NULL,0,0,0,0,0},
+    {"subscribe",subscribeCommand,-2,"pub-sub",0,NULL,0,0,0,0,0},
+    {"unsubscribe",unsubscribeCommand,-1,"pub-sub",0,NULL,0,0,0,0,0},
+    {"psubscribe",psubscribeCommand,-2,"pub-sub",0,NULL,0,0,0,0,0},
+    {"punsubscribe",punsubscribeCommand,-1,"pub-sub",0,NULL,0,0,0,0,0},
+    {"publish",sentinelPublishCommand,3,"pub-sub fast",0,NULL,0,0,0,0,0},
+    {"info",sentinelInfoCommand,-1,"random @dangerous",0,NULL,0,0,0,0,0},
+    {"role",sentinelRoleCommand,1,"fast read-only @dangerous",0,NULL,0,0,0,0,0},
+    {"client",clientCommand,-2,"admin random @connection",0,NULL,0,0,0,0,0},
+    {"shutdown",shutdownCommand,-1,"admin",0,NULL,0,0,0,0,0},
+    {"auth",authCommand,-2,"no-auth fast @connection",0,NULL,0,0,0,0,0},
+    {"hello",helloCommand,-2,"no-auth fast @connection",0,NULL,0,0,0,0,0},
+    {"acl",aclCommand,-2,"admin",0,NULL,0,0,0,0,0,0}
 };
 
 /* This function overwrites a few normal Redis config default with Sentinel
@@ -480,12 +483,16 @@ void initSentinel(void) {
     /* Remove usual Redis commands from the command table, then just add
      * the SENTINEL command. */
     dictEmpty(server.commands,NULL);
+    dictEmpty(server.orig_commands,NULL);
+    ACLClearCommandID();
     for (j = 0; j < sizeof(sentinelcmds)/sizeof(sentinelcmds[0]); j++) {
         int retval;
         struct redisCommand *cmd = sentinelcmds+j;
-
+        cmd->id = ACLGetCommandID(cmd->name); /* Assign the ID used for ACL. */
         retval = dictAdd(server.commands, sdsnew(cmd->name), cmd);
         serverAssert(retval == DICT_OK);
+        retval = dictAdd(server.orig_commands, sdsnew(cmd->name), cmd);
+        serverAssert(retval == DICT_OK);
 
         /* Translate the command string flags description into an actual
          * set of flags. */
@@ -505,6 +512,8 @@ void initSentinel(void) {
     sentinel.announce_port = 0;
     sentinel.simfailure_flags = SENTINEL_SIMFAILURE_NONE;
     sentinel.deny_scripts_reconfig = SENTINEL_DEFAULT_DENY_SCRIPTS_RECONFIG;
+    sentinel.sentinel_auth_pass = NULL;
+    sentinel.sentinel_auth_user = NULL;
     memset(sentinel.myid,0,sizeof(sentinel.myid));
 }
 
@@ -1765,6 +1774,14 @@ char *sentinelHandleConfiguration(char **argv, int argc) {
             return "Please specify yes or no for the "
                    "deny-scripts-reconfig options.";
         }
+    } else if (!strcasecmp(argv[0],"sentinel-user") && argc == 2) {
+        /* sentinel-user <user-name> */
+        if (strlen(argv[1]))
+            sentinel.sentinel_auth_user = sdsnew(argv[1]);
+    } else if (!strcasecmp(argv[0],"sentinel-pass") && argc == 2) {
+        /* sentinel-pass <password> */
+        if (strlen(argv[1]))
+            sentinel.sentinel_auth_pass = sdsnew(argv[1]);
     } else {
         return "Unrecognized sentinel configuration statement.";
     }
@@ -1938,6 +1955,19 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
         rewriteConfigRewriteLine(state,"sentinel",line,1);
     }
 
+    /* sentinel sentinel-user. */
+    if (sentinel.sentinel_auth_user) {
+        line = sdscatprintf(sdsempty(), "sentinel sentinel-user %s", sentinel.sentinel_auth_user);
+        rewriteConfigRewriteLine(state,"sentinel",line,1);
+    }
+
+    /* sentinel sentinel-pass. */
+    if (sentinel.sentinel_auth_pass) {
+        line = sdscatprintf(sdsempty(), "sentinel sentinel-pass %s", sentinel.sentinel_auth_pass);
+        rewriteConfigRewriteLine(state,"sentinel",line,1);
+    }
+
+
     dictReleaseIterator(di);
 }
 
@@ -1993,8 +2023,17 @@ void sentinelSendAuthIfNeeded(sentinelRedisInstance *ri, redisAsyncContext *c) {
         auth_pass = ri->master->auth_pass;
         auth_user = ri->master->auth_user;
     } else if (ri->flags & SRI_SENTINEL) {
-        auth_pass = server.requirepass;
-        auth_user = NULL;
+        /* If sentinel_auth_user is NULL, AUTH will use default user
+           with sentinel_auth_pass to autenticate */
+        if (sentinel.sentinel_auth_pass) {
+            auth_pass = sentinel.sentinel_auth_pass;
+            auth_user = sentinel.sentinel_auth_user;
+        } else {
+            /* Compatibility with old configs. requirepass is used
+             * for both incoming and outgoing authentication. */
+            auth_pass = server.requirepass;
+            auth_user = NULL;
+        }
     }
 
     if (auth_pass && auth_user == NULL) {
diff --git a/src/server.c b/src/server.c
index 43d7c337c..67d050ea0 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5453,6 +5453,7 @@ int main(int argc, char **argv) {
             }
         }
     } else {
+        ACLLoadUsersAtStartup();
         InitServerLast();
         sentinelIsRunning();
         if (server.supervised_mode == SUPERVISED_SYSTEMD) {
diff --git a/src/server.h b/src/server.h
index 33ce2b89c..5d53d9473 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1945,6 +1945,7 @@ void ACLInit(void);
 int ACLCheckUserCredentials(robj *username, robj *password);
 int ACLAuthenticateUser(client *c, robj *username, robj *password);
 unsigned long ACLGetCommandID(const char *cmdname);
+void ACLClearCommandID(void);
 user *ACLGetUserByName(const char *name, size_t namelen);
 int ACLCheckCommandPerm(client *c, int *keyidxptr);
 int ACLSetUser(user *u, const char *op, ssize_t oplen);

From 04a0af9085ed91eace8af94f71a07b651a1af454 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Mon, 19 Oct 2020 08:32:18 -0400
Subject: [PATCH 0740/1098] fix double fclose in aofrewrite (#7919)

minor fix for a bug which happen on error handling code
and doesn't look like it could have caused any real harm
(fd number wouldn't have been reused yet)
---
 src/aof.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index 9cd487cc4..5c46c76f6 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1449,7 +1449,7 @@ werr:
  * are inserted using a single command. */
 int rewriteAppendOnlyFile(char *filename) {
     rio aof;
-    FILE *fp;
+    FILE *fp = NULL;
     char tmpfile[256];
     char byte;
 
@@ -1526,9 +1526,10 @@ int rewriteAppendOnlyFile(char *filename) {
         goto werr;
 
     /* Make sure data will not remain on the OS's output buffers */
-    if (fflush(fp) == EOF) goto werr;
-    if (fsync(fileno(fp)) == -1) goto werr;
-    if (fclose(fp) == EOF) goto werr;
+    if (fflush(fp)) goto werr;
+    if (fsync(fileno(fp))) goto werr;
+    if (fclose(fp)) { fp = NULL; goto werr; }
+    fp = NULL;
 
     /* Use RENAME to make sure the DB file is changed atomically only
      * if the generate DB file is ok. */
@@ -1544,7 +1545,7 @@ int rewriteAppendOnlyFile(char *filename) {
 
 werr:
     serverLog(LL_WARNING,"Write error writing append only file on disk: %s", strerror(errno));
-    fclose(fp);
+    if (fp) fclose(fp);
     unlink(tmpfile);
     stopSaving(0);
     return C_ERR;

From a425e1d26df02ac5ce0fc9a0c679b99d72550009 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 20 Oct 2020 09:12:24 +0300
Subject: [PATCH 0741/1098] fix 32bit build warnings (#7926)

---
 src/config.c   | 2 +-
 src/listpack.c | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/config.c b/src/config.c
index 2a9f49d80..afd4bc81c 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1610,7 +1610,7 @@ int rewriteConfigOverwriteFile(char *configfile, sds content) {
          written_bytes = write(fd, content + offset, sdslen(content) - offset);
          if (written_bytes <= 0) {
              if (errno == EINTR) continue; /* FD is blocking, no other retryable errors */
-             serverLog(LL_WARNING, "Failed after writing (%ld) bytes to tmp config file (%s)", offset, strerror(errno));
+             serverLog(LL_WARNING, "Failed after writing (%zd) bytes to tmp config file (%s)", offset, strerror(errno));
              goto cleanup;
          }
          offset+=written_bytes;
diff --git a/src/listpack.c b/src/listpack.c
index 7e2da9b74..f8c34429e 100644
--- a/src/listpack.c
+++ b/src/listpack.c
@@ -768,10 +768,10 @@ unsigned char *lpSeek(unsigned char *lp, long index) {
     if (numele != LP_HDR_NUMELE_UNKNOWN) {
         if (index < 0) index = (long)numele+index;
         if (index < 0) return NULL; /* Index still < 0 means out of range. */
-        if ((long)index >= numele) return NULL; /* Out of range the other side. */
+        if (index >= (long)numele) return NULL; /* Out of range the other side. */
         /* We want to scan right-to-left if the element we are looking for
          * is past the half of the listpack. */
-        if ((long)index > numele/2) {
+        if (index > (long)numele/2) {
             forward = 0;
             /* Right to left scanning always expects a negative index. Convert
              * our index to negative form. */

From 6cf23d6610677d38e4892e02649fc2de7095274b Mon Sep 17 00:00:00 2001
From: filipe oliveira <filipecosta.90@gmail.com>
Date: Tue, 20 Oct 2020 17:52:05 +0100
Subject: [PATCH 0742/1098] Fixed bug concerning redis-benchmark non clustered
 benchmark forcing always the same hash tag {tag} (#7931)

Adding the ":{tag}" only if --cluster is used, so that when used against
a proxy it generates traffic to all shards.

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/redis-benchmark.c | 55 ++++++++++++++++++++++++-------------------
 1 file changed, 31 insertions(+), 24 deletions(-)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 7a7828184..c17178e85 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -779,7 +779,7 @@ static client createClient(char *cmd, size_t len, client from, int thread_id) {
                 }
                 c->stagptr[c->staglen++] = p;
                 c->stagfree--;
-                p += 5; /* 12 is strlen("{tag}"). */
+                p += 5; /* 5 is strlen("{tag}"). */
             }
         }
     }
@@ -1570,7 +1570,7 @@ int test_is_selected(char *name) {
 
 int main(int argc, const char **argv) {
     int i;
-    char *data, *cmd;
+    char *data, *cmd, *tag;
     int len;
 
     client c;
@@ -1617,7 +1617,12 @@ int main(int argc, const char **argv) {
     argc -= i;
     argv += i;
 
+    tag = "";
+
     if (config.cluster_mode) {
+        // We only include the slot placeholder {tag} if cluster mode is enabled
+        tag = ":{tag}";
+
         /* Fetch cluster configuration. */
         if (!fetchClusterConfiguration() || !config.cluster_nodes) {
             if (!config.hostsocket) {
@@ -1723,63 +1728,63 @@ int main(int argc, const char **argv) {
         }
 
         if (test_is_selected("set")) {
-            len = redisFormatCommand(&cmd,"SET key:{tag}:__rand_int__ %s",data);
+            len = redisFormatCommand(&cmd,"SET key%s:__rand_int__ %s",tag,data);
             benchmark("SET",cmd,len);
             free(cmd);
         }
 
         if (test_is_selected("get")) {
-            len = redisFormatCommand(&cmd,"GET key:{tag}:__rand_int__");
+            len = redisFormatCommand(&cmd,"GET key%s:__rand_int__",tag);
             benchmark("GET",cmd,len);
             free(cmd);
         }
 
         if (test_is_selected("incr")) {
-            len = redisFormatCommand(&cmd,"INCR counter:{tag}:__rand_int__");
+            len = redisFormatCommand(&cmd,"INCR counter%s:__rand_int__",tag);
             benchmark("INCR",cmd,len);
             free(cmd);
         }
 
         if (test_is_selected("lpush")) {
-            len = redisFormatCommand(&cmd,"LPUSH mylist:{tag} %s",data);
+            len = redisFormatCommand(&cmd,"LPUSH mylist%s %s",tag,data);
             benchmark("LPUSH",cmd,len);
             free(cmd);
         }
 
         if (test_is_selected("rpush")) {
-            len = redisFormatCommand(&cmd,"RPUSH mylist:{tag} %s",data);
+            len = redisFormatCommand(&cmd,"RPUSH mylist%s %s",tag,data);
             benchmark("RPUSH",cmd,len);
             free(cmd);
         }
 
         if (test_is_selected("lpop")) {
-            len = redisFormatCommand(&cmd,"LPOP mylist:{tag}");
+            len = redisFormatCommand(&cmd,"LPOP mylist%s",tag);
             benchmark("LPOP",cmd,len);
             free(cmd);
         }
 
         if (test_is_selected("rpop")) {
-            len = redisFormatCommand(&cmd,"RPOP mylist:{tag}");
+            len = redisFormatCommand(&cmd,"RPOP mylist%s",tag);
             benchmark("RPOP",cmd,len);
             free(cmd);
         }
 
         if (test_is_selected("sadd")) {
             len = redisFormatCommand(&cmd,
-                "SADD myset:{tag} element:__rand_int__");
+                "SADD myset%s element:__rand_int__",tag);
             benchmark("SADD",cmd,len);
             free(cmd);
         }
 
         if (test_is_selected("hset")) {
             len = redisFormatCommand(&cmd,
-                "HSET myhash:{tag} element:__rand_int__ %s",data);
+                "HSET myhash%s element:__rand_int__ %s",tag,data);
             benchmark("HSET",cmd,len);
             free(cmd);
         }
 
         if (test_is_selected("spop")) {
-            len = redisFormatCommand(&cmd,"SPOP myset:{tag}");
+            len = redisFormatCommand(&cmd,"SPOP myset%s",tag);
             benchmark("SPOP",cmd,len);
             free(cmd);
         }
@@ -1788,13 +1793,13 @@ int main(int argc, const char **argv) {
             char *score = "0";
             if (config.randomkeys) score = "__rand_int__";
             len = redisFormatCommand(&cmd,
-                "ZADD myzset:{tag} %s element:__rand_int__",score);
+                "ZADD myzset%s %s element:__rand_int__",tag,score);
             benchmark("ZADD",cmd,len);
             free(cmd);
         }
 
         if (test_is_selected("zpopmin")) {
-            len = redisFormatCommand(&cmd,"ZPOPMIN myzset:{tag}");
+            len = redisFormatCommand(&cmd,"ZPOPMIN myzset%s",tag);
             benchmark("ZPOPMIN",cmd,len);
             free(cmd);
         }
@@ -1805,45 +1810,47 @@ int main(int argc, const char **argv) {
             test_is_selected("lrange_500") ||
             test_is_selected("lrange_600"))
         {
-            len = redisFormatCommand(&cmd,"LPUSH mylist:{tag} %s",data);
+            len = redisFormatCommand(&cmd,"LPUSH mylist%s %s",tag,data);
             benchmark("LPUSH (needed to benchmark LRANGE)",cmd,len);
             free(cmd);
         }
 
         if (test_is_selected("lrange") || test_is_selected("lrange_100")) {
-            len = redisFormatCommand(&cmd,"LRANGE mylist:{tag} 0 99");
+            len = redisFormatCommand(&cmd,"LRANGE mylist%s 0 99",tag);
             benchmark("LRANGE_100 (first 100 elements)",cmd,len);
             free(cmd);
         }
 
         if (test_is_selected("lrange") || test_is_selected("lrange_300")) {
-            len = redisFormatCommand(&cmd,"LRANGE mylist:{tag} 0 299");
+            len = redisFormatCommand(&cmd,"LRANGE mylist%s 0 299",tag);
             benchmark("LRANGE_300 (first 300 elements)",cmd,len);
             free(cmd);
         }
 
         if (test_is_selected("lrange") || test_is_selected("lrange_500")) {
-            len = redisFormatCommand(&cmd,"LRANGE mylist:{tag} 0 449");
+            len = redisFormatCommand(&cmd,"LRANGE mylist%s 0 449",tag);
             benchmark("LRANGE_500 (first 450 elements)",cmd,len);
             free(cmd);
         }
 
         if (test_is_selected("lrange") || test_is_selected("lrange_600")) {
-            len = redisFormatCommand(&cmd,"LRANGE mylist:{tag} 0 599");
+            len = redisFormatCommand(&cmd,"LRANGE mylist%s 0 599",tag);
             benchmark("LRANGE_600 (first 600 elements)",cmd,len);
             free(cmd);
         }
 
         if (test_is_selected("mset")) {
-            const char *argv[21];
-            argv[0] = "MSET";
+            const char *cmd_argv[21];
+            cmd_argv[0] = "MSET";
+            sds key_placeholder = sdscatprintf(sdsnew(""),"key%s:__rand_int__",tag);
             for (i = 1; i < 21; i += 2) {
-                argv[i] = "key:{tag}:__rand_int__";
-                argv[i+1] = data;
+                cmd_argv[i] = key_placeholder;
+                cmd_argv[i+1] = data;
             }
-            len = redisFormatCommandArgv(&cmd,21,argv,NULL);
+            len = redisFormatCommandArgv(&cmd,21,cmd_argv,NULL);
             benchmark("MSET (10 keys)",cmd,len);
             free(cmd);
+            sdsfree(key_placeholder);
         }
 
         if (!config.csv) printf("\n");

From 843a13e88f3544e936ead06a9d05af28eeefd5a1 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 22 Oct 2020 11:10:53 +0300
Subject: [PATCH 0743/1098] Add a --no-latency tests flag. (#7939)

Useful for running tests on systems which may be way slower than usual.
---
 .github/workflows/daily.yml       |  2 +-
 tests/integration/replication.tcl |  4 ++--
 tests/test_helper.tcl             |  4 ++++
 tests/unit/latency-monitor.tcl    | 10 +++++++---
 tests/unit/memefficiency.tcl      | 16 ++++++++++++----
 5 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 1ccd99805..d78252026 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -200,7 +200,7 @@ jobs:
       run: make
     - name: test
       run: |
-        ./runtest --accurate --verbose
+        ./runtest --accurate --verbose --no-latency
     - name: module api test
       run: ./runtest-moduleapi --verbose
     - name: sentinel tests
diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl
index 06414dd8a..f7f8d241b 100644
--- a/tests/integration/replication.tcl
+++ b/tests/integration/replication.tcl
@@ -655,11 +655,11 @@ start_server {tags {"repl"}} {
                             puts "master utime: $master_utime"
                             puts "master stime: $master_stime"
                         }
-                        if {$all_drop == "all" || $all_drop == "slow"} {
+                        if {!$::no_latency && ($all_drop == "all" || $all_drop == "slow")} {
                             assert {$master_utime < 70}
                             assert {$master_stime < 70}
                         }
-                        if {$all_drop == "none" || $all_drop == "fast"} {
+                        if {!$::no_latency && ($all_drop == "none" || $all_drop == "fast")} {
                             assert {$master_utime < 15}
                             assert {$master_stime < 15}
                         }
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index b60adb881..37af46947 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -88,6 +88,7 @@ set ::quiet 0
 set ::denytags {}
 set ::skiptests {}
 set ::skipunits {}
+set ::no_latency 0
 set ::allowtags {}
 set ::only_tests {}
 set ::single_tests {}
@@ -540,6 +541,7 @@ proc print_help_screen {} {
         "--skipfile <file>  Name of a file containing test names that should be skipped (one per line)."
         "--skiptest <name>  Name of a file containing test names that should be skipped (one per line)."
         "--dont-clean       Don't delete redis log files after the run."
+        "--no-latency       Skip latency measurements and validation by some tests."
         "--stop             Blocks once the first test fails."
         "--loop             Execute the specified set of tests forever."
         "--wait-server      Wait after server is started (so that you can attach a debugger)."
@@ -641,6 +643,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
         set ::durable 1
     } elseif {$opt eq {--dont-clean}} {
         set ::dont_clean 1
+    } elseif {$opt eq {--no-latency}} {
+        set ::no_latency 1
     } elseif {$opt eq {--wait-server}} {
         set ::wait_server 1
     } elseif {$opt eq {--stop}} {
diff --git a/tests/unit/latency-monitor.tcl b/tests/unit/latency-monitor.tcl
index 18b9ecebb..9ef0c6154 100644
--- a/tests/unit/latency-monitor.tcl
+++ b/tests/unit/latency-monitor.tcl
@@ -17,7 +17,9 @@ start_server {tags {"latency-monitor"}} {
         set max 450
         foreach event [r latency history command] {
             lassign $event time latency
-            assert {$latency >= $min && $latency <= $max}
+            if {!$::no_latency} {
+                assert {$latency >= $min && $latency <= $max}
+            }
             incr min 100
             incr max 100
             set last_time $time ; # Used in the next test
@@ -28,8 +30,10 @@ start_server {tags {"latency-monitor"}} {
         foreach event [r latency latest] {
             lassign $event eventname time latency max
             assert {$eventname eq "command"}
-            assert {$max >= 450 & $max <= 650}
-            assert {$time == $last_time}
+            if {!$::no_latency} {
+                assert {$max >= 450 & $max <= 650}
+                assert {$time == $last_time}
+            }
             break
         }
     }
diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl
index fe4e44d3e..a81d33d9b 100644
--- a/tests/unit/memefficiency.tcl
+++ b/tests/unit/memefficiency.tcl
@@ -105,7 +105,9 @@ start_server {tags {"defrag"} overrides {appendonly yes auto-aof-rewrite-percent
                 assert {$frag < 1.1}
                 # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75,
                 # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher
-                assert {$max_latency <= 30}
+                if {!$::no_latency} {
+                    assert {$max_latency <= 30}
+                }
             }
             # verify the data isn't corrupted or changed
             set newdigest [r debug digest]
@@ -148,7 +150,9 @@ start_server {tags {"defrag"} overrides {appendonly yes auto-aof-rewrite-percent
                 assert {$frag < 1.4}
                 # since the AOF contains simple (fast) SET commands (and the cron during loading runs every 1000 commands),
                 # it'll still not block the loading for long periods of time.
-                assert {$max_latency <= 30}
+                if {!$::no_latency} {
+                    assert {$max_latency <= 30}
+                }
             }
         }
         r config set appendonly no
@@ -273,7 +277,9 @@ start_server {tags {"defrag"} overrides {appendonly yes auto-aof-rewrite-percent
                 assert {$frag < 1.1}
                 # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75,
                 # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher
-                assert {$max_latency <= 30}
+                if {!$::no_latency} {
+                    assert {$max_latency <= 30}
+                }
             }
             # verify the data isn't corrupted or changed
             set newdigest [r debug digest]
@@ -368,7 +374,9 @@ start_server {tags {"defrag"} overrides {appendonly yes auto-aof-rewrite-percent
                 assert {$frag < 1.1}
                 # due to high fragmentation, 100hz, and active-defrag-cycle-max set to 75,
                 # we expect max latency to be not much higher than 7.5ms but due to rare slowness threshold is set higher
-                assert {$max_latency <= 30}
+                if {!$::no_latency} {
+                    assert {$max_latency <= 30}
+                }
 
                 # in extreme cases of stagnation, we see over 20m misses before the tests aborts with "defrag didn't stop",
                 # in normal cases we only see 100k misses out of 500k elements

From c96ece9f5e7b80d65ca4d1a2b801effe68425c90 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 22 Oct 2020 11:34:54 +0300
Subject: [PATCH 0744/1098] improve verbose logging on failed test. print log
 file lines (#7938)

---
 tests/support/util.tcl | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index ac4fca501..b4c1a3841 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -142,6 +142,10 @@ proc wait_for_log_messages {srv_idx patterns from_line maxtries delay} {
         after $delay
     }
     if {$retry == 0} {
+        if {$::verbose} {
+            puts "content of $stdout from line: $from_line:"
+            puts [exec tail -n +$from_line < $stdout]
+        }
         fail "log message of '$patterns' not found in $stdout after line: $from_line till line: [expr $next_line -1]"
     }
 }

From 556acefe7556443b6d1741d804add92047bf4a8b Mon Sep 17 00:00:00 2001
From: Qu Chen <QuChen88@users.noreply.github.com>
Date: Thu, 22 Oct 2020 02:57:45 -0700
Subject: [PATCH 0745/1098] WATCH no longer ignores keys which have expired for
 MULTI/EXEC. (#7920)

This wrong behavior was backed by a test, and also documentation, and dates back to 2010.
But it makes no sense to anyone involved so it was decided to change that.

Note that 20eeddf (invalidate watch on expire on access) was released in 6.0 RC2
and 2d1968f released in in 6.0.0 GA (invalidate watch when key is evicted).
both of which do similar changes.
---
 src/expire.c         | 2 +-
 tests/unit/multi.tcl | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/expire.c b/src/expire.c
index 85fd59fe2..cac208573 100644
--- a/src/expire.c
+++ b/src/expire.c
@@ -64,7 +64,7 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) {
             dbSyncDelete(db,keyobj);
         notifyKeyspaceEvent(NOTIFY_EXPIRED,
             "expired",keyobj,db->id);
-        trackingInvalidateKey(NULL,keyobj);
+        signalModifiedKey(NULL, db, keyobj);
         decrRefCount(keyobj);
         server.stat_expiredkeys++;
         return 1;
diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl
index 817d509c5..e45a2d8f4 100644
--- a/tests/unit/multi.tcl
+++ b/tests/unit/multi.tcl
@@ -221,7 +221,7 @@ start_server {tags {"multi"}} {
         r exec
     } {}
 
-    test {WATCH will not consider touched expired keys} {
+    test {WATCH will consider touched expired keys} {
         r del x
         r set x foo
         r expire x 1
@@ -230,7 +230,7 @@ start_server {tags {"multi"}} {
         r multi
         r ping
         r exec
-    } {PONG}
+    } {}
 
     test {DISCARD should clear the WATCH dirty flag on the client} {
         r watch x

From 0f370f9b667fab4bede084f1b8b47c93f0d56069 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Thu, 22 Oct 2020 12:47:32 -0400
Subject: [PATCH 0746/1098] do not add save parameter during config rewrite in
 sentinel mode (#7945)

Previous code would have added default redis save parameters
to the config file on rewrite, which would have been silently ignored
when the config file is loaded.

The new code avoids adding this, and also actively removes these lines
If added by a previous config rewrite.
---
 src/config.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/config.c b/src/config.c
index afd4bc81c..aaee2db65 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1335,6 +1335,12 @@ void rewriteConfigSaveOption(struct rewriteConfigState *state) {
     int j;
     sds line;
 
+    /* In Sentinel mode we don't need to rewrite the save parameters */
+    if (server.sentinel_mode) {
+        rewriteConfigMarkAsProcessed(state,"save");
+        return;
+    }
+
     /* Note that if there are no save parameters at all, all the current
      * config line with "save" will be detected as orphaned and deleted,
      * resulting into no RDB persistence as expected. */

From e05a7df7f942a2e17dee3e0e3d67d80e8d1d2e6e Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Fri, 23 Oct 2020 20:26:30 +0800
Subject: [PATCH 0747/1098] Update rdb_last_bgsave_time_sec in INFO on diskless
 replication (#7917)

`info Persistence` will include correct (updated) rdb_last_bgsave_time_sec
For diskless bgsave (sockets) too (like a few other persistence info fields).

Refactor code to reduce duplicate code.
---
 src/rdb.c | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/src/rdb.c b/src/rdb.c
index 73bc143f5..e94a79125 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2425,7 +2425,7 @@ int rdbLoad(char *filename, rdbSaveInfo *rsi, int rdbflags) {
 
 /* A background saving child (BGSAVE) terminated its work. Handle this.
  * This function covers the case of actual BGSAVEs. */
-void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal) {
+static void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal) {
     if (!bysignal && exitcode == 0) {
         serverLog(LL_NOTICE,
             "Background saving terminated with success");
@@ -2449,19 +2449,12 @@ void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal) {
         if (bysignal != SIGUSR1)
             server.lastbgsave_status = C_ERR;
     }
-    server.rdb_child_pid = -1;
-    server.rdb_child_type = RDB_CHILD_TYPE_NONE;
-    server.rdb_save_time_last = time(NULL)-server.rdb_save_time_start;
-    server.rdb_save_time_start = -1;
-    /* Possibly there are slaves waiting for a BGSAVE in order to be served
-     * (the first stage of SYNC is a bulk transfer of dump.rdb) */
-    updateSlavesWaitingBgsave((!bysignal && exitcode == 0) ? C_OK : C_ERR, RDB_CHILD_TYPE_DISK);
 }
 
 /* A background saving child (BGSAVE) terminated its work. Handle this.
  * This function covers the case of RDB -> Slaves socket transfers for
  * diskless replication. */
-void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) {
+static void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) {
     if (!bysignal && exitcode == 0) {
         serverLog(LL_NOTICE,
             "Background RDB transfer terminated with success");
@@ -2471,9 +2464,6 @@ void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) {
         serverLog(LL_WARNING,
             "Background transfer terminated by signal %d", bysignal);
     }
-    server.rdb_child_pid = -1;
-    server.rdb_child_type = RDB_CHILD_TYPE_NONE;
-    server.rdb_save_time_start = -1;
     if (server.rdb_child_exit_pipe!=-1)
         close(server.rdb_child_exit_pipe);
     close(server.rdb_pipe_read);
@@ -2486,12 +2476,11 @@ void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) {
     zfree(server.rdb_pipe_buff);
     server.rdb_pipe_buff = NULL;
     server.rdb_pipe_bufflen = 0;
-
-    updateSlavesWaitingBgsave((!bysignal && exitcode == 0) ? C_OK : C_ERR, RDB_CHILD_TYPE_SOCKET);
 }
 
 /* When a background RDB saving/transfer terminates, call the right handler. */
 void backgroundSaveDoneHandler(int exitcode, int bysignal) {
+    int type = server.rdb_child_type;
     switch(server.rdb_child_type) {
     case RDB_CHILD_TYPE_DISK:
         backgroundSaveDoneHandlerDisk(exitcode,bysignal);
@@ -2503,6 +2492,14 @@ void backgroundSaveDoneHandler(int exitcode, int bysignal) {
         serverPanic("Unknown RDB child type.");
         break;
     }
+
+    server.rdb_child_pid = -1;
+    server.rdb_child_type = RDB_CHILD_TYPE_NONE;
+    server.rdb_save_time_last = time(NULL)-server.rdb_save_time_start;
+    server.rdb_save_time_start = -1;
+    /* Possibly there are slaves waiting for a BGSAVE in order to be served
+     * (the first stage of SYNC is a bulk transfer of dump.rdb) */
+    updateSlavesWaitingBgsave((!bysignal && exitcode == 0) ? C_OK : C_ERR, type);
 }
 
 /* Kill the RDB saving child using SIGUSR1 (so that the parent will know

From ebfa7699252b858af57cdd5388c5bd6d7439a021 Mon Sep 17 00:00:00 2001
From: Zach Fewtrell <fewtrell@users.noreply.github.com>
Date: Sun, 25 Oct 2020 01:05:38 -0700
Subject: [PATCH 0748/1098] fix invalid 'failover' identifier in cluster slave
 selection test (#7942)

---
 tests/cluster/tests/05-slave-selection.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/cluster/tests/05-slave-selection.tcl b/tests/cluster/tests/05-slave-selection.tcl
index bcb0fa1ea..6769d3070 100644
--- a/tests/cluster/tests/05-slave-selection.tcl
+++ b/tests/cluster/tests/05-slave-selection.tcl
@@ -163,7 +163,7 @@ test "New Master down consecutively" {
         wait_for_condition 1000 50 {
             [master_detected $instances]
         } else {
-            failover "No failover detected when master $master_id fails"
+            fail "No failover detected when master $master_id fails"
         }
 
         assert_cluster_state ok

From 27f4c212f377dfbe1a60fb96635ae16bc405b959 Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Sun, 25 Oct 2020 12:14:05 +0000
Subject: [PATCH 0749/1098] cpu affinity: DragonFlyBSD support (#7956)

---
 src/config.h         | 2 +-
 src/setcpuaffinity.c | 9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/config.h b/src/config.h
index 320837b7e..b9c68f38b 100644
--- a/src/config.h
+++ b/src/config.h
@@ -253,7 +253,7 @@ int pthread_setname_np(const char *name);
 #endif
 
 /* Check if we can use setcpuaffinity(). */
-#if (defined __linux || defined __NetBSD__ || defined __FreeBSD__)
+#if (defined __linux || defined __NetBSD__ || defined __FreeBSD__ || defined __DragonFly__)
 #define USE_SETCPUAFFINITY
 void setcpuaffinity(const char *cpulist);
 #endif
diff --git a/src/setcpuaffinity.c b/src/setcpuaffinity.c
index 13594113c..77b162103 100644
--- a/src/setcpuaffinity.c
+++ b/src/setcpuaffinity.c
@@ -36,6 +36,10 @@
 #include <sys/param.h>
 #include <sys/cpuset.h>
 #endif
+#ifdef __DragonFly__
+#include <pthread.h>
+#include <pthread_np.h>
+#endif
 #ifdef __NetBSD__
 #include <pthread.h>
 #include <sched.h>
@@ -72,7 +76,7 @@ void setcpuaffinity(const char *cpulist) {
 #ifdef __linux__
     cpu_set_t cpuset;
 #endif
-#ifdef __FreeBSD__
+#if defined (__FreeBSD__) || defined(__DragonFly__)
     cpuset_t cpuset;
 #endif
 #ifdef __NetBSD__
@@ -139,6 +143,9 @@ void setcpuaffinity(const char *cpulist) {
 #ifdef __FreeBSD__
     cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, sizeof(cpuset), &cpuset);
 #endif
+#ifdef __DragonFly__
+    pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
+#endif
 #ifdef __NetBSD__
     pthread_setaffinity_np(pthread_self(), cpuset_size(cpuset), cpuset);
     cpuset_destroy(cpuset);

From d2af0f25be788ebe47b55ef82d494e0b95860f1f Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Mon, 26 Oct 2020 00:37:58 +0200
Subject: [PATCH 0750/1098] Adds command introspection to Sentinel (#7940)

Adds the `COMMAND` command to Sentinel.
---
 src/sentinel.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/sentinel.c b/src/sentinel.c
index 31a66a6ff..71e28e1d2 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -466,7 +466,8 @@ struct redisCommand sentinelcmds[] = {
     {"shutdown",shutdownCommand,-1,"admin",0,NULL,0,0,0,0,0},
     {"auth",authCommand,-2,"no-auth fast @connection",0,NULL,0,0,0,0,0},
     {"hello",helloCommand,-2,"no-auth fast @connection",0,NULL,0,0,0,0,0},
-    {"acl",aclCommand,-2,"admin",0,NULL,0,0,0,0,0,0}
+    {"acl",aclCommand,-2,"admin",0,NULL,0,0,0,0,0,0},
+    {"command",commandCommand,-1, "random @connection", 0,NULL,0,0,0,0,0,0}
 };
 
 /* This function overwrites a few normal Redis config default with Sentinel

From 01acfa71ca972a79d215160104fad5f8e6e824af Mon Sep 17 00:00:00 2001
From: filipe oliveira <filipecosta.90@gmail.com>
Date: Mon, 26 Oct 2020 06:04:59 +0000
Subject: [PATCH 0751/1098] redis-benchmark: add tests, --version, a minor bug
 fixes (#7947)

- add test suite coverage for redis-benchmark
- add --version (similar to what redis-cli has)
- fix bug sending more requests than intended when pipeline > 1.
- when done sending requests, avoid freeing client in the write handler, in theory before
  responses are received (probably dead code since the read handler will call clientDone first)

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/Makefile                          |   2 +-
 src/redis-benchmark.c                 |  29 +++++-
 tests/integration/redis-benchmark.tcl | 127 ++++++++++++++++++++++++++
 tests/support/benchmark.tcl           |   5 +
 tests/test_helper.tcl                 |   1 +
 5 files changed, 160 insertions(+), 4 deletions(-)
 create mode 100644 tests/integration/redis-benchmark.tcl
 create mode 100644 tests/support/benchmark.tcl

diff --git a/src/Makefile b/src/Makefile
index daa201d73..45ee66267 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -259,7 +259,7 @@ REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.
 REDIS_CLI_NAME=redis-cli$(PROG_SUFFIX)
 REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o
 REDIS_BENCHMARK_NAME=redis-benchmark$(PROG_SUFFIX)
-REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o monotonic.o
+REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o release.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o
 REDIS_CHECK_RDB_NAME=redis-check-rdb$(PROG_SUFFIX)
 REDIS_CHECK_AOF_NAME=redis-check-aof$(PROG_SUFFIX)
 
diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index c17178e85..328f08077 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -29,6 +29,7 @@
  */
 
 #include "fmacros.h"
+#include "version.h"
 
 #include <stdio.h>
 #include <string.h>
@@ -179,6 +180,8 @@ typedef struct redisConfig {
 } redisConfig;
 
 /* Prototypes */
+char *redisGitSHA1(void);
+char *redisGitDirty(void);
 static void writeHandler(aeEventLoop *el, int fd, void *privdata, int mask);
 static void createMissingClients(client c);
 static benchmarkThread *createBenchmarkThread(int index);
@@ -196,6 +199,20 @@ static void updateClusterSlotsConfiguration();
 int showThroughput(struct aeEventLoop *eventLoop, long long id,
                    void *clientData);
 
+static sds benchmarkVersion(void) {
+    sds version;
+    version = sdscatprintf(sdsempty(), "%s", REDIS_VERSION);
+
+    /* Add git commit and working tree status when available */
+    if (strtoll(redisGitSHA1(),NULL,16)) {
+        version = sdscatprintf(version, " (git:%s", redisGitSHA1());
+        if (strtoll(redisGitDirty(),NULL,10))
+            version = sdscatprintf(version, "-dirty");
+        version = sdscat(version, ")");
+    }
+    return version;
+}
+
 /* Dict callbacks */
 static uint64_t dictSdsHash(const void *key);
 static int dictSdsKeyCompare(void *privdata, const void *key1,
@@ -577,9 +594,8 @@ static void writeHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
     if (c->written == 0) {
         /* Enforce upper bound to number of requests. */
         int requests_issued = 0;
-        atomicGetIncr(config.requests_issued, requests_issued, 1);
+        atomicGetIncr(config.requests_issued, requests_issued, config.pipeline);
         if (requests_issued >= config.requests) {
-            freeClient(c);
             return;
         }
 
@@ -1364,6 +1380,11 @@ int parseOptions(int argc, const char **argv) {
         if (!strcmp(argv[i],"-c")) {
             if (lastarg) goto invalid;
             config.numclients = atoi(argv[++i]);
+        } else if (!strcmp(argv[i],"-v") || !strcmp(argv[i], "--version")) {
+            sds version = benchmarkVersion();
+            printf("redis-benchmark %s\n", version);
+            sdsfree(version);
+            exit(0);
         } else if (!strcmp(argv[i],"-n")) {
             if (lastarg) goto invalid;
             config.requests = atoi(argv[++i]);
@@ -1496,7 +1517,9 @@ usage:
 " -l                 Loop. Run the tests forever\n"
 " -t <tests>         Only run the comma separated list of tests. The test\n"
 "                    names are the same as the ones produced as output.\n"
-" -I                 Idle mode. Just open N idle connections and wait.\n\n"
+" -I                 Idle mode. Just open N idle connections and wait.\n"
+" --help             Output this help and exit.\n"
+" --version          Output version and exit.\n\n"
 "Examples:\n\n"
 " Run the benchmark with the default configuration against 127.0.0.1:6379:\n"
 "   $ redis-benchmark\n\n"
diff --git a/tests/integration/redis-benchmark.tcl b/tests/integration/redis-benchmark.tcl
new file mode 100644
index 000000000..8abeabf0b
--- /dev/null
+++ b/tests/integration/redis-benchmark.tcl
@@ -0,0 +1,127 @@
+source tests/support/benchmark.tcl
+
+
+proc cmdstat {cmd} {
+    return [cmdrstat $cmd r]
+}
+
+start_server {tags {"benchmark"}} {
+    start_server {} {
+        set master_host [srv 0 host]
+        set master_port [srv 0 port]
+
+        test {benchmark: set,get} {
+            r config resetstat
+            r flushall
+            set cmd [redisbenchmark $master_host $master_port "-c 5 -n 10 -e -t set,get"]
+            if {[catch { exec {*}$cmd } error]} {
+                set first_line [lindex [split $error "\n"] 0]
+                puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
+                fail "redis-benchmark non zero code. first line: $first_line"
+            }
+            assert_match  {*calls=10,*} [cmdstat set]
+            assert_match  {*calls=10,*} [cmdstat get]
+            # assert one of the non benchmarked commands is not present
+            assert_match  {} [cmdstat lrange]
+        }
+
+        test {benchmark: full test suite} {
+            r config resetstat
+            set cmd [redisbenchmark $master_host $master_port "-c 10 -n 100 -e"]
+            if {[catch { exec {*}$cmd } error]} {
+                set first_line [lindex [split $error "\n"] 0]
+                puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
+                fail "redis-benchmark non zero code. first line: $first_line"
+            }
+            # ping total calls are 2*issued commands per test due to PING_INLINE and PING_BULK
+            assert_match  {*calls=200,*} [cmdstat ping]
+            assert_match  {*calls=100,*} [cmdstat set]
+            assert_match  {*calls=100,*} [cmdstat get]
+            assert_match  {*calls=100,*} [cmdstat incr]
+            # lpush total calls are 2*issued commands per test due to the lrange tests
+            assert_match  {*calls=200,*} [cmdstat lpush]
+            assert_match  {*calls=100,*} [cmdstat rpush]
+            assert_match  {*calls=100,*} [cmdstat lpop]
+            assert_match  {*calls=100,*} [cmdstat rpop]
+            assert_match  {*calls=100,*} [cmdstat sadd]
+            assert_match  {*calls=100,*} [cmdstat hset]
+            assert_match  {*calls=100,*} [cmdstat spop]
+            assert_match  {*calls=100,*} [cmdstat zadd]
+            assert_match  {*calls=100,*} [cmdstat zpopmin]
+            assert_match  {*calls=400,*} [cmdstat lrange]
+            assert_match  {*calls=100,*} [cmdstat mset]
+            # assert one of the non benchmarked commands is not present
+            assert_match {} [cmdstat rpoplpush]
+        }
+
+        test {benchmark: multi-thread set,get} {
+            r config resetstat
+            r flushall
+            set cmd [redisbenchmark $master_host $master_port "--threads 10 -c 5 -n 10 -e -t set,get"]
+            if {[catch { exec {*}$cmd } error]} {
+                set first_line [lindex [split $error "\n"] 0]
+                puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
+                fail "redis-benchmark non zero code. first line: $first_line"
+            }
+            assert_match  {*calls=10,*} [cmdstat set]
+            assert_match  {*calls=10,*} [cmdstat get]
+            # assert one of the non benchmarked commands is not present
+            assert_match  {} [cmdstat lrange]
+
+            # ensure only one key was populated
+            assert_match  {1} [scan [regexp -inline {keys\=([\d]*)} [r info keyspace]] keys=%d]
+        }
+
+        test {benchmark: pipelined full set,get} {
+            r config resetstat
+            r flushall
+            set cmd [redisbenchmark $master_host $master_port "-P 5 -c 10 -n 10010 -e -t set,get"]
+            if {[catch { exec {*}$cmd } error]} {
+                set first_line [lindex [split $error "\n"] 0]
+                puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
+                fail "redis-benchmark non zero code. first line: $first_line"
+            }
+            assert_match  {*calls=10010,*} [cmdstat set]
+            assert_match  {*calls=10010,*} [cmdstat get]
+            # assert one of the non benchmarked commands is not present
+            assert_match  {} [cmdstat lrange]
+
+            # ensure only one key was populated
+            assert_match  {1} [scan [regexp -inline {keys\=([\d]*)} [r info keyspace]] keys=%d]
+        }
+
+        test {benchmark: arbitrary command} {
+            r config resetstat
+            r flushall
+            set cmd [redisbenchmark $master_host $master_port "-c 5 -n 150 -e INCRBYFLOAT mykey 10.0"]
+            if {[catch { exec {*}$cmd } error]} {
+                set first_line [lindex [split $error "\n"] 0]
+                puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
+                fail "redis-benchmark non zero code. first line: $first_line"
+            }
+            assert_match  {*calls=150,*} [cmdstat incrbyfloat]
+            # assert one of the non benchmarked commands is not present
+            assert_match  {} [cmdstat get]
+
+            # ensure only one key was populated
+            assert_match  {1} [scan [regexp -inline {keys\=([\d]*)} [r info keyspace]] keys=%d]
+        }
+
+        test {benchmark: keyspace length} {
+            r flushall
+            r config resetstat
+            set cmd [redisbenchmark $master_host $master_port "-r 50 -t set -n 1000"]
+            if {[catch { exec {*}$cmd } error]} {
+                set first_line [lindex [split $error "\n"] 0]
+                puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
+                fail "redis-benchmark non zero code. first line: $first_line"
+            }
+            assert_match  {*calls=1000,*} [cmdstat set]
+            # assert one of the non benchmarked commands is not present
+            assert_match  {} [cmdstat get]
+
+            # ensure the keyspace has the desired size
+            assert_match  {50} [scan [regexp -inline {keys\=([\d]*)} [r info keyspace]] keys=%d]
+        }
+    }
+}
diff --git a/tests/support/benchmark.tcl b/tests/support/benchmark.tcl
new file mode 100644
index 000000000..1130c9992
--- /dev/null
+++ b/tests/support/benchmark.tcl
@@ -0,0 +1,5 @@
+proc redisbenchmark {host port {opts {}}} {
+    set cmd [list src/redis-benchmark -h $host -p $port]
+    lappend cmd {*}$opts
+    return $cmd
+}
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 37af46947..37ab23d0c 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -50,6 +50,7 @@ set ::all_tests {
     integration/psync2-reg
     integration/psync2-pingoff
     integration/redis-cli
+    integration/redis-benchmark
     unit/pubsub
     unit/slowlog
     unit/scripting

From 4e2e5be201439cae4c0a03cfc8b6a60be4bff625 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 26 Oct 2020 11:55:24 +0200
Subject: [PATCH 0752/1098] Attempt to fix sporadic test failures due to
 wait_for_log_messages (#7955)

The tests sometimes fail to find a log message.
Recently i added a print that shows the log files that are searched
and it shows that the message was in deed there.
The only reason i can't think of for this seach to fail, is we we
happened to read an incomplete line, which didn't match our pattern and
then on the next iteration we would continue reading from the line after
it.

The fix is to always re-evaluation the previous line.
---
 tests/support/util.tcl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index b4c1a3841..3268bc974 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -128,6 +128,8 @@ proc wait_for_log_messages {srv_idx patterns from_line maxtries delay} {
     set next_line [expr $from_line + 1] ;# searching form the line after
     set stdout [srv $srv_idx stdout]
     while {$retry} {
+        # re-read the last line (unless it's before to our first), last time we read it, it might have been incomplete
+        set next_line [expr $next_line - 1 > $from_line + 1 ? $next_line - 1 : $from_line + 1]
         set result [exec tail -n +$next_line < $stdout]
         set result [split $result "\n"]
         foreach line $result {

From 9824fe3e392caa04dc1b4071886e9ac402dd6d95 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 26 Oct 2020 14:49:08 +0200
Subject: [PATCH 0753/1098] Fix wrong zmalloc_size() assumption. (#7963)

When using a system with no malloc_usable_size(), zmalloc_size() assumed
that the heap allocator always returns blocks that are long-padded.

This may not always be the case, and will result with zmalloc_size()
returning a size that is bigger than allocated. At least in one case
this leads to out of bound write, process crash and a potential security
vulnerability.

Effectively this does not affect the vast majority of users, who use
jemalloc or glibc.

This problem along with a (different) fix was reported by Drew DeVault.
---
 src/zmalloc.c | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/zmalloc.c b/src/zmalloc.c
index 4a7f2028e..565376721 100644
--- a/src/zmalloc.c
+++ b/src/zmalloc.c
@@ -236,9 +236,6 @@ void *zrealloc_usable(void *ptr, size_t size, size_t *usable) {
 size_t zmalloc_size(void *ptr) {
     void *realptr = (char*)ptr-PREFIX_SIZE;
     size_t size = *((size_t*)realptr);
-    /* Assume at least that all the allocations are padded at sizeof(long) by
-     * the underlying allocator. */
-    if (size&(sizeof(long)-1)) size += sizeof(long)-(size&(sizeof(long)-1));
     return size+PREFIX_SIZE;
 }
 size_t zmalloc_usable_size(void *ptr) {

From 4d1120f5fd5e74eaaf883b0233d3055a19cd56a9 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <madelyneolson@gmail.com>
Date: Mon, 19 Oct 2020 17:26:57 -0700
Subject: [PATCH 0754/1098] Update CI so that warnings cause build failures

---
 .github/workflows/ci.yml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 68593717f..8da286152 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -9,8 +9,9 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: make
-      # build with TLS just for compilatoin coverage
-      run: make BUILD_TLS=yes
+      # Fail build if there are warnings
+      # build with TLS just for compilation coverage
+      run: make REDIS_CFLAGS='-Werror' BUILD_TLS=yes
     - name: test
       run: |
         sudo apt-get install tcl8.5
@@ -30,7 +31,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: make
-      run: make
+      run: make REDIS_CFLAGS='-Werror'
 
   build-32bit:
     runs-on: ubuntu-latest
@@ -39,14 +40,14 @@ jobs:
     - name: make
       run: |
         sudo apt-get update && sudo apt-get install libc6-dev-i386
-        make 32bit
+        make REDIS_CFLAGS='-Werror' 32bit
 
   build-libc-malloc:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
     - name: make
-      run: make MALLOC=libc
+      run: make REDIS_CFLAGS='-Werror' MALLOC=libc
 
   build-centos7-jemalloc:
     runs-on: ubuntu-latest
@@ -56,7 +57,7 @@ jobs:
     - name: make
       run: |
         yum -y install gcc make
-        make
+        make REDIS_CFLAGS='-Werror'
 
   build-centos6-jemalloc:
     runs-on: ubuntu-latest

From dac26729a9259754eda0403cd32f011a3d2a89e1 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <madelyneolson@gmail.com>
Date: Fri, 23 Oct 2020 10:49:15 -0700
Subject: [PATCH 0755/1098] Only supress implitic fallthrough on GCC 7

---
 .github/workflows/ci.yml | 2 +-
 src/lzf_d.c              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8da286152..be2be1789 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,7 +24,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: make
-      run: make
+      run: make REDIS_CFLAGS='-Werror'
 
   build-macos-latest:
     runs-on: macos-latest
diff --git a/src/lzf_d.c b/src/lzf_d.c
index d44bfcc8d..5f55d471d 100644
--- a/src/lzf_d.c
+++ b/src/lzf_d.c
@@ -52,7 +52,7 @@
 #endif
 #endif
 
-#if defined(__GNUC__) && __GNUC__ >= 5
+#if defined(__GNUC__) && __GNUC__ >= 7
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
 #endif

From 7fa56dd773cac7155f7a7768458b1f59d1f39a13 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Tue, 27 Oct 2020 14:13:59 +0800
Subject: [PATCH 0756/1098] Speedup cluster failover. (#7948)

This commit deals with manual failover as well as non-manual failover.

We did tests with manual failover as follows:
1, Setup redis cluster which holds 16 partions, each having only
   1 corresponding replica.
2, Write a batch of data to redis cluster and make sure the redis is doing
   a active expire in serverCron.
3, Do a manual failover sequentially to each partions with a time interval
   of 3 minutes.
4, Collect logs and do some computaiton work.

The result:
case    avgTime    maxTime    minTime
C1      95.8ms	   227ms      25ms
C2      47.9ms     96ms       12ms
C3      12.6ms     27ms       7ms

Explanation
case C1: All nodes use the version before optimization
case C2: Masters use the elder version while replicas use the optimized version
case C3: All nodes use the optimized version
failover time: The time between when replica got a `manual failover request` and
               when it `won the failover election`.
avgTime: average failover time
maxTime: maximum failover time
minTime: mimimum failover time
ms: millisecond

Co-authored-by: chendq8 <c.d_q@163.com>
---
 src/cluster.c | 50 +++++++++++++++++++++++++++++++++++---------------
 src/cluster.h |  1 +
 2 files changed, 36 insertions(+), 15 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 248d5c17d..6da71c5da 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -1816,6 +1816,7 @@ int clusterProcessPacket(clusterLink *link) {
             server.cluster->mf_master_offset == 0)
         {
             server.cluster->mf_master_offset = sender->repl_offset;
+            clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_MANUALFAILOVER);
             serverLog(LL_WARNING,
                 "Received replication offset for paused "
                 "master manual failover: %lld",
@@ -2160,6 +2161,12 @@ int clusterProcessPacket(clusterLink *link) {
         pauseClients(now+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT));
         serverLog(LL_WARNING,"Manual failover requested by replica %.40s.",
             sender->name);
+        /* We need to send a ping message to the replica, as it would carry
+         * `server.cluster->mf_master_offset`, which means the master paused clients
+         * at offset `server.cluster->mf_master_offset`, so that the replica would
+         * know that it is safe to set its `server.cluster->mf_can_start` to 1 so as
+         * to complete failover as quickly as possible. */
+        clusterSendPing(link, CLUSTERMSG_TYPE_PING);
     } else if (type == CLUSTERMSG_TYPE_UPDATE) {
         clusterNode *n; /* The node the update is about. */
         uint64_t reportedConfigEpoch =
@@ -3434,7 +3441,10 @@ void clusterHandleManualFailover(void) {
         serverLog(LL_WARNING,
             "All master replication stream processed, "
             "manual failover can start.");
+        clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_FAILOVER);
+        return;
     }
+    clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_MANUALFAILOVER);
 }
 
 /* -----------------------------------------------------------------------------
@@ -3709,25 +3719,35 @@ void clusterCron(void) {
  * handlers, or to perform potentially expansive tasks that we need to do
  * a single time before replying to clients. */
 void clusterBeforeSleep(void) {
-    /* Handle failover, this is needed when it is likely that there is already
-     * the quorum from masters in order to react fast. */
-    if (server.cluster->todo_before_sleep & CLUSTER_TODO_HANDLE_FAILOVER)
-        clusterHandleSlaveFailover();
-
-    /* Update the cluster state. */
-    if (server.cluster->todo_before_sleep & CLUSTER_TODO_UPDATE_STATE)
-        clusterUpdateState();
-
-    /* Save the config, possibly using fsync. */
-    if (server.cluster->todo_before_sleep & CLUSTER_TODO_SAVE_CONFIG) {
-        int fsync = server.cluster->todo_before_sleep &
-                    CLUSTER_TODO_FSYNC_CONFIG;
-        clusterSaveConfigOrDie(fsync);
-    }
+    int flags = server.cluster->todo_before_sleep;
 
     /* Reset our flags (not strictly needed since every single function
      * called for flags set should be able to clear its flag). */
     server.cluster->todo_before_sleep = 0;
+
+    if (flags & CLUSTER_TODO_HANDLE_MANUALFAILOVER) {
+        /* Handle manual failover as soon as possible so that won't have a 100ms
+         * as it was handled only in clusterCron */
+        if(nodeIsSlave(myself)) {
+            clusterHandleManualFailover();
+            if (!(server.cluster_module_flags & CLUSTER_MODULE_FLAG_NO_FAILOVER))
+                clusterHandleSlaveFailover();
+        }
+    } else if (flags & CLUSTER_TODO_HANDLE_FAILOVER) {
+        /* Handle failover, this is needed when it is likely that there is already
+         * the quorum from masters in order to react fast. */
+        clusterHandleSlaveFailover();
+    }
+
+    /* Update the cluster state. */
+    if (flags & CLUSTER_TODO_UPDATE_STATE)
+        clusterUpdateState();
+
+    /* Save the config, possibly using fsync. */
+    if (flags & CLUSTER_TODO_SAVE_CONFIG) {
+        int fsync = flags & CLUSTER_TODO_FSYNC_CONFIG;
+        clusterSaveConfigOrDie(fsync);
+    }
 }
 
 void clusterDoBeforeSleep(int flags) {
diff --git a/src/cluster.h b/src/cluster.h
index 48a111764..95b710383 100644
--- a/src/cluster.h
+++ b/src/cluster.h
@@ -77,6 +77,7 @@ typedef struct clusterLink {
 #define CLUSTER_TODO_UPDATE_STATE (1<<1)
 #define CLUSTER_TODO_SAVE_CONFIG (1<<2)
 #define CLUSTER_TODO_FSYNC_CONFIG (1<<3)
+#define CLUSTER_TODO_HANDLE_MANUALFAILOVER (1<<4)
 
 /* Message types.
  *

From dc899c4c883faa04477acf33f8facf8a4ef26e4a Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuancode@163.com>
Date: Tue, 27 Oct 2020 15:36:42 +0800
Subject: [PATCH 0757/1098] Fix timing dependence in replication tcl tests
 (#7969)

Remove 'fork child $pid' log in replication.tcl
---
 tests/integration/replication.tcl | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl
index f7f8d241b..fb0b9b965 100644
--- a/tests/integration/replication.tcl
+++ b/tests/integration/replication.tcl
@@ -725,7 +725,6 @@ test "diskless replication child being killed is collected" {
 
             # simulate the OOM killer or anyone else kills the child
             set fork_child_pid [get_child_pid -1]
-            puts "fork child is $fork_child_pid"
             exec kill -9 $fork_child_pid
 
             # wait for the parent to notice the child have exited
@@ -812,7 +811,9 @@ test {Kill rdb child process if its dumping RDB is not useful} {
 
                 # Wait for starting child
                 wait_for_condition 50 100 {
-                    [s 0 rdb_bgsave_in_progress] == 1
+                    ([s 0 rdb_bgsave_in_progress] == 1) &&
+                    ([string match "*wait_bgsave*" [s 0 slave0]]) &&
+                    ([string match "*wait_bgsave*" [s 0 slave1]])
                 } else {
                     fail "rdb child didn't start"
                 }
@@ -837,7 +838,9 @@ test {Kill rdb child process if its dumping RDB is not useful} {
                 $slave1 slaveof $master_host $master_port
                 $slave2 slaveof $master_host $master_port
                 wait_for_condition 50 100 {
-                    [s 0 rdb_bgsave_in_progress] == 1
+                    ([s 0 rdb_bgsave_in_progress] == 1) &&
+                    ([string match "*wait_bgsave*" [s 0 slave0]]) &&
+                    ([string match "*wait_bgsave*" [s 0 slave1]])
                 } else {
                     fail "rdb child didn't start"
                 }

From 825fe7bd23051d804ebbe2f4be5ad67a7e4769bb Mon Sep 17 00:00:00 2001
From: Andrij Fedyk <1037995+fedyk@users.noreply.github.com>
Date: Tue, 27 Oct 2020 10:27:27 +0100
Subject: [PATCH 0758/1098] rdb.c: fix typo in a comment (#7970)

---
 src/rdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rdb.c b/src/rdb.c
index e94a79125..9301cdf71 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -43,7 +43,7 @@
 #include <sys/stat.h>
 #include <sys/param.h>
 
-/* This macro is called when the internal RDB stracture is corrupt */
+/* This macro is called when the internal RDB structure is corrupt */
 #define rdbExitReportCorruptRDB(...) rdbReportError(1, __LINE__,__VA_ARGS__)
 /* This macro is called when RDB read failed (possibly a short read) */
 #define rdbReportReadError(...) rdbReportError(0, __LINE__,__VA_ARGS__)

From a9c06021498444b8d785066a2fae7d49693721dc Mon Sep 17 00:00:00 2001
From: zhenwei pi <pizhenwei@bytedance.com>
Date: Tue, 27 Oct 2020 21:04:18 +0800
Subject: [PATCH 0759/1098] Disable THP if enabled (#7381)

In case redis starts and find that THP is enabled ("always"), instead
of printing a log message, which might go unnoticed, redis will try to
disable it (just for the redis process).

Note: it looks like on self-bulit kernels THP is likely be set to "always" by default.

Some discuss about THP side effect on Linux:
according to http://www.antirez.com/news/84, we can see that
redis latency spikes are caused by linux kernel THP feature.
I have tested on E3-2650 v3, and found that 2M huge page costs
about 0.25ms to fix COW page fault.

Add a new config 'disable-thp', the recommended setting is 'yes',
(default) the redis tries to disable THP by prctl syscall. But
users who really want THP can set it to "no"

Thanks to Oran & Yossi for suggestions.

Signed-off-by: zhenwei pi <pizhenwei@bytedance.com>
---
 redis.conf    | 13 +++++++++++++
 src/config.c  |  1 +
 src/latency.c | 16 ++++++++++++++++
 src/latency.h |  1 +
 src/server.c  |  2 +-
 src/server.h  |  1 +
 6 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/redis.conf b/redis.conf
index 561aaa9db..119f7b6f9 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1088,6 +1088,19 @@ oom-score-adj no
 
 oom-score-adj-values 0 200 800
 
+
+#################### KERNEL transparent hugepage CONTROL ######################
+
+# Usually the kernel Transparent Huge Pages control is set to "madvise" or
+# or "never" by default (/sys/kernel/mm/transparent_hugepage/enabled), in which
+# case this config has no effect. On systems in which it is set to "always",
+# redis will attempt to disable it specifically for the redis process in order
+# to avoid latency problems specifically with fork(2) and CoW.
+# If for some reason you prefer to keep it enabled, you can set this config to
+# "no" and the kernel global to "always".
+
+disable-thp yes
+
 ############################## APPEND ONLY MODE ###############################
 
 # By default Redis asynchronously dumps the dataset on disk. This mode is
diff --git a/src/config.c b/src/config.c
index aaee2db65..07ed91e5f 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2314,6 +2314,7 @@ standardConfig configs[] = {
     createBoolConfig("crash-memcheck-enabled", NULL, MODIFIABLE_CONFIG, server.memcheck_enabled, 1, NULL, NULL),
     createBoolConfig("use-exit-on-panic", NULL, MODIFIABLE_CONFIG, server.use_exit_on_panic, 0, NULL, NULL),
     createBoolConfig("oom-score-adj", NULL, MODIFIABLE_CONFIG, server.oom_score_adj, 0, NULL, updateOOMScoreAdj),
+    createBoolConfig("disable-thp", NULL, MODIFIABLE_CONFIG, server.disable_thp, 1, NULL, NULL),
 
     /* String Configs */
     createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, "", NULL, NULL),
diff --git a/src/latency.c b/src/latency.c
index 6148543c8..c661c5d6d 100644
--- a/src/latency.c
+++ b/src/latency.c
@@ -59,6 +59,7 @@ dictType latencyTimeSeriesDictType = {
 /* ------------------------- Utility functions ------------------------------ */
 
 #ifdef __linux__
+#include <sys/prctl.h>
 /* Returns 1 if Transparent Huge Pages support is enabled in the kernel.
  * Otherwise (or if we are unable to check) 0 is returned. */
 int THPIsEnabled(void) {
@@ -73,6 +74,21 @@ int THPIsEnabled(void) {
     fclose(fp);
     return (strstr(buf,"[always]") != NULL) ? 1 : 0;
 }
+
+/* since linux-3.5, kernel supports to set the state of the "THP disable" flag
+ * for the calling thread. PR_SET_THP_DISABLE is defined in linux/prctl.h */
+int THPDisable(void) {
+    int ret = -EINVAL;
+
+    if (!server.disable_thp)
+        return ret;
+
+#ifdef PR_SET_THP_DISABLE
+    ret = prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0);
+#endif
+
+    return ret;
+}
 #endif
 
 /* Report the amount of AnonHugePages in smap, in bytes. If the return
diff --git a/src/latency.h b/src/latency.h
index 76640cfce..b17f403a5 100644
--- a/src/latency.h
+++ b/src/latency.h
@@ -64,6 +64,7 @@ struct latencyStats {
 void latencyMonitorInit(void);
 void latencyAddSample(const char *event, mstime_t latency);
 int THPIsEnabled(void);
+int THPDisable(void);
 
 /* Latency monitoring macros. */
 
diff --git a/src/server.c b/src/server.c
index 67d050ea0..093c20630 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4874,7 +4874,7 @@ void linuxMemoryWarnings(void) {
     if (linuxOvercommitMemoryValue() == 0) {
         serverLog(LL_WARNING,"WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect.");
     }
-    if (THPIsEnabled()) {
+    if (THPIsEnabled() && THPDisable()) {
         serverLog(LL_WARNING,"WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo madvise > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled (set to 'madvise' or 'never').");
     }
 }
diff --git a/src/server.h b/src/server.h
index 5d53d9473..b68d06497 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1381,6 +1381,7 @@ struct redisServer {
     int oom_score_adj_base;         /* Base oom_score_adj value, as observed on startup */
     int oom_score_adj_values[CONFIG_OOM_COUNT];   /* Linux oom_score_adj configuration */
     int oom_score_adj;                            /* If true, oom_score_adj is managed */
+    int disable_thp;                              /* If true, disable THP by syscall */
     /* Blocked clients */
     unsigned int blocked_clients;   /* # of clients executing a blocking cmd.*/
     unsigned int blocked_clients_by_type[BLOCKED_NUM];

From 380f6048e0bbc762f12fa50b57b73cf29049f967 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 27 Oct 2020 16:36:00 +0200
Subject: [PATCH 0760/1098] Fix cluster access to unaligned memory (SIGBUS on
 old ARM) (#7958)

Turns out this was broken since version 4.0 when we added sds size
classes.
The cluster code uses sds for the receive buffer, and then casts it to a
struct and accesses a 64 bit variable.
This commit replaces the use of sds with a simple reallocated buffer.
---
 src/cluster.c | 29 ++++++++++++++++++++++-------
 src/cluster.h |  4 +++-
 2 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 6da71c5da..6116614ea 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -77,6 +77,9 @@ uint64_t clusterGetMaxEpoch(void);
 int clusterBumpConfigEpochWithoutConsensus(void);
 void moduleCallClusterReceivers(const char *sender_id, uint64_t module_id, uint8_t type, const unsigned char *payload, uint32_t len);
 
+#define RCVBUF_INIT_LEN 1024
+#define RCVBUF_MAX_PREALLOC (1<<20) /* 1MB */
+
 /* -----------------------------------------------------------------------------
  * Initialization
  * -------------------------------------------------------------------------- */
@@ -613,7 +616,8 @@ clusterLink *createClusterLink(clusterNode *node) {
     clusterLink *link = zmalloc(sizeof(*link));
     link->ctime = mstime();
     link->sndbuf = sdsempty();
-    link->rcvbuf = sdsempty();
+    link->rcvbuf = zmalloc(link->rcvbuf_alloc = RCVBUF_INIT_LEN);
+    link->rcvbuf_len = 0;
     link->node = node;
     link->conn = NULL;
     return link;
@@ -628,7 +632,7 @@ void freeClusterLink(clusterLink *link) {
         link->conn = NULL;
     }
     sdsfree(link->sndbuf);
-    sdsfree(link->rcvbuf);
+    zfree(link->rcvbuf);
     if (link->node)
         link->node->link = NULL;
     zfree(link);
@@ -1728,7 +1732,7 @@ int clusterProcessPacket(clusterLink *link) {
 
     /* Perform sanity checks */
     if (totlen < 16) return 1; /* At least signature, version, totlen, count. */
-    if (totlen > sdslen(link->rcvbuf)) return 1;
+    if (totlen > link->rcvbuf_len) return 1;
 
     if (ntohs(hdr->ver) != CLUSTER_PROTO_VER) {
         /* Can't handle messages of different versions. */
@@ -2289,7 +2293,7 @@ void clusterReadHandler(connection *conn) {
     unsigned int readlen, rcvbuflen;
 
     while(1) { /* Read as long as there is data to read. */
-        rcvbuflen = sdslen(link->rcvbuf);
+        rcvbuflen = link->rcvbuf_len;
         if (rcvbuflen < 8) {
             /* First, obtain the first 8 bytes to get the full message
              * length. */
@@ -2325,7 +2329,15 @@ void clusterReadHandler(connection *conn) {
             return;
         } else {
             /* Read data and recast the pointer to the new buffer. */
-            link->rcvbuf = sdscatlen(link->rcvbuf,buf,nread);
+            size_t unused = link->rcvbuf_alloc - link->rcvbuf_len;
+            if ((size_t)nread > unused) {
+                size_t required = link->rcvbuf_len + nread;
+                /* If less than 1mb, grow to twice the needed size, if larger grow by 1mb. */
+                link->rcvbuf_alloc = required < RCVBUF_MAX_PREALLOC ? required * 2: required + RCVBUF_MAX_PREALLOC;
+                link->rcvbuf = zrealloc(link->rcvbuf, link->rcvbuf_alloc);
+            }
+            memcpy(link->rcvbuf + link->rcvbuf_len, buf, nread);
+            link->rcvbuf_len += nread;
             hdr = (clusterMsg*) link->rcvbuf;
             rcvbuflen += nread;
         }
@@ -2333,8 +2345,11 @@ void clusterReadHandler(connection *conn) {
         /* Total length obtained? Process this packet. */
         if (rcvbuflen >= 8 && rcvbuflen == ntohl(hdr->totlen)) {
             if (clusterProcessPacket(link)) {
-                sdsfree(link->rcvbuf);
-                link->rcvbuf = sdsempty();
+                if (link->rcvbuf_alloc > RCVBUF_INIT_LEN) {
+                    zfree(link->rcvbuf);
+                    link->rcvbuf = zmalloc(link->rcvbuf_alloc = RCVBUF_INIT_LEN);
+                }
+                link->rcvbuf_len = 0;
             } else {
                 return; /* Link no longer valid. */
             }
diff --git a/src/cluster.h b/src/cluster.h
index 95b710383..7e5f79c87 100644
--- a/src/cluster.h
+++ b/src/cluster.h
@@ -38,7 +38,9 @@ typedef struct clusterLink {
     mstime_t ctime;             /* Link creation time */
     connection *conn;           /* Connection to remote node */
     sds sndbuf;                 /* Packet send buffer */
-    sds rcvbuf;                 /* Packet reception buffer */
+    char *rcvbuf;               /* Packet reception buffer */
+    size_t rcvbuf_len;          /* Used size of rcvbuf */
+    size_t rcvbuf_alloc;        /* Used size of rcvbuf */
     struct clusterNode *node;   /* Node related to this link if any, or NULL */
 } clusterLink;
 

From 66037309c63c53a28c9553070a4557fabb84ed5a Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Tue, 27 Oct 2020 13:02:06 +0800
Subject: [PATCH 0761/1098] Fix waste of CPU time about server log in
 serverCron.

When all the work is just adding logs, we could pull
the condition out so as to use less CPU time when
loglevel is bigger than LL_VERBOSE.
---
 src/server.c | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/server.c b/src/server.c
index 093c20630..bb035f795 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1968,16 +1968,17 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
     }
 
     /* Show some info about non-empty databases */
-    run_with_period(5000) {
-        for (j = 0; j < server.dbnum; j++) {
-            long long size, used, vkeys;
+    if (server.verbosity <= LL_VERBOSE) {
+        run_with_period(5000) {
+            for (j = 0; j < server.dbnum; j++) {
+                long long size, used, vkeys;
 
-            size = dictSlots(server.db[j].dict);
-            used = dictSize(server.db[j].dict);
-            vkeys = dictSize(server.db[j].expires);
-            if (used || vkeys) {
-                serverLog(LL_VERBOSE,"DB %d: %lld keys (%lld volatile) in %lld slots HT.",j,used,vkeys,size);
-                /* dictPrintStats(server.dict); */
+                size = dictSlots(server.db[j].dict);
+                used = dictSize(server.db[j].dict);
+                vkeys = dictSize(server.db[j].expires);
+                if (used || vkeys) {
+                    serverLog(LL_VERBOSE,"DB %d: %lld keys (%lld volatile) in %lld slots HT.",j,used,vkeys,size);
+                }
             }
         }
     }

From 39436b21527d878f23eef0bd64edbff47af073c5 Mon Sep 17 00:00:00 2001
From: filipe oliveira <filipecosta.90@gmail.com>
Date: Wed, 28 Oct 2020 06:00:54 +0000
Subject: [PATCH 0762/1098] TLS Support for redis-benchmark (#7959)

---
 src/Makefile                |   4 +-
 src/cli_common.c            | 185 ++++++++++++++++++++++++++++++++++++
 src/cli_common.h            |  44 +++++++++
 src/redis-benchmark.c       |  92 +++++++++++++++---
 src/redis-cli.c             | 162 +++----------------------------
 tests/support/benchmark.tcl |  14 +++
 6 files changed, 337 insertions(+), 164 deletions(-)
 create mode 100644 src/cli_common.c
 create mode 100644 src/cli_common.h

diff --git a/src/Makefile b/src/Makefile
index 45ee66267..a32e4c150 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -257,9 +257,9 @@ REDIS_SERVER_NAME=redis-server$(PROG_SUFFIX)
 REDIS_SENTINEL_NAME=redis-sentinel$(PROG_SUFFIX)
 REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o
 REDIS_CLI_NAME=redis-cli$(PROG_SUFFIX)
-REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o
+REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o
 REDIS_BENCHMARK_NAME=redis-benchmark$(PROG_SUFFIX)
-REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o release.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o
+REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o release.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o
 REDIS_CHECK_RDB_NAME=redis-check-rdb$(PROG_SUFFIX)
 REDIS_CHECK_AOF_NAME=redis-check-aof$(PROG_SUFFIX)
 
diff --git a/src/cli_common.c b/src/cli_common.c
new file mode 100644
index 000000000..c45e3de96
--- /dev/null
+++ b/src/cli_common.c
@@ -0,0 +1,185 @@
+/* CLI (command line interface) common methods
+ * 
+ * Copyright (c) 2020, Redis Labs
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "cli_common.h"
+#include <errno.h>
+#include <hiredis.h>
+#include <sdscompat.h> /* Use hiredis' sds compat header that maps sds calls to their hi_ variants */
+#include <sds.h> /* use sds.h from hiredis, so that only one set of sds functions will be present in the binary */
+#ifdef USE_OPENSSL
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+#include <hiredis_ssl.h>
+#endif
+
+
+/* Wrapper around redisSecureConnection to avoid hiredis_ssl dependencies if
+ * not building with TLS support.
+ */
+int cliSecureConnection(redisContext *c, cliSSLconfig config, const char **err) {
+#ifdef USE_OPENSSL
+    static SSL_CTX *ssl_ctx = NULL;
+
+    if (!ssl_ctx) {
+        ssl_ctx = SSL_CTX_new(SSLv23_client_method());
+        if (!ssl_ctx) {
+            *err = "Failed to create SSL_CTX";
+            goto error;
+        }
+
+        SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3);
+        SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL);
+
+        if (config.cacert || config.cacertdir) {
+            if (!SSL_CTX_load_verify_locations(ssl_ctx, config.cacert, config.cacertdir)) {
+                *err = "Invalid CA Certificate File/Directory";
+                goto error;
+            }
+        } else {
+            if (!SSL_CTX_set_default_verify_paths(ssl_ctx)) {
+                *err = "Failed to use default CA paths";
+                goto error;
+            }
+        }
+
+        if (config.cert && !SSL_CTX_use_certificate_chain_file(ssl_ctx, config.cert)) {
+            *err = "Invalid client certificate";
+            goto error;
+        }
+
+        if (config.key && !SSL_CTX_use_PrivateKey_file(ssl_ctx, config.key, SSL_FILETYPE_PEM)) {
+            *err = "Invalid private key";
+            goto error;
+        }
+    }
+
+    SSL *ssl = SSL_new(ssl_ctx);
+    if (!ssl) {
+        *err = "Failed to create SSL object";
+        return REDIS_ERR;
+    }
+
+    if (config.sni && !SSL_set_tlsext_host_name(ssl, config.sni)) {
+        *err = "Failed to configure SNI";
+        SSL_free(ssl);
+        return REDIS_ERR;
+    }
+
+    return redisInitiateSSL(c, ssl);
+
+error:
+    SSL_CTX_free(ssl_ctx);
+    ssl_ctx = NULL;
+    return REDIS_ERR;
+#else
+    (void) config;
+    (void) c;
+    (void) err;
+    return REDIS_OK;
+#endif
+}
+
+/* Wrapper around hiredis to allow arbitrary reads and writes.
+ *
+ * We piggybacks on top of hiredis to achieve transparent TLS support,
+ * and use its internal buffers so it can co-exist with commands
+ * previously/later issued on the connection.
+ *
+ * Interface is close to enough to read()/write() so things should mostly
+ * work transparently.
+ */
+
+/* Write a raw buffer through a redisContext. If we already have something
+ * in the buffer (leftovers from hiredis operations) it will be written
+ * as well.
+ */
+ssize_t cliWriteConn(redisContext *c, const char *buf, size_t buf_len)
+{
+    int done = 0;
+
+    /* Append data to buffer which is *usually* expected to be empty
+     * but we don't assume that, and write.
+     */
+    c->obuf = sdscatlen(c->obuf, buf, buf_len);
+    if (redisBufferWrite(c, &done) == REDIS_ERR) {
+        if (!(c->flags & REDIS_BLOCK))
+            errno = EAGAIN;
+
+        /* On error, we assume nothing was written and we roll back the
+         * buffer to its original state.
+         */
+        if (sdslen(c->obuf) > buf_len)
+            sdsrange(c->obuf, 0, -(buf_len+1));
+        else
+            sdsclear(c->obuf);
+
+        return -1;
+    }
+
+    /* If we're done, free up everything. We may have written more than
+     * buf_len (if c->obuf was not initially empty) but we don't have to
+     * tell.
+     */
+    if (done) {
+        sdsclear(c->obuf);
+        return buf_len;
+    }
+
+    /* Write was successful but we have some leftovers which we should
+     * remove from the buffer.
+     *
+     * Do we still have data that was there prior to our buf? If so,
+     * restore buffer to it's original state and report no new data was
+     * writen.
+     */
+    if (sdslen(c->obuf) > buf_len) {
+        sdsrange(c->obuf, 0, -(buf_len+1));
+        return 0;
+    }
+
+    /* At this point we're sure no prior data is left. We flush the buffer
+     * and report how much we've written.
+     */
+    size_t left = sdslen(c->obuf);
+    sdsclear(c->obuf);
+    return buf_len - left;
+}
+
+/* Wrapper around OpenSSL (libssl and libcrypto) initialisation
+ */
+int cliSecureInit()
+{
+#ifdef USE_OPENSSL
+    ERR_load_crypto_strings();
+    SSL_load_error_strings();
+    SSL_library_init();
+#endif
+    return REDIS_OK;
+}
diff --git a/src/cli_common.h b/src/cli_common.h
new file mode 100644
index 000000000..700a834ce
--- /dev/null
+++ b/src/cli_common.h
@@ -0,0 +1,44 @@
+#ifndef __CLICOMMON_H
+#define __CLICOMMON_H
+
+#include <hiredis.h>
+
+typedef struct cliSSLconfig {
+    /* Requested SNI, or NULL */
+    char *sni;
+    /* CA Certificate file, or NULL */
+    char *cacert;
+    /* Directory where trusted CA certificates are stored, or NULL */
+    char *cacertdir;
+    /* Client certificate to authenticate with, or NULL */
+    char *cert;
+    /* Private key file to authenticate with, or NULL */
+    char *key;
+} cliSSLconfig;
+
+/* Wrapper around redisSecureConnection to avoid hiredis_ssl dependencies if
+ * not building with TLS support.
+ */
+int cliSecureConnection(redisContext *c, cliSSLconfig config, const char **err);
+
+/* Wrapper around hiredis to allow arbitrary reads and writes.
+ *
+ * We piggybacks on top of hiredis to achieve transparent TLS support,
+ * and use its internal buffers so it can co-exist with commands
+ * previously/later issued on the connection.
+ *
+ * Interface is close to enough to read()/write() so things should mostly
+ * work transparently.
+ */
+
+/* Write a raw buffer through a redisContext. If we already have something
+ * in the buffer (leftovers from hiredis operations) it will be written
+ * as well.
+ */
+ssize_t cliWriteConn(redisContext *c, const char *buf, size_t buf_len);
+
+/* Wrapper around OpenSSL (libssl and libcrypto) initialisation.
+ */
+int cliSecureInit();
+
+#endif /* __CLICOMMON_H */
diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 328f08077..9f27ad804 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -46,13 +46,19 @@
 #include <sdscompat.h> /* Use hiredis' sds compat header that maps sds calls to their hi_ variants */
 #include <sds.h> /* Use hiredis sds. */
 #include "ae.h"
-#include "hiredis.h"
+#include <hiredis.h>
+#ifdef USE_OPENSSL
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+#include <hiredis_ssl.h>
+#endif
 #include "adlist.h"
 #include "dict.h"
 #include "zmalloc.h"
 #include "atomicvar.h"
 #include "crc16_slottable.h"
 #include "hdr_histogram.h"
+#include "cli_common.h"
 
 #define UNUSED(V) ((void) V)
 #define RANDPTR_INITIAL_SIZE 8
@@ -76,6 +82,8 @@ static struct config {
     const char *hostip;
     int hostport;
     const char *hostsocket;
+    int tls;
+    struct cliSSLconfig sslconfig;
     int numclients;
     redisAtomic int liveclients;
     int requests;
@@ -280,6 +288,13 @@ static redisContext *getRedisContext(const char *ip, int port,
             fprintf(stderr,"%s: %s\n",hostsocket,err);
         goto cleanup;
     }
+    if (config.tls==1) {
+        const char *err = NULL;
+        if (cliSecureConnection(ctx, config.sslconfig, &err) == REDIS_ERR && err) {
+            fprintf(stderr, "Could not negotiate a TLS connection: %s\n", err);
+            goto cleanup;
+        }
+    }
     if (config.auth == NULL)
         return ctx;
     if (config.user == NULL)
@@ -308,6 +323,8 @@ cleanup:
     return NULL;
 }
 
+
+
 static redisConfig *getRedisConfig(const char *ip, int port,
                                    const char *hostsocket)
 {
@@ -606,19 +623,26 @@ static void writeHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
         c->start = ustime();
         c->latency = -1;
     }
-    if (sdslen(c->obuf) > c->written) {
+    const ssize_t buflen = sdslen(c->obuf);
+    const ssize_t writeLen = buflen-c->written;
+    if (writeLen > 0) {
         void *ptr = c->obuf+c->written;
-        ssize_t nwritten = write(c->context->fd,ptr,sdslen(c->obuf)-c->written);
-        if (nwritten == -1) {
-            if (errno != EPIPE)
-                fprintf(stderr, "Writing to socket: %s\n", strerror(errno));
-            freeClient(c);
-            return;
-        }
-        c->written += nwritten;
-        if (sdslen(c->obuf) == c->written) {
-            aeDeleteFileEvent(el,c->context->fd,AE_WRITABLE);
-            aeCreateFileEvent(el,c->context->fd,AE_READABLE,readHandler,c);
+        while(1) {
+            /* Optimistically try to write before checking if the file descriptor
+             * is actually writable. At worst we get EAGAIN. */
+            const ssize_t nwritten = cliWriteConn(c->context,ptr,writeLen);
+            if (nwritten != writeLen) {
+                if (nwritten == -1 && errno != EAGAIN) {
+                    if (errno != EPIPE)
+                        fprintf(stderr, "Error writing to the server: %s\n", strerror(errno));
+                    freeClient(c);
+                    return;
+                }
+            } else {
+                aeDeleteFileEvent(el,c->context->fd,AE_WRITABLE);
+                aeCreateFileEvent(el,c->context->fd,AE_READABLE,readHandler,c);
+                return;
+            }
         }
     }
 }
@@ -680,6 +704,13 @@ static client createClient(char *cmd, size_t len, client from, int thread_id) {
             fprintf(stderr,"%s: %s\n",config.hostsocket,c->context->errstr);
         exit(1);
     }
+    if (config.tls==1) {
+        const char *err = NULL;
+        if (cliSecureConnection(c->context, config.sslconfig, &err) == REDIS_ERR && err) {
+            fprintf(stderr, "Could not negotiate a TLS connection: %s\n", err);
+            exit(1);
+        }
+    }
     c->thread_id = thread_id;
     /* Suppress hiredis cleanup of unused buffers for max speed. */
     c->context->reader->maxbuf = 0;
@@ -1471,6 +1502,25 @@ int parseOptions(int argc, const char **argv) {
         } else if (!strcmp(argv[i],"--help")) {
             exit_status = 0;
             goto usage;
+        #ifdef USE_OPENSSL
+        } else if (!strcmp(argv[i],"--tls")) {
+            config.tls = 1;
+        } else if (!strcmp(argv[i],"--sni")) {
+            if (lastarg) goto invalid;
+            config.sslconfig.sni = strdup(argv[++i]);
+        } else if (!strcmp(argv[i],"--cacertdir")) {
+            if (lastarg) goto invalid;
+            config.sslconfig.cacertdir = strdup(argv[++i]);
+        } else if (!strcmp(argv[i],"--cacert")) {
+            if (lastarg) goto invalid;
+            config.sslconfig.cacert = strdup(argv[++i]);
+        } else if (!strcmp(argv[i],"--cert")) {
+            if (lastarg) goto invalid;
+            config.sslconfig.cert = strdup(argv[++i]);
+        } else if (!strcmp(argv[i],"--key")) {
+            if (lastarg) goto invalid;
+            config.sslconfig.key = strdup(argv[++i]);
+        #endif
         } else {
             /* Assume the user meant to provide an option when the arg starts
              * with a dash. We're done otherwise and should use the remainder
@@ -1518,6 +1568,16 @@ usage:
 " -t <tests>         Only run the comma separated list of tests. The test\n"
 "                    names are the same as the ones produced as output.\n"
 " -I                 Idle mode. Just open N idle connections and wait.\n"
+#ifdef USE_OPENSSL
+" --tls              Establish a secure TLS connection.\n"
+" --sni <host>       Server name indication for TLS.\n"
+" --cacert <file>    CA Certificate file to verify with.\n"
+" --cacertdir <dir>  Directory where trusted CA certificates are stored.\n"
+"                    If neither cacert nor cacertdir are specified, the default\n"
+"                    system-wide trusted root certs configuration will apply.\n"
+" --cert <file>      Client certificate to authenticate with.\n"
+" --key <file>       Private key file to authenticate with.\n"
+#endif
 " --help             Output this help and exit.\n"
 " --version          Output version and exit.\n\n"
 "Examples:\n\n"
@@ -1642,6 +1702,12 @@ int main(int argc, const char **argv) {
 
     tag = "";
 
+#ifdef USE_OPENSSL
+    if (config.tls) {
+        cliSecureInit();
+    }
+#endif
+
     if (config.cluster_mode) {
         // We only include the slot placeholder {tag} if cluster mode is enabled
         tag = ":{tag}";
diff --git a/src/redis-cli.c b/src/redis-cli.c
index 4c9e46926..50edc0c0e 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -61,6 +61,7 @@
 #include "help.h"
 #include "anet.h"
 #include "ae.h"
+#include "cli_common.h"
 
 #define UNUSED(V) ((void) V)
 
@@ -199,11 +200,7 @@ static struct config {
     int hostport;
     char *hostsocket;
     int tls;
-    char *sni;
-    char *cacert;
-    char *cacertdir;
-    char *cert;
-    char *key;
+    cliSSLconfig sslconfig;
     long repeat;
     long interval;
     int dbnum;
@@ -788,71 +785,6 @@ static int cliSelect(void) {
     return REDIS_ERR;
 }
 
-/* Wrapper around redisSecureConnection to avoid hiredis_ssl dependencies if
- * not building with TLS support.
- */
-static int cliSecureConnection(redisContext *c, const char **err) {
-#ifdef USE_OPENSSL
-    static SSL_CTX *ssl_ctx = NULL;
-
-    if (!ssl_ctx) {
-        ssl_ctx = SSL_CTX_new(SSLv23_client_method());
-        if (!ssl_ctx) {
-            *err = "Failed to create SSL_CTX";
-            goto error;
-        }
-
-        SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3);
-        SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL);
-
-        if (config.cacert || config.cacertdir) {
-            if (!SSL_CTX_load_verify_locations(ssl_ctx, config.cacert, config.cacertdir)) {
-                *err = "Invalid CA Certificate File/Directory";
-                goto error;
-            }
-        } else {
-            if (!SSL_CTX_set_default_verify_paths(ssl_ctx)) {
-                *err = "Failed to use default CA paths";
-                goto error;
-            }
-        }
-
-        if (config.cert && !SSL_CTX_use_certificate_chain_file(ssl_ctx, config.cert)) {
-            *err = "Invalid client certificate";
-            goto error;
-        }
-
-        if (config.key && !SSL_CTX_use_PrivateKey_file(ssl_ctx, config.key, SSL_FILETYPE_PEM)) {
-            *err = "Invalid private key";
-            goto error;
-        }
-    }
-
-    SSL *ssl = SSL_new(ssl_ctx);
-    if (!ssl) {
-        *err = "Failed to create SSL object";
-        return REDIS_ERR;
-    }
-
-    if (config.sni && !SSL_set_tlsext_host_name(ssl, config.sni)) {
-        *err = "Failed to configure SNI";
-        SSL_free(ssl);
-        return REDIS_ERR;
-    }
-
-    return redisInitiateSSL(c, ssl);
-
-error:
-    SSL_CTX_free(ssl_ctx);
-    ssl_ctx = NULL;
-    return REDIS_ERR;
-#else
-    (void) c;
-    (void) err;
-    return REDIS_OK;
-#endif
-}
-
 /* Select RESP3 mode if redis-cli was started with the -3 option.  */
 static int cliSwitchProto(void) {
     redisReply *reply;
@@ -886,7 +818,7 @@ static int cliConnect(int flags) {
 
         if (!context->err && config.tls) {
             const char *err = NULL;
-            if (cliSecureConnection(context, &err) == REDIS_ERR && err) {
+            if (cliSecureConnection(context, config.sslconfig, &err) == REDIS_ERR && err) {
                 fprintf(stderr, "Could not negotiate a TLS connection: %s\n", err);
                 redisFree(context);
                 context = NULL;
@@ -1484,7 +1416,7 @@ static redisReply *reconnectingRedisCommand(redisContext *c, const char *fmt, ..
             c = redisConnect(config.hostip,config.hostport);
             if (!c->err && config.tls) {
                 const char *err = NULL;
-                if (cliSecureConnection(c, &err) == REDIS_ERR && err) {
+                if (cliSecureConnection(c, config.sslconfig, &err) == REDIS_ERR && err) {
                     fprintf(stderr, "TLS Error: %s\n", err);
                     exit(1);
                 }
@@ -1700,15 +1632,15 @@ static int parseOptions(int argc, char **argv) {
         } else if (!strcmp(argv[i],"--tls")) {
             config.tls = 1;
         } else if (!strcmp(argv[i],"--sni") && !lastarg) {
-            config.sni = argv[++i];
+            config.sslconfig.sni = argv[++i];
         } else if (!strcmp(argv[i],"--cacertdir") && !lastarg) {
-            config.cacertdir = argv[++i];
+            config.sslconfig.cacertdir = argv[++i];
         } else if (!strcmp(argv[i],"--cacert") && !lastarg) {
-            config.cacert = argv[++i];
+            config.sslconfig.cacert = argv[++i];
         } else if (!strcmp(argv[i],"--cert") && !lastarg) {
-            config.cert = argv[++i];
+            config.sslconfig.cert = argv[++i];
         } else if (!strcmp(argv[i],"--key") && !lastarg) {
-            config.key = argv[++i];
+            config.sslconfig.key = argv[++i];
 #endif
         } else if (!strcmp(argv[i],"-v") || !strcmp(argv[i], "--version")) {
             sds version = cliVersion();
@@ -2677,7 +2609,7 @@ static int clusterManagerNodeConnect(clusterManagerNode *node) {
     node->context = redisConnect(node->ip, node->port);
     if (!node->context->err && config.tls) {
         const char *err = NULL;
-        if (cliSecureConnection(node->context, &err) == REDIS_ERR && err) {
+        if (cliSecureConnection(node->context, config.sslconfig, &err) == REDIS_ERR && err) {
             fprintf(stderr,"TLS Error: %s\n", err);
             redisFree(node->context);
             node->context = NULL;
@@ -6898,72 +6830,6 @@ void sendCapa() {
     sendReplconf("capa", "eof");
 }
 
-/* Wrapper around hiredis to allow arbitrary reads and writes.
- *
- * We piggybacks on top of hiredis to achieve transparent TLS support,
- * and use its internal buffers so it can co-exist with commands
- * previously/later issued on the connection.
- *
- * Interface is close to enough to read()/write() so things should mostly
- * work transparently.
- */
-
-/* Write a raw buffer through a redisContext. If we already have something
- * in the buffer (leftovers from hiredis operations) it will be written
- * as well.
- */
-static ssize_t writeConn(redisContext *c, const char *buf, size_t buf_len)
-{
-    int done = 0;
-
-    /* Append data to buffer which is *usually* expected to be empty
-     * but we don't assume that, and write.
-     */
-    c->obuf = sdscatlen(c->obuf, buf, buf_len);
-    if (redisBufferWrite(c, &done) == REDIS_ERR) {
-        if (!(c->flags & REDIS_BLOCK))
-            errno = EAGAIN;
-
-        /* On error, we assume nothing was written and we roll back the
-         * buffer to its original state.
-         */
-        if (sdslen(c->obuf) > buf_len)
-            sdsrange(c->obuf, 0, -(buf_len+1));
-        else
-            sdsclear(c->obuf);
-
-        return -1;
-    }
-
-    /* If we're done, free up everything. We may have written more than
-     * buf_len (if c->obuf was not initially empty) but we don't have to
-     * tell.
-     */
-    if (done) {
-        sdsclear(c->obuf);
-        return buf_len;
-    }
-
-    /* Write was successful but we have some leftovers which we should
-     * remove from the buffer.
-     *
-     * Do we still have data that was there prior to our buf? If so,
-     * restore buffer to it's original state and report no new data was
-     * writen.
-     */
-    if (sdslen(c->obuf) > buf_len) {
-        sdsrange(c->obuf, 0, -(buf_len+1));
-        return 0;
-    }
-
-    /* At this point we're sure no prior data is left. We flush the buffer
-     * and report how much we've written.
-     */
-    size_t left = sdslen(c->obuf);
-    sdsclear(c->obuf);
-    return buf_len - left;
-}
-
 /* Read raw bytes through a redisContext. The read operation is not greedy
  * and may not fill the buffer entirely.
  */
@@ -6984,7 +6850,7 @@ unsigned long long sendSync(redisContext *c, char *out_eof) {
     ssize_t nread;
 
     /* Send the SYNC command. */
-    if (writeConn(c, "SYNC\r\n", 6) != 6) {
+    if (cliWriteConn(c, "SYNC\r\n", 6) != 6) {
         fprintf(stderr,"Error writing to master\n");
         exit(1);
     }
@@ -7249,7 +7115,7 @@ static void pipeMode(void) {
             while(1) {
                 /* Transfer current buffer to server. */
                 if (obuf_len != 0) {
-                    ssize_t nwritten = writeConn(context,obuf+obuf_pos,obuf_len);
+                    ssize_t nwritten = cliWriteConn(context,obuf+obuf_pos,obuf_len);
 
                     if (nwritten == -1) {
                         if (errno != EAGAIN && errno != EINTR) {
@@ -8229,9 +8095,7 @@ int main(int argc, char **argv) {
 
 #ifdef USE_OPENSSL
     if (config.tls) {
-        ERR_load_crypto_strings();
-        SSL_load_error_strings();
-        SSL_library_init();
+        cliSecureInit();
     }
 #endif
 
diff --git a/tests/support/benchmark.tcl b/tests/support/benchmark.tcl
index 1130c9992..ed75bfeda 100644
--- a/tests/support/benchmark.tcl
+++ b/tests/support/benchmark.tcl
@@ -1,5 +1,19 @@
+proc redisbenchmark_tls_config {testsdir} {
+    set tlsdir [file join $testsdir tls]
+    set cert [file join $tlsdir redis.crt]
+    set key [file join $tlsdir redis.key]
+    set cacert [file join $tlsdir ca.crt]
+
+    if {$::tls} {
+        return [list --tls --cert $cert --key $key --cacert $cacert]
+    } else {
+        return {}
+    }
+}
+
 proc redisbenchmark {host port {opts {}}} {
     set cmd [list src/redis-benchmark -h $host -p $port]
+    lappend cmd {*}[redisbenchmark_tls_config "tests"]
     lappend cmd {*}$opts
     return $cmd
 }

From 69871760599faafb1a6201b49ec5197ec3294b64 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Wed, 28 Oct 2020 14:51:35 +0800
Subject: [PATCH 0763/1098] docs: Fix some typos in comments and log messge
 (#7975)

---
 src/childinfo.c | 2 +-
 src/db.c        | 2 +-
 src/dict.c      | 2 +-
 src/rdb.c       | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/childinfo.c b/src/childinfo.c
index f95ae9647..d11aa7bcf 100644
--- a/src/childinfo.c
+++ b/src/childinfo.c
@@ -36,7 +36,7 @@
 void openChildInfoPipe(void) {
     if (pipe(server.child_info_pipe) == -1) {
         /* On error our two file descriptors should be still set to -1,
-         * but we call anyway cloesChildInfoPipe() since can't hurt. */
+         * but we call anyway closeChildInfoPipe() since can't hurt. */
         closeChildInfoPipe();
     } else if (anetNonBlock(NULL,server.child_info_pipe[0]) != ANET_OK) {
         closeChildInfoPipe();
diff --git a/src/db.c b/src/db.c
index a26280719..0f37c7441 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1691,7 +1691,7 @@ void slotToKeyFlush(void) {
            sizeof(server.cluster->slots_keys_count));
 }
 
-/* Pupulate the specified array of objects with keys in the specified slot.
+/* Populate the specified array of objects with keys in the specified slot.
  * New objects are returned to represent keys, it's up to the caller to
  * decrement the reference count to release the keys names. */
 unsigned int getKeysInSlot(unsigned int hashslot, robj **keys, unsigned int count) {
diff --git a/src/dict.c b/src/dict.c
index 4760a5d26..57287442a 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -759,7 +759,7 @@ dictEntry *dictGetFairRandomKey(dict *d) {
     /* Note that dictGetSomeKeys() may return zero elements in an unlucky
      * run() even if there are actually elements inside the hash table. So
      * when we get zero, we call the true dictGetRandomKey() that will always
-     * yeld the element if the hash table has at least one. */
+     * yield the element if the hash table has at least one. */
     if (count == 0) return dictGetRandomKey(d);
     unsigned int idx = rand() % count;
     return entries[idx];
diff --git a/src/rdb.c b/src/rdb.c
index 9301cdf71..db5698c1d 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1419,7 +1419,7 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
 
 /* Note that we may call this function in signal handle 'sigShutdownHandler',
  * so we need guarantee all functions we call are async-signal-safe.
- * If  we call this function from signal handle, we won't call bg_unlik that
+ * If  we call this function from signal handle, we won't call bg_unlink that
  * is not async-signal-safe. */
 void rdbRemoveTempFile(pid_t childpid, int from_signal) {
     char tmpfile[256];
@@ -1892,7 +1892,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                     return NULL;
                 }
                 if (!raxInsert(cgroup->pel,rawid,sizeof(rawid),nack,NULL))
-                    rdbExitReportCorruptRDB("Duplicated gobal PEL entry "
+                    rdbExitReportCorruptRDB("Duplicated global PEL entry "
                                             "loading stream consumer group");
             }
 

From 434270374344bbdfa36545a93868b7920e4f47ce Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Wed, 28 Oct 2020 06:35:28 -0400
Subject: [PATCH 0764/1098] refactor aof rewrite code to avoid memory leaks in
 error handling (#7976)

---
 src/aof.c | 97 ++++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 68 insertions(+), 29 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index 5c46c76f6..b9072da93 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -961,15 +961,25 @@ int rewriteListObject(rio *r, robj *key, robj *o) {
             if (count == 0) {
                 int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ?
                     AOF_REWRITE_ITEMS_PER_CMD : items;
-                if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0;
-                if (rioWriteBulkString(r,"RPUSH",5) == 0) return 0;
-                if (rioWriteBulkObject(r,key) == 0) return 0;
+                if (!rioWriteBulkCount(r,'*',2+cmd_items) ||
+                    !rioWriteBulkString(r,"RPUSH",5) ||
+                    !rioWriteBulkObject(r,key)) 
+                {
+                    quicklistReleaseIterator(li);
+                    return 0;
+                }
             }
 
             if (entry.value) {
-                if (rioWriteBulkString(r,(char*)entry.value,entry.sz) == 0) return 0;
+                if (!rioWriteBulkString(r,(char*)entry.value,entry.sz)) {
+                    quicklistReleaseIterator(li);
+                    return 0;
+                }
             } else {
-                if (rioWriteBulkLongLong(r,entry.longval) == 0) return 0;
+                if (!rioWriteBulkLongLong(r,entry.longval)) {
+                    quicklistReleaseIterator(li);
+                    return 0;
+                }
             }
             if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0;
             items--;
@@ -995,11 +1005,14 @@ int rewriteSetObject(rio *r, robj *key, robj *o) {
                 int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ?
                     AOF_REWRITE_ITEMS_PER_CMD : items;
 
-                if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0;
-                if (rioWriteBulkString(r,"SADD",4) == 0) return 0;
-                if (rioWriteBulkObject(r,key) == 0) return 0;
+                if (!rioWriteBulkCount(r,'*',2+cmd_items) ||
+                    !rioWriteBulkString(r,"SADD",4) ||
+                    !rioWriteBulkObject(r,key)) 
+                {
+                    return 0;
+                }
             }
-            if (rioWriteBulkLongLong(r,llval) == 0) return 0;
+            if (!rioWriteBulkLongLong(r,llval)) return 0;
             if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0;
             items--;
         }
@@ -1013,11 +1026,18 @@ int rewriteSetObject(rio *r, robj *key, robj *o) {
                 int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ?
                     AOF_REWRITE_ITEMS_PER_CMD : items;
 
-                if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0;
-                if (rioWriteBulkString(r,"SADD",4) == 0) return 0;
-                if (rioWriteBulkObject(r,key) == 0) return 0;
+                if (!rioWriteBulkCount(r,'*',2+cmd_items) ||
+                    !rioWriteBulkString(r,"SADD",4) ||
+                    !rioWriteBulkObject(r,key)) 
+                {
+                    dictReleaseIterator(di);
+                    return 0;
+                }
+            }
+            if (!rioWriteBulkString(r,ele,sdslen(ele))) {
+                dictReleaseIterator(di);
+                return 0;          
             }
-            if (rioWriteBulkString(r,ele,sdslen(ele)) == 0) return 0;
             if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0;
             items--;
         }
@@ -1054,15 +1074,18 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) {
                 int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ?
                     AOF_REWRITE_ITEMS_PER_CMD : items;
 
-                if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0;
-                if (rioWriteBulkString(r,"ZADD",4) == 0) return 0;
-                if (rioWriteBulkObject(r,key) == 0) return 0;
+                if (!rioWriteBulkCount(r,'*',2+cmd_items*2) ||
+                    !rioWriteBulkString(r,"ZADD",4) ||
+                    !rioWriteBulkObject(r,key)) 
+                {
+                    return 0;
+                }
             }
-            if (rioWriteBulkDouble(r,score) == 0) return 0;
+            if (!rioWriteBulkDouble(r,score)) return 0;
             if (vstr != NULL) {
-                if (rioWriteBulkString(r,(char*)vstr,vlen) == 0) return 0;
+                if (!rioWriteBulkString(r,(char*)vstr,vlen)) return 0;
             } else {
-                if (rioWriteBulkLongLong(r,vll) == 0) return 0;
+                if (!rioWriteBulkLongLong(r,vll)) return 0;
             }
             zzlNext(zl,&eptr,&sptr);
             if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0;
@@ -1081,12 +1104,20 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) {
                 int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ?
                     AOF_REWRITE_ITEMS_PER_CMD : items;
 
-                if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0;
-                if (rioWriteBulkString(r,"ZADD",4) == 0) return 0;
-                if (rioWriteBulkObject(r,key) == 0) return 0;
+                if (!rioWriteBulkCount(r,'*',2+cmd_items*2) ||
+                    !rioWriteBulkString(r,"ZADD",4) ||
+                    !rioWriteBulkObject(r,key)) 
+                {
+                    dictReleaseIterator(di);
+                    return 0;
+                }
+            }
+            if (!rioWriteBulkDouble(r,*score) ||
+                !rioWriteBulkString(r,ele,sdslen(ele)))
+            {
+                dictReleaseIterator(di);
+                return 0;
             }
-            if (rioWriteBulkDouble(r,*score) == 0) return 0;
-            if (rioWriteBulkString(r,ele,sdslen(ele)) == 0) return 0;
             if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0;
             items--;
         }
@@ -1135,13 +1166,21 @@ int rewriteHashObject(rio *r, robj *key, robj *o) {
             int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ?
                 AOF_REWRITE_ITEMS_PER_CMD : items;
 
-            if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0;
-            if (rioWriteBulkString(r,"HMSET",5) == 0) return 0;
-            if (rioWriteBulkObject(r,key) == 0) return 0;
+            if (!rioWriteBulkCount(r,'*',2+cmd_items*2) ||
+                !rioWriteBulkString(r,"HMSET",5) ||
+                !rioWriteBulkObject(r,key)) 
+            {
+                hashTypeReleaseIterator(hi);
+                return 0;
+            }
         }
 
-        if (rioWriteHashIteratorCursor(r, hi, OBJ_HASH_KEY) == 0) return 0;
-        if (rioWriteHashIteratorCursor(r, hi, OBJ_HASH_VALUE) == 0) return 0;
+        if (!rioWriteHashIteratorCursor(r, hi, OBJ_HASH_KEY) ||
+            !rioWriteHashIteratorCursor(r, hi, OBJ_HASH_VALUE))
+        {
+            hashTypeReleaseIterator(hi);
+            return 0;           
+        }
         if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0;
         items--;
     }

From efd17316ab122f5ddd9ecd89a98f30759b076a10 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Wed, 28 Oct 2020 09:05:00 -0400
Subject: [PATCH 0765/1098] add acl related config in sentinel.conf (#7952)

---
 sentinel.conf | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/sentinel.conf b/sentinel.conf
index 50a36a314..39d6929e7 100644
--- a/sentinel.conf
+++ b/sentinel.conf
@@ -124,6 +124,42 @@ sentinel monitor mymaster 127.0.0.1 6379 2
 # Default is 30 seconds.
 sentinel down-after-milliseconds mymaster 30000
 
+# IMPORTANT NOTE: starting with Redis 6.2 ACL capability is supported for
+# Sentinel mode, please refer to the Redis website https://redis.io/topics/acl
+# for more details.
+
+# Sentinel's ACL users are defined in the following format:
+#
+#   user <username> ... acl rules ...
+#
+# For example:
+#
+#   user worker +@admin +@connection ~* on >ffa9203c493aa99
+#
+# For more information about ACL configuration please refer to the Redis
+# website at https://redis.io/topics/acl and redis server configuration 
+# template redis.conf.
+
+# ACL LOG
+#
+# The ACL Log tracks failed commands and authentication events associated
+# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked 
+# by ACLs. The ACL Log is stored in memory. You can reclaim memory with 
+# ACL LOG RESET. Define the maximum entry length of the ACL Log below.
+acllog-max-len 128
+
+# Using an external ACL file
+#
+# Instead of configuring users here in this file, it is possible to use
+# a stand-alone file just listing users. The two methods cannot be mixed:
+# if you configure users here and at the same time you activate the external
+# ACL file, the server will refuse to start.
+#
+# The format of the external ACL user file is exactly the same as the
+# format that is used inside redis.conf to describe users.
+#
+# aclfile /etc/redis/sentinel-users.acl
+
 # requirepass <password>
 #
 # You can configure Sentinel itself to require a password, however when doing

From 411bcf1a41d2758823d17e0864ef45e5f3948b7a Mon Sep 17 00:00:00 2001
From: Madelyn Olson <madelyneolson@gmail.com>
Date: Mon, 26 Oct 2020 21:23:30 -0700
Subject: [PATCH 0766/1098] Further improved ACL algorithm for picking
 categories

---
 src/acl.c          | 67 +++++++++++++++++++++++++++++++---------------
 tests/unit/acl.tcl | 22 +++++++++------
 2 files changed, 60 insertions(+), 29 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index d9ee774c8..6f28818a0 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -479,36 +479,61 @@ sds ACLDescribeUserCommandRules(user *u) {
         ACLSetUser(fakeuser,"-@all",-1);
     }
 
-    /* Try to add or subtract each category one after the other. Often a
-     * single category will not perfectly match the set of commands into
-     * it, so at the end we do a final pass adding/removing the single commands
-     * needed to make the bitmap exactly match. A temp user is maintained to
-     * keep track of categories already applied. */
+    /* Attempt to find a good approximation for categories and commands
+     * based on the current bits used, by looping over the category list
+     * and applying the best fit each time. Often a set of categories will not 
+     * perfectly match the set of commands into it, so at the end we do a 
+     * final pass adding/removing the single commands needed to make the bitmap
+     * exactly match. A temp user is maintained to keep track of categories 
+     * already applied. */
     user tu = {0};
     user *tempuser = &tu;
     memcpy(tempuser->allowed_commands,
         u->allowed_commands, 
         sizeof(u->allowed_commands));
+    while (1) {
+        int best = -1;
+        unsigned long mindiff = INT_MAX, maxsame = 0;
+        for (int j = 0; ACLCommandCategories[j].flag != 0; j++) {
+            unsigned long on, off, diff, same;
+            ACLCountCategoryBitsForUser(tempuser,&on,&off,ACLCommandCategories[j].name);
 
-    for (int j = 0; ACLCommandCategories[j].flag != 0; j++) {
-        unsigned long on, off;
-        ACLCountCategoryBitsForUser(tempuser,&on,&off,ACLCommandCategories[j].name);
-        if ((additive && on > off) || (!additive && off > on)) {
-            sds op = sdsnewlen(additive ? "+@" : "-@", 2);
-            op = sdscat(op,ACLCommandCategories[j].name);
-            ACLSetUser(fakeuser,op,-1);
-
-            sds invop = sdsnewlen(additive ? "-@" : "+@", 2);
-            invop = sdscat(invop,ACLCommandCategories[j].name);
-            ACLSetUser(tempuser,invop,-1);
-
-            rules = sdscatsds(rules,op);
-            rules = sdscatlen(rules," ",1);
-            sdsfree(op);
-            sdsfree(invop);
+            /* Check if the current category is the best this loop:
+             * * It has more commands in common with the user than commands
+             *   that are different.
+             * AND EITHER
+             * * It has the fewest number of differences
+             *    than the best match we have found so far. 
+             * * OR it matches the fewest number of differences
+             *   that we've seen but it has more in common. */
+            diff = additive ? off : on;
+            same = additive ? on : off;
+            if (same > diff && 
+                    ((diff < mindiff) || (diff == mindiff && same > maxsame))) {
+                best = j;
+                mindiff = diff;
+                maxsame = same;
+            }
         }
+
+        /* We didn't find a match */
+        if (best == -1) break;
+
+        sds op = sdsnewlen(additive ? "+@" : "-@", 2);
+        op = sdscat(op,ACLCommandCategories[best].name);
+        ACLSetUser(fakeuser,op,-1);
+
+        sds invop = sdsnewlen(additive ? "-@" : "+@", 2);
+        invop = sdscat(invop,ACLCommandCategories[best].name);
+        ACLSetUser(tempuser,invop,-1);
+
+        rules = sdscatsds(rules,op);
+        rules = sdscatlen(rules," ",1);
+        sdsfree(op);
+        sdsfree(invop);
     }
 
+
     /* Fix the final ACLs with single commands differences. */
     dictIterator *di = dictGetIterator(server.orig_commands);
     dictEntry *de;
diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl
index 12f59e749..d35a012f4 100644
--- a/tests/unit/acl.tcl
+++ b/tests/unit/acl.tcl
@@ -138,15 +138,21 @@ start_server {tags {"acl"}} {
     # A regression test make sure that as long as there is a simple
     # category defining the commands, that it will be used as is.
     test {ACL GETUSER provides reasonable results} {
-        # Test for future commands where allowed
-        r ACL setuser additive reset +@all -@write
-        set cmdstr [dict get [r ACL getuser additive] commands]
-        assert_match {+@all -@write} $cmdstr
+        set categories [r ACL CAT]
 
-        # Test for future commands are disallowed
-        r ACL setuser subtractive reset -@all +@read
-        set cmdstr [dict get [r ACL getuser subtractive] commands]
-        assert_match {-@all +@read} $cmdstr
+        # Test that adding each single category will
+        # result in just that category with both +@all and -@all
+        foreach category $categories {
+            # Test for future commands where allowed
+            r ACL setuser additive reset +@all "-@$category"
+            set cmdstr [dict get [r ACL getuser additive] commands]
+            assert_equal "+@all -@$category" $cmdstr
+
+            # Test for future commands where disallowed
+            r ACL setuser restrictive reset -@all "+@$category"
+            set cmdstr [dict get [r ACL getuser restrictive] commands]
+            assert_equal "-@all +@$category" $cmdstr
+        }
     }
 
     test {ACL #5998 regression: memory leaks adding / removing subcommands} {

From d310beb4170ebbc8985ae120ee301f9213d33e39 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <madelyneolson@gmail.com>
Date: Tue, 27 Oct 2020 12:47:02 -0700
Subject: [PATCH 0767/1098] White space tweaks and skip categories already
 applied

---
 src/acl.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 6f28818a0..f0523b635 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -488,6 +488,12 @@ sds ACLDescribeUserCommandRules(user *u) {
      * already applied. */
     user tu = {0};
     user *tempuser = &tu;
+    
+    /* Keep track of the categories that have been applied, to prevent
+     * applying them twice.  */
+    char applied[sizeof(ACLCommandCategories)/sizeof(ACLCommandCategories[0])];
+    memset(applied, 0, sizeof(applied));
+
     memcpy(tempuser->allowed_commands,
         u->allowed_commands, 
         sizeof(u->allowed_commands));
@@ -495,9 +501,10 @@ sds ACLDescribeUserCommandRules(user *u) {
         int best = -1;
         unsigned long mindiff = INT_MAX, maxsame = 0;
         for (int j = 0; ACLCommandCategories[j].flag != 0; j++) {
+            if (applied[j]) continue;
+
             unsigned long on, off, diff, same;
             ACLCountCategoryBitsForUser(tempuser,&on,&off,ACLCommandCategories[j].name);
-
             /* Check if the current category is the best this loop:
              * * It has more commands in common with the user than commands
              *   that are different.
@@ -509,7 +516,8 @@ sds ACLDescribeUserCommandRules(user *u) {
             diff = additive ? off : on;
             same = additive ? on : off;
             if (same > diff && 
-                    ((diff < mindiff) || (diff == mindiff && same > maxsame))) {
+                ((diff < mindiff) || (diff == mindiff && same > maxsame)))
+            {
                 best = j;
                 mindiff = diff;
                 maxsame = same;
@@ -531,8 +539,9 @@ sds ACLDescribeUserCommandRules(user *u) {
         rules = sdscatlen(rules," ",1);
         sdsfree(op);
         sdsfree(invop);
-    }
 
+        applied[best] = 1;
+    }
 
     /* Fix the final ACLs with single commands differences. */
     dictIterator *di = dictGetIterator(server.orig_commands);

From 441bfa2dfb57e0dbfddad167bafa559b3f051b5b Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 28 Oct 2020 21:09:15 +0200
Subject: [PATCH 0768/1098] Optionally (default) fail to start if requested
 bind address is not available (#7936)

Background:
#3467 (redis 4.0.0), started ignoring ENOPROTOOPT, but did that only for
the default bind (in case bind config wasn't explicitly set).
#5598 (redis 5.0.3), added that for bind addresses explicitly set
(following bug reports in Debian for redis 4.0.9 and 5.0.1), it
also ignored a bunch of other errors like EPROTONOSUPPORT which was
requested in #3894, and also added EADDRNOTAVAIL (wasn't clear why).

This (ignoring EADDRNOTAVAIL) makes redis start successfully, even if a
certain network interface isn't up yet , in which case we rather redis
fail and will be re-tried when the NIC is up, see #7933.

However, it turns out that when IPv6 is disabled (supported but unused),
the error we're getting is EADDRNOTAVAIL. and in many systems the
default config file tries to bind to localhost for both v4 and v6 and
would like to silently ignore the error on v6 if disabled.
This means that we sometimes want to ignore EADDRNOTAVAIL and other times
we wanna fail.

So this commit changes these main things:
1. Ignore all the errors we ignore for both explicitly requested bind
   address and a default implicit one.
2. Add a '-' prefix to allow EADDRNOTAVAIL be ignored (by default that's
   different than the previous behavior).
3. Restructure that function in a more readable and maintainable way see
   below.
4. Make the default behavior of listening to all achievable by setting
  a bind config directive to * (previously only possible by omitting
  it)
5. document everything.

The old structure of this function was that even if there are no bind
addresses requested, the loop that runs though the bind addresses runs
at least once anyway!
In that one iteration of the loop it binds to both v4 and v6 addresses,
handles errors for each of them separately, and then eventually at the
if-else chain, handles the error of the last bind attempt again!
This was very hard to read and very error prone to maintain, instead now
when the bind info is missing we create one with two entries, and run
the simple loop twice.
---
 redis.conf   | 17 ++++++++++----
 src/anet.c   |  4 ++++
 src/server.c | 66 ++++++++++++++++++----------------------------------
 3 files changed, 38 insertions(+), 49 deletions(-)

diff --git a/redis.conf b/redis.conf
index 119f7b6f9..4caa70f38 100644
--- a/redis.conf
+++ b/redis.conf
@@ -49,23 +49,30 @@
 # for connections from all available network interfaces on the host machine.
 # It is possible to listen to just one or multiple selected interfaces using
 # the "bind" configuration directive, followed by one or more IP addresses.
+# Each address can be prefixed by "-", which means that redis will not fail to
+# start if the address is not available. Being not available only refers to
+# addresses that does not correspond to any network interfece. Addresses that
+# are already in use will always fail, and unsupported protocols will always BE
+# silently skipped.
 #
 # Examples:
 #
-# bind 192.168.1.100 10.0.0.1
-# bind 127.0.0.1 ::1
+# bind 192.168.1.100 10.0.0.1     # listens on two specific IPv4 addresses
+# bind 127.0.0.1 ::1              # listens on loopback IPv4 and IPv6
+# bind * -::*                     # like the default, all available interfaces
 #
 # ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the
 # internet, binding to all the interfaces is dangerous and will expose the
 # instance to everybody on the internet. So by default we uncomment the
 # following bind directive, that will force Redis to listen only on the
-# IPv4 loopback interface address (this means Redis will only be able to
-# accept client connections from the same host that it is running on).
+# IPv4 and IPv6 (if available) loopback interface addresses (this means Redis
+# will only be able to accept client connections from the same host that it is
+# running on).
 #
 # IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES
 # JUST COMMENT OUT THE FOLLOWING LINE.
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-bind 127.0.0.1
+bind 127.0.0.1 -::1
 
 # Protected mode is a layer of security protection, in order to avoid that
 # Redis instances left open on the internet are accessed and exploited.
diff --git a/src/anet.c b/src/anet.c
index 46ea7e145..e9615a128 100644
--- a/src/anet.c
+++ b/src/anet.c
@@ -487,6 +487,10 @@ static int _anetTcpServer(char *err, int port, char *bindaddr, int af, int backl
     hints.ai_family = af;
     hints.ai_socktype = SOCK_STREAM;
     hints.ai_flags = AI_PASSIVE;    /* No effect if bindaddr != NULL */
+    if (bindaddr && !strcmp("*", bindaddr))
+        bindaddr = NULL;
+    if (af == AF_INET6 && bindaddr && !strcmp("::*", bindaddr))
+        bindaddr = NULL;
 
     if ((rv = getaddrinfo(bindaddr,_port,&hints,&servinfo)) != 0) {
         anetSetError(err, "%s", gai_strerror(rv));
diff --git a/src/server.c b/src/server.c
index bb035f795..cd45dd837 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2799,59 +2799,37 @@ void checkTcpBacklogSettings(void) {
  * one of the IPv4 or IPv6 protocols. */
 int listenToPort(int port, int *fds, int *count) {
     int j;
+    char **bindaddr = server.bindaddr;
+    int bindaddr_count = server.bindaddr_count;
+    char *default_bindaddr[2] = {"*", "-::*"};
 
-    /* Force binding of 0.0.0.0 if no bind address is specified, always
-     * entering the loop if j == 0. */
-    if (server.bindaddr_count == 0) server.bindaddr[0] = NULL;
-    for (j = 0; j < server.bindaddr_count || j == 0; j++) {
-        if (server.bindaddr[j] == NULL) {
-            int unsupported = 0;
-            /* Bind * for both IPv6 and IPv4, we enter here only if
-             * server.bindaddr_count == 0. */
-            fds[*count] = anetTcp6Server(server.neterr,port,NULL,
-                server.tcp_backlog);
-            if (fds[*count] != ANET_ERR) {
-                anetNonBlock(NULL,fds[*count]);
-                (*count)++;
-            } else if (errno == EAFNOSUPPORT) {
-                unsupported++;
-                serverLog(LL_WARNING,"Not listening to IPv6: unsupported");
-            }
+    /* Force binding of 0.0.0.0 if no bind address is specified. */
+    if (server.bindaddr_count == 0) {
+        bindaddr_count = 2;
+        bindaddr = default_bindaddr;
+    }
 
-            if (*count == 1 || unsupported) {
-                /* Bind the IPv4 address as well. */
-                fds[*count] = anetTcpServer(server.neterr,port,NULL,
-                    server.tcp_backlog);
-                if (fds[*count] != ANET_ERR) {
-                    anetNonBlock(NULL,fds[*count]);
-                    (*count)++;
-                } else if (errno == EAFNOSUPPORT) {
-                    unsupported++;
-                    serverLog(LL_WARNING,"Not listening to IPv4: unsupported");
-                }
-            }
-            /* Exit the loop if we were able to bind * on IPv4 and IPv6,
-             * otherwise fds[*count] will be ANET_ERR and we'll print an
-             * error and return to the caller with an error. */
-            if (*count + unsupported == 2) break;
-        } else if (strchr(server.bindaddr[j],':')) {
+    for (j = 0; j < bindaddr_count; j++) {
+        char* addr = bindaddr[j];
+        int optional = *addr == '-';
+        if (optional) addr++;
+        if (strchr(addr,':')) {
             /* Bind IPv6 address. */
-            fds[*count] = anetTcp6Server(server.neterr,port,server.bindaddr[j],
-                server.tcp_backlog);
+            fds[*count] = anetTcp6Server(server.neterr,port,addr,server.tcp_backlog);
         } else {
             /* Bind IPv4 address. */
-            fds[*count] = anetTcpServer(server.neterr,port,server.bindaddr[j],
-                server.tcp_backlog);
+            fds[*count] = anetTcpServer(server.neterr,port,addr,server.tcp_backlog);
         }
         if (fds[*count] == ANET_ERR) {
             serverLog(LL_WARNING,
                 "Could not create server TCP listening socket %s:%d: %s",
-                server.bindaddr[j] ? server.bindaddr[j] : "*",
-                port, server.neterr);
-                if (errno == ENOPROTOOPT     || errno == EPROTONOSUPPORT ||
-                    errno == ESOCKTNOSUPPORT || errno == EPFNOSUPPORT ||
-                    errno == EAFNOSUPPORT    || errno == EADDRNOTAVAIL)
-                    continue;
+                addr, port, server.neterr);
+            if (errno == EADDRNOTAVAIL && optional)
+                continue;
+            if (errno == ENOPROTOOPT     || errno == EPROTONOSUPPORT ||
+                errno == ESOCKTNOSUPPORT || errno == EPFNOSUPPORT ||
+                errno == EAFNOSUPPORT)
+                continue;
             return C_ERR;
         }
         anetNonBlock(NULL,fds[*count]);

From 84b3c18f71ec5b03e58a823fbf75de9c4c80969c Mon Sep 17 00:00:00 2001
From: yoav-steinberg <yoav@monfort.co.il>
Date: Wed, 28 Oct 2020 21:13:44 +0200
Subject: [PATCH 0769/1098] Add local address to CLIENT LIST, and a CLIENT KILL
 filter. (#7913)

Useful when you want to know through which bind address the client connected to
the server in case of multiple bind addresses.

- Adding `laddr` field to CLIENT list showing the local (bind) address.
- Adding `LADDR` option to CLIENT KILL to kill all the clients connected
  to a specific local address.
- Refactoring to share code.
---
 src/anet.c        | 46 ++++++++++-----------------------------------
 src/anet.h        | 10 ++++++----
 src/aof.c         |  1 +
 src/connection.c  | 12 ++++++------
 src/connection.h  |  2 +-
 src/networking.c  | 48 ++++++++++++++++++++++++++++++++++-------------
 src/replication.c |  7 ++++++-
 src/sentinel.c    |  6 +++---
 src/server.h      |  4 +++-
 9 files changed, 71 insertions(+), 65 deletions(-)

diff --git a/src/anet.c b/src/anet.c
index e9615a128..d46ce5398 100644
--- a/src/anet.c
+++ b/src/anet.c
@@ -592,11 +592,15 @@ int anetUnixAccept(char *err, int s) {
     return fd;
 }
 
-int anetPeerToString(int fd, char *ip, size_t ip_len, int *port) {
+int anetFdToString(int fd, char *ip, size_t ip_len, int *port, int fd_to_str_type) {
     struct sockaddr_storage sa;
     socklen_t salen = sizeof(sa);
 
-    if (getpeername(fd,(struct sockaddr*)&sa,&salen) == -1) goto error;
+    if (fd_to_str_type == FD_TO_PEER_NAME) {
+        if (getpeername(fd, (struct sockaddr *)&sa, &salen) == -1) goto error;
+    } else {
+        if (getsockname(fd, (struct sockaddr *)&sa, &salen) == -1) goto error;
+    }
     if (ip_len == 0) goto error;
 
     if (sa.ss_family == AF_INET) {
@@ -608,7 +612,7 @@ int anetPeerToString(int fd, char *ip, size_t ip_len, int *port) {
         if (ip) inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len);
         if (port) *port = ntohs(s->sin6_port);
     } else if (sa.ss_family == AF_UNIX) {
-        if (ip) strncpy(ip,"/unixsocket",ip_len);
+        if (ip) snprintf(ip, ip_len, "/unixsocket");
         if (port) *port = 0;
     } else {
         goto error;
@@ -636,41 +640,11 @@ int anetFormatAddr(char *buf, size_t buf_len, char *ip, int port) {
            "[%s]:%d" : "%s:%d", ip, port);
 }
 
-/* Like anetFormatAddr() but extract ip and port from the socket's peer. */
-int anetFormatPeer(int fd, char *buf, size_t buf_len) {
+/* Like anetFormatAddr() but extract ip and port from the socket's peer/sockname. */
+int anetFormatFdAddr(int fd, char *buf, size_t buf_len, int fd_to_str_type) {
     char ip[INET6_ADDRSTRLEN];
     int port;
 
-    anetPeerToString(fd,ip,sizeof(ip),&port);
+    anetFdToString(fd,ip,sizeof(ip),&port,fd_to_str_type);
     return anetFormatAddr(buf, buf_len, ip, port);
 }
-
-int anetSockName(int fd, char *ip, size_t ip_len, int *port) {
-    struct sockaddr_storage sa;
-    socklen_t salen = sizeof(sa);
-
-    if (getsockname(fd,(struct sockaddr*)&sa,&salen) == -1) {
-        if (port) *port = 0;
-        ip[0] = '?';
-        ip[1] = '\0';
-        return -1;
-    }
-    if (sa.ss_family == AF_INET) {
-        struct sockaddr_in *s = (struct sockaddr_in *)&sa;
-        if (ip) inet_ntop(AF_INET,(void*)&(s->sin_addr),ip,ip_len);
-        if (port) *port = ntohs(s->sin_port);
-    } else {
-        struct sockaddr_in6 *s = (struct sockaddr_in6 *)&sa;
-        if (ip) inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len);
-        if (port) *port = ntohs(s->sin6_port);
-    }
-    return 0;
-}
-
-int anetFormatSock(int fd, char *fmt, size_t fmt_len) {
-    char ip[INET6_ADDRSTRLEN];
-    int port;
-
-    anetSockName(fd,ip,sizeof(ip),&port);
-    return anetFormatAddr(fmt, fmt_len, ip, port);
-}
diff --git a/src/anet.h b/src/anet.h
index 23f19643c..fbf41cd17 100644
--- a/src/anet.h
+++ b/src/anet.h
@@ -49,6 +49,10 @@
 #undef ip_len
 #endif
 
+/* FD to address string conversion types */
+#define FD_TO_PEER_NAME 0
+#define FD_TO_SOCK_NAME 1
+
 int anetTcpConnect(char *err, const char *addr, int port);
 int anetTcpNonBlockConnect(char *err, const char *addr, int port);
 int anetTcpNonBlockBindConnect(char *err, const char *addr, int port, const char *source_addr);
@@ -71,11 +75,9 @@ int anetDisableTcpNoDelay(char *err, int fd);
 int anetTcpKeepAlive(char *err, int fd);
 int anetSendTimeout(char *err, int fd, long long ms);
 int anetRecvTimeout(char *err, int fd, long long ms);
-int anetPeerToString(int fd, char *ip, size_t ip_len, int *port);
+int anetFdToString(int fd, char *ip, size_t ip_len, int *port, int fd_to_str_type);
 int anetKeepAlive(char *err, int fd, int interval);
-int anetSockName(int fd, char *ip, size_t ip_len, int *port);
 int anetFormatAddr(char *fmt, size_t fmt_len, char *ip, int port);
-int anetFormatPeer(int fd, char *fmt, size_t fmt_len);
-int anetFormatSock(int fd, char *fmt, size_t fmt_len);
+int anetFormatFdAddr(int fd, char *buf, size_t buf_len, int fd_to_str_type);
 
 #endif
diff --git a/src/aof.c b/src/aof.c
index b9072da93..0f67f0575 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -681,6 +681,7 @@ struct client *createAOFClient(void) {
     c->obuf_soft_limit_reached_time = 0;
     c->watched_keys = listCreate();
     c->peerid = NULL;
+    c->sockname = NULL;
     c->resp = 2;
     c->user = NULL;
     listSetFreeMethod(c->reply,freeClientReplyValue);
diff --git a/src/connection.c b/src/connection.c
index 83fb84d6d..a4610a42e 100644
--- a/src/connection.c
+++ b/src/connection.c
@@ -374,15 +374,15 @@ int connGetSocketError(connection *conn) {
 }
 
 int connPeerToString(connection *conn, char *ip, size_t ip_len, int *port) {
-    return anetPeerToString(conn ? conn->fd : -1, ip, ip_len, port);
-}
-
-int connFormatPeer(connection *conn, char *buf, size_t buf_len) {
-    return anetFormatPeer(conn ? conn->fd : -1, buf, buf_len);
+    return anetFdToString(conn ? conn->fd : -1, ip, ip_len, port, FD_TO_PEER_NAME);
 }
 
 int connSockName(connection *conn, char *ip, size_t ip_len, int *port) {
-    return anetSockName(conn->fd, ip, ip_len, port);
+    return anetFdToString(conn->fd, ip, ip_len, port, FD_TO_SOCK_NAME);
+}
+
+int connFormatFdAddr(connection *conn, char *buf, size_t buf_len, int fd_to_str_type) {
+    return anetFormatFdAddr(conn ? conn->fd : -1, buf, buf_len, fd_to_str_type);
 }
 
 int connBlock(connection *conn) {
diff --git a/src/connection.h b/src/connection.h
index 03281a3d9..60f1f70cb 100644
--- a/src/connection.h
+++ b/src/connection.h
@@ -225,7 +225,7 @@ int connKeepAlive(connection *conn, int interval);
 int connSendTimeout(connection *conn, long long ms);
 int connRecvTimeout(connection *conn, long long ms);
 int connPeerToString(connection *conn, char *ip, size_t ip_len, int *port);
-int connFormatPeer(connection *conn, char *buf, size_t buf_len);
+int connFormatFdAddr(connection *conn, char *buf, size_t buf_len, int fd_to_str_type);
 int connSockName(connection *conn, char *ip, size_t ip_len, int *port);
 const char *connGetInfo(connection *conn, char *buf, size_t buf_len);
 
diff --git a/src/networking.c b/src/networking.c
index fcb612e0b..4f0e378da 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -168,6 +168,7 @@ client *createClient(connection *conn) {
     c->pubsub_channels = dictCreate(&objectKeyPointerValueDictType,NULL);
     c->pubsub_patterns = listCreate();
     c->peerid = NULL;
+    c->sockname = NULL;
     c->client_list_node = NULL;
     c->client_tracking_redirection = 0;
     c->client_tracking_prefixes = NULL;
@@ -1300,6 +1301,7 @@ void freeClient(client *c) {
     c->argv_len_sum = 0;
     freeClientMultiState(c);
     sdsfree(c->peerid);
+    sdsfree(c->sockname);
     zfree(c);
 }
 
@@ -2079,25 +2081,25 @@ void getClientsMaxBuffers(unsigned long *longest_output_list,
     *biggest_input_buffer = bib;
 }
 
-/* A Redis "Peer ID" is a colon separated ip:port pair.
+/* A Redis "Address String" is a colon separated ip:port pair.
  * For IPv4 it's in the form x.y.z.k:port, example: "127.0.0.1:1234".
  * For IPv6 addresses we use [] around the IP part, like in "[::1]:1234".
  * For Unix sockets we use path:0, like in "/tmp/redis:0".
  *
- * A Peer ID always fits inside a buffer of NET_PEER_ID_LEN bytes, including
- * the null term.
+ * An Address String always fits inside a buffer of NET_ADDR_STR_LEN bytes,
+ * including the null term.
  *
- * On failure the function still populates 'peerid' with the "?:0" string
- * in case you want to relax error checking or need to display something
- * anyway (see anetPeerToString implementation for more info). */
-void genClientPeerId(client *client, char *peerid,
-                            size_t peerid_len) {
+ * On failure the function still populates 'addr' with the "?:0" string in case
+ * you want to relax error checking or need to display something anyway (see
+ * anetFdToString implementation for more info). */
+void genClientAddrString(client *client, char *addr,
+                         size_t addr_len, int fd_to_str_type) {
     if (client->flags & CLIENT_UNIX_SOCKET) {
         /* Unix socket client. */
-        snprintf(peerid,peerid_len,"%s:0",server.unixsocket);
+        snprintf(addr,addr_len,"%s:0",server.unixsocket);
     } else {
         /* TCP client. */
-        connFormatPeer(client->conn,peerid,peerid_len);
+        connFormatFdAddr(client->conn,addr,addr_len,fd_to_str_type);
     }
 }
 
@@ -2106,15 +2108,29 @@ void genClientPeerId(client *client, char *peerid,
  * The Peer ID never changes during the life of the client, however it
  * is expensive to compute. */
 char *getClientPeerId(client *c) {
-    char peerid[NET_PEER_ID_LEN];
+    char peerid[NET_ADDR_STR_LEN];
 
     if (c->peerid == NULL) {
-        genClientPeerId(c,peerid,sizeof(peerid));
+        genClientAddrString(c,peerid,sizeof(peerid),FD_TO_PEER_NAME);
         c->peerid = sdsnew(peerid);
     }
     return c->peerid;
 }
 
+/* This function returns the client bound socket name, by creating and caching
+ * it if client->sockname is NULL, otherwise returning the cached value.
+ * The Socket Name never changes during the life of the client, however it
+ * is expensive to compute. */
+char *getClientSockname(client *c) {
+    char sockname[NET_ADDR_STR_LEN];
+
+    if (c->sockname == NULL) {
+        genClientAddrString(c,sockname,sizeof(sockname),FD_TO_SOCK_NAME);
+        c->sockname = sdsnew(sockname);
+    }
+    return c->sockname;
+}
+
 /* Concatenate a string representing the state of a client in a human
  * readable format, into the sds string 's'. */
 sds catClientInfoString(sds s, client *client) {
@@ -2162,9 +2178,10 @@ sds catClientInfoString(sds s, client *client) {
         total_mem += zmalloc_size(client->argv);
 
     return sdscatfmt(s,
-        "id=%U addr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U argv-mem=%U obl=%U oll=%U omem=%U tot-mem=%U events=%s cmd=%s user=%s",
+        "id=%U addr=%s laddr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U argv-mem=%U obl=%U oll=%U omem=%U tot-mem=%U events=%s cmd=%s user=%s",
         (unsigned long long) client->id,
         getClientPeerId(client),
+        getClientSockname(client),
         connGetInfo(client->conn, conninfo, sizeof(conninfo)),
         client->name ? (char*)client->name->ptr : "",
         (long long)(server.unixtime - client->ctime),
@@ -2251,6 +2268,7 @@ void clientCommand(client *c) {
 "KILL <ip:port>         -- Kill connection made from <ip:port>.",
 "KILL <option> <value> [option value ...] -- Kill connections. Options are:",
 "     ADDR <ip:port>                      -- Kill connection made from <ip:port>",
+"     LADDR <ip:port>                     -- Kill connection made to <ip:port>",
 "     TYPE (normal|master|replica|pubsub) -- Kill connections by type.",
 "     USER <username>   -- Kill connections authenticated with such user.",
 "     SKIPME (yes|no)   -- Skip killing current connection (default: yes).",
@@ -2304,6 +2322,7 @@ NULL
         /* CLIENT KILL <ip:port>
          * CLIENT KILL <option> [value] ... <option> [value] */
         char *addr = NULL;
+        char *laddr = NULL;
         user *user = NULL;
         int type = -1;
         uint64_t id = 0;
@@ -2336,6 +2355,8 @@ NULL
                     }
                 } else if (!strcasecmp(c->argv[i]->ptr,"addr") && moreargs) {
                     addr = c->argv[i+1]->ptr;
+                } else if (!strcasecmp(c->argv[i]->ptr,"laddr") && moreargs) {
+                    laddr = c->argv[i+1]->ptr;
                 } else if (!strcasecmp(c->argv[i]->ptr,"user") && moreargs) {
                     user = ACLGetUserByName(c->argv[i+1]->ptr,
                                             sdslen(c->argv[i+1]->ptr));
@@ -2369,6 +2390,7 @@ NULL
         while ((ln = listNext(&li)) != NULL) {
             client *client = listNodeValue(ln);
             if (addr && strcmp(getClientPeerId(client),addr) != 0) continue;
+            if (laddr && strcmp(getClientSockname(client),laddr) != 0) continue;
             if (type != -1 && getClientType(client) != type) continue;
             if (id != 0 && client->id != id) continue;
             if (user && client->user != user) continue;
diff --git a/src/replication.c b/src/replication.c
index da70c4082..d1d26a970 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -57,7 +57,7 @@ int RDBGeneratedByReplication = 0;
  * IP address and its listening port which is more clear for the user, for
  * example: "Closing connection with replica 10.1.2.3:6380". */
 char *replicationGetSlaveName(client *c) {
-    static char buf[NET_PEER_ID_LEN];
+    static char buf[NET_ADDR_STR_LEN];
     char ip[NET_IP_STR_LEN];
 
     ip[0] = '\0';
@@ -2792,6 +2792,11 @@ void replicationCacheMaster(client *c) {
         sdsfree(c->peerid);
         c->peerid = NULL;
     }
+    /* Invalidate the Sock Name cache. */
+    if (c->sockname) {
+        sdsfree(c->sockname);
+        c->sockname = NULL;
+    }
 
     /* Caching the master happens instead of the actual freeClient() call,
      * so make sure to adjust the replication state. This function will
diff --git a/src/sentinel.c b/src/sentinel.c
index 71e28e1d2..87fc20e9f 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -1194,7 +1194,7 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char *
     sentinelRedisInstance *ri;
     sentinelAddr *addr;
     dict *table = NULL;
-    char slavename[NET_PEER_ID_LEN], *sdsname;
+    char slavename[NET_ADDR_STR_LEN], *sdsname;
 
     serverAssert(flags & (SRI_MASTER|SRI_SLAVE|SRI_SENTINEL));
     serverAssert((flags & SRI_MASTER) || master != NULL);
@@ -1320,7 +1320,7 @@ sentinelRedisInstance *sentinelRedisInstanceLookupSlave(
 {
     sds key;
     sentinelRedisInstance *slave;
-    char buf[NET_PEER_ID_LEN];
+    char buf[NET_ADDR_STR_LEN];
 
     serverAssert(ri->flags & SRI_MASTER);
     anetFormatAddr(buf,sizeof(buf),ip,port);
@@ -2674,7 +2674,7 @@ int sentinelSendHello(sentinelRedisInstance *ri) {
     if (sentinel.announce_ip) {
         announce_ip = sentinel.announce_ip;
     } else {
-        if (anetSockName(ri->link->cc->c.fd,ip,sizeof(ip),NULL) == -1)
+        if (anetFdToString(ri->link->cc->c.fd,ip,sizeof(ip),NULL,FD_TO_SOCK_NAME) == -1)
             return C_ERR;
         announce_ip = ip;
     }
diff --git a/src/server.h b/src/server.h
index b68d06497..b1e645b0c 100644
--- a/src/server.h
+++ b/src/server.h
@@ -112,7 +112,7 @@ typedef long long ustime_t; /* microsecond time type. */
 #define CONFIG_DEFAULT_UNIX_SOCKET_PERM 0
 #define CONFIG_DEFAULT_LOGFILE ""
 #define NET_IP_STR_LEN 46 /* INET6_ADDRSTRLEN is 46, but we need to be sure */
-#define NET_PEER_ID_LEN (NET_IP_STR_LEN+32) /* Must be enough for ip:port */
+#define NET_ADDR_STR_LEN (NET_IP_STR_LEN+32) /* Must be enough for ip:port */
 #define CONFIG_BINDADDR_MAX 16
 #define CONFIG_MIN_RESERVED_FDS 32
 
@@ -850,6 +850,7 @@ typedef struct client {
     dict *pubsub_channels;  /* channels a client is interested in (SUBSCRIBE) */
     list *pubsub_patterns;  /* patterns a client is interested in (SUBSCRIBE) */
     sds peerid;             /* Cached peer ID. */
+    sds sockname;           /* Cached connection target address. */
     listNode *client_list_node; /* list node in client list */
     RedisModuleUserChangedFunc auth_callback; /* Module callback to execute
                                                * when the authenticated user
@@ -1716,6 +1717,7 @@ void *dupClientReplyValue(void *o);
 void getClientsMaxBuffers(unsigned long *longest_output_list,
                           unsigned long *biggest_input_buffer);
 char *getClientPeerId(client *client);
+char *getClientSockName(client *client);
 sds catClientInfoString(sds s, client *client);
 sds getAllClientsInfoString(int type);
 void rewriteClientCommandVector(client *c, int argc, ...);

From b5f3996250eac3adbda407086fc519aebcb29797 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 1 Nov 2020 13:56:44 +0200
Subject: [PATCH 0770/1098] Update redis.conf to recommend use of /run. (#8002)

---
 redis.conf | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/redis.conf b/redis.conf
index 4caa70f38..c96aebea4 100644
--- a/redis.conf
+++ b/redis.conf
@@ -112,7 +112,7 @@ tcp-backlog 511
 # incoming connections. There is no default, so Redis will not listen
 # on a unix socket when not specified.
 #
-# unixsocket /tmp/redis.sock
+# unixsocket /run/redis.sock
 # unixsocketperm 700
 
 # Close the connection after a client is idle for N seconds (0 to disable)
@@ -251,6 +251,9 @@ supervised no
 #
 # Creating a pid file is best effort: if Redis is not able to create it
 # nothing bad happens, the server will start and run normally.
+#
+# Note that on modern Linux systems "/run/redis.pid" is more conforming
+# and should be used instead.
 pidfile /var/run/redis_6379.pid
 
 # Specify the server verbosity level.

From 1a91a2700b24211e90c695d3fdbbfe7e8d75dbe4 Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Mon, 2 Nov 2020 17:18:42 +0100
Subject: [PATCH 0771/1098] Modules: Improve timer accuracy (#7987)

The bug occurs when 'callback' re-registers itself to a point
in the future and the execution time in non-negligible:
'now' refers to time BEFORE callback was executed and is used
to calculate 'next_period'.
We must get the actual current time when calculating 'next_period'
---
 src/module.c | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/module.c b/src/module.c
index a1bbbf47b..65a4932b7 100644
--- a/src/module.c
+++ b/src/module.c
@@ -5442,7 +5442,10 @@ int moduleTimerHandler(struct aeEventLoop *eventLoop, long long id, void *client
             raxRemove(Timers,(unsigned char*)ri.key,ri.key_len,NULL);
             zfree(timer);
         } else {
-            next_period = (expiretime-now)/1000; /* Scale to milliseconds. */
+            /* We call ustime() again instead of using the cached 'now' so that
+             * 'next_period' isn't affected by the time it took to execute
+             * previous calls to 'callback. */
+            next_period = (expiretime-ustime())/1000; /* Scale to milliseconds. */
             break;
         }
     }
@@ -5455,7 +5458,16 @@ int moduleTimerHandler(struct aeEventLoop *eventLoop, long long id, void *client
 
 /* Create a new timer that will fire after `period` milliseconds, and will call
  * the specified function using `data` as argument. The returned timer ID can be
- * used to get information from the timer or to stop it before it fires. */
+ * used to get information from the timer or to stop it before it fires.
+ * Note that for the common use case of a repeating timer (Re-registration
+ * of the timer inside the RedisModuleTimerProc callback) it matters when
+ * this API is called:
+ * If it is called at the beginning of 'callback' it means
+ * the event will triggered every 'period'.
+ * If it is called at the end of 'callback' it means
+ * there will 'period' milliseconds gaps between events.
+ * (If the time it takes to execute 'callback' is negligible the two
+ * statements above mean the same) */
 RedisModuleTimerID RM_CreateTimer(RedisModuleCtx *ctx, mstime_t period, RedisModuleTimerProc callback, void *data) {
     RedisModuleTimer *timer = zmalloc(sizeof(*timer));
     timer->module = ctx->module;

From 9122379abc3b15dfd2af3624586cc1e3f826eb55 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 3 Nov 2020 14:56:57 +0200
Subject: [PATCH 0772/1098] Propagate GETSET and SET-GET as SET (#7957)

- Generates a more backwards compatible command stream
- Slightly more efficient execution in replica/AOF
- Add a test for coverage
---
 src/t_string.c                    | 25 ++++++++++++++++++++++++-
 tests/integration/replication.tcl | 21 +++++++++++++++++++++
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/src/t_string.c b/src/t_string.c
index 188f27dc0..e8b37663d 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -90,7 +90,7 @@ void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire,
     }
 
     if (flags & OBJ_SET_GET) {
-       getGenericCommand(c);
+        getGenericCommand(c);
     }
 
     genericSetKey(c,c->db,key,val,flags & OBJ_SET_KEEPTTL,1);
@@ -160,6 +160,24 @@ void setCommand(client *c) {
 
     c->argv[2] = tryObjectEncoding(c->argv[2]);
     setGenericCommand(c,flags,c->argv[1],c->argv[2],expire,unit,NULL,NULL);
+
+    /* Propagate without the GET argument */
+    if (flags & OBJ_SET_GET) {
+        int argc = 0;
+        robj **argv = zmalloc((c->argc-1)*sizeof(robj*));
+        for (j=0; j < c->argc; j++) {
+            char *a = c->argv[j]->ptr;
+            /* Skip GET which may be repeated multiple times. */
+            if (j >= 3 &&
+                (a[0] == 'g' || a[0] == 'G') &&
+                (a[1] == 'e' || a[1] == 'E') &&
+                (a[2] == 't' || a[2] == 'T') && a[3] == '\0')
+                continue;
+            argv[argc++] = c->argv[j];
+            incrRefCount(c->argv[j]);
+        }
+        replaceClientCommandVector(c, argc, argv);
+    }
 }
 
 void setnxCommand(client *c) {
@@ -201,6 +219,11 @@ void getsetCommand(client *c) {
     setKey(c,c->db,c->argv[1],c->argv[2]);
     notifyKeyspaceEvent(NOTIFY_STRING,"set",c->argv[1],c->db->id);
     server.dirty++;
+
+    /* Propagate as SET command */
+    robj *setcmd = createStringObject("SET",3);
+    rewriteClientCommandArgument(c,0,setcmd);
+    decrRefCount(setcmd);
 }
 
 void setrangeCommand(client *c) {
diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl
index fb0b9b965..6c437ba71 100644
--- a/tests/integration/replication.tcl
+++ b/tests/integration/replication.tcl
@@ -77,6 +77,27 @@ start_server {tags {"repl"}} {
             assert_equal [$A debug digest] [$B debug digest]
         }
 
+        test {GETSET replication} {
+            $A config resetstat
+            $A config set loglevel debug
+            $B config set loglevel debug
+            r set test foo
+            assert_equal [r getset test bar] foo
+            wait_for_condition 500 10 {
+                [$A get test] eq "bar"
+            } else {
+                fail "getset wasn't propagated"
+            }
+            assert_equal [r set test vaz get] bar
+            wait_for_condition 500 10 {
+                [$A get test] eq "vaz"
+            } else {
+                fail "set get wasn't propagated"
+            }
+            assert_match {*calls=3,*} [cmdrstat set $A]
+            assert_match {} [cmdrstat getset $A]
+        }
+
         test {BRPOPLPUSH replication, when blocking against empty list} {
             $A config resetstat
             set rd [redis_deferring_client]

From f210e197f3b9d0b4a81c6c354296da8f1ddad513 Mon Sep 17 00:00:00 2001
From: "Meir Shpilraien (Spielrein)" <meir@redislabs.com>
Date: Tue, 3 Nov 2020 14:59:21 +0200
Subject: [PATCH 0773/1098] Added crash report on SIGABRT (#8004)

The reason that we want to get a full crash report on SIGABRT
is that the jmalloc, when detecting a corruption, calls abort().
This will cause the Redis to exist silently without any report
and without any way to analyze what happened.
---
 src/debug.c                   |  6 ++++++
 src/redis-check-rdb.c         |  1 +
 src/server.c                  |  2 ++
 tests/integration/logging.tcl | 11 +++++++++++
 4 files changed, 20 insertions(+)

diff --git a/src/debug.c b/src/debug.c
index d1a30ae1b..e21440f84 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -839,6 +839,9 @@ void _serverAssert(const char *estr, const char *file, int line) {
 #endif
         printCrashReport();
     }
+
+    // remove the signal handler so on abort() we will output the crash report.
+    removeSignalHandlers();
     bugReportEnd(0, 0);
 }
 
@@ -923,6 +926,9 @@ void _serverPanic(const char *file, int line, const char *msg, ...) {
 #endif
         printCrashReport();
     }
+
+    // remove the signal handler so on abort() we will output the crash report.
+    removeSignalHandlers();
     bugReportEnd(0, 0);
 }
 
diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c
index a9d110aa8..f399864dd 100644
--- a/src/redis-check-rdb.c
+++ b/src/redis-check-rdb.c
@@ -172,6 +172,7 @@ void rdbCheckSetupSignals(void) {
     sigaction(SIGBUS, &act, NULL);
     sigaction(SIGFPE, &act, NULL);
     sigaction(SIGILL, &act, NULL);
+    sigaction(SIGABRT, &act, NULL);
 }
 
 /* Check the specified RDB file. Return 0 if the RDB looks sane, otherwise
diff --git a/src/server.c b/src/server.c
index cd45dd837..f57bf01a7 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5004,6 +5004,7 @@ void setupSignalHandlers(void) {
         sigaction(SIGBUS, &act, NULL);
         sigaction(SIGFPE, &act, NULL);
         sigaction(SIGILL, &act, NULL);
+        sigaction(SIGABRT, &act, NULL);
     }
     return;
 }
@@ -5017,6 +5018,7 @@ void removeSignalHandlers(void) {
     sigaction(SIGBUS, &act, NULL);
     sigaction(SIGFPE, &act, NULL);
     sigaction(SIGILL, &act, NULL);
+    sigaction(SIGABRT, &act, NULL);
 }
 
 /* This is the signal handler for children process. It is currently useful
diff --git a/tests/integration/logging.tcl b/tests/integration/logging.tcl
index c1f4854d4..3314f0f1d 100644
--- a/tests/integration/logging.tcl
+++ b/tests/integration/logging.tcl
@@ -22,3 +22,14 @@ if {$system_name eq {linux} || $system_name eq {darwin}} {
         }
     }
 }
+
+set server_path [tmpdir server1.log]
+start_server [list overrides [list dir $server_path]] {
+    test "Crash report generated on SIGABRT" {
+        set pid [s process_id]
+        exec kill -SIGABRT $pid
+        set pattern "*STACK TRACE*"
+        set result [exec tail -1000 < [srv 0 stdout]]
+        assert {[string match $pattern $result]}
+    }
+}

From 89c78a980807aa1de0a6d0ccde6042450333a783 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuancode@163.com>
Date: Tue, 3 Nov 2020 23:16:11 +0800
Subject: [PATCH 0774/1098] Disable rehash when redis has child process (#8007)

In redisFork(), we don't set child pid, so updateDictResizePolicy()
doesn't take effect, that isn't friendly for copy-on-write.

The bug was introduced this in redis 6.0: 56258c6
---
 src/aof.c            |  1 +
 src/module.c         |  1 +
 src/rdb.c            |  2 ++
 src/server.c         |  4 +++-
 tests/unit/other.tcl | 20 ++++++++++++++++++++
 5 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/src/aof.c b/src/aof.c
index 0f67f0575..f3176934f 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1715,6 +1715,7 @@ int rewriteAppendOnlyFileBackground(void) {
         server.aof_rewrite_scheduled = 0;
         server.aof_rewrite_time_start = time(NULL);
         server.aof_child_pid = childpid;
+        updateDictResizePolicy();
         /* We set appendseldb to -1 in order to force the next call to the
          * feedAppendOnlyFile() to issue a SELECT command, so the differences
          * accumulated by the parent into server.aof_rewrite_buf will start
diff --git a/src/module.c b/src/module.c
index 65a4932b7..b4a15acae 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7001,6 +7001,7 @@ int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) {
         server.module_child_pid = childpid;
         moduleForkInfo.done_handler = cb;
         moduleForkInfo.done_handler_user_data = user_data;
+        updateDictResizePolicy();
         serverLog(LL_VERBOSE, "Module fork started pid: %d ", childpid);
     }
     return childpid;
diff --git a/src/rdb.c b/src/rdb.c
index db5698c1d..3ea55fa67 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1412,6 +1412,7 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
         server.rdb_save_time_start = time(NULL);
         server.rdb_child_pid = childpid;
         server.rdb_child_type = RDB_CHILD_TYPE_DISK;
+        updateDictResizePolicy();
         return C_OK;
     }
     return C_OK; /* unreached */
@@ -2618,6 +2619,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
             server.rdb_save_time_start = time(NULL);
             server.rdb_child_pid = childpid;
             server.rdb_child_type = RDB_CHILD_TYPE_SOCKET;
+            updateDictResizePolicy();
             close(rdb_pipe_write); /* close write in parent so that it can detect the close on the child. */
             if (aeCreateFileEvent(server.el, server.rdb_pipe_read, AE_READABLE, rdbPipeReadHandler,NULL) == AE_ERR) {
                 serverPanic("Unrecoverable error creating server.rdb_pipe_read file event.");
diff --git a/src/server.c b/src/server.c
index f57bf01a7..ffe2b6be6 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2052,6 +2052,9 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
             }
         }
     }
+    /* Just for the sake of defensive programming, to avoid forgeting to
+     * call this function when need. */
+    updateDictResizePolicy();
 
 
     /* AOF postponed flush: Try at every cron cycle if the slow fsync
@@ -5072,7 +5075,6 @@ int redisFork(int purpose) {
         if (childpid == -1) {
             return -1;
         }
-        updateDictResizePolicy();
     }
     return childpid;
 }
diff --git a/tests/unit/other.tcl b/tests/unit/other.tcl
index 20a32e795..3ca70a77a 100644
--- a/tests/unit/other.tcl
+++ b/tests/unit/other.tcl
@@ -250,3 +250,23 @@ start_server {tags {"other"}} {
         r save
     } {OK}
 }
+
+start_server {tags {"other"}} {
+    test {Don't rehash if redis has child proecess} {
+        r config set save ""
+        r config set rdb-key-save-delay 1000000
+
+        populate 4096 "" 1
+        r bgsave
+        r mset k1 v1 k2 v2
+        # Hash table should not rehash
+        assert_no_match "*table size: 8192*" [r debug HTSTATS 9]
+        exec kill -9 [get_child_pid 0]
+        after 200
+
+        # Hash table should rehash since there is no child process,
+        # size is power of two and over 4098, so it is 16384
+        r set k3 v3
+        assert_match "*table size: 16384*" [r debug HTSTATS 9]
+    }
+}

From a698a6391a4a71cb695478c7448fe9613167a369 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 4 Nov 2020 09:53:43 +0200
Subject: [PATCH 0775/1098] Add maxclients and cluster_connections to INFO
 CLIENTS (#7979)

Few config settings are also reflected by the INFO command.
these are mainly ones that are important for either an instant view of
the server status (to compare a metric to it's limit config),
Important configurations that are necessary in the crash log (which
currently doesn't print the config),
And things that are important for monitoring solutions (such as
Prometheus), which rely on INFO to collect their data.

Add cluster_connections to INFO CLUSTER:
This makes it possible to be combined together with connected_clients
and connected_slaves and be matched against maxclients
---
 src/server.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/server.c b/src/server.c
index ffe2b6be6..a5aa64c77 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4295,12 +4295,16 @@ sds genRedisInfoString(const char *section) {
         info = sdscatprintf(info,
             "# Clients\r\n"
             "connected_clients:%lu\r\n"
+            "cluster_connections:%lu\r\n"
+            "maxclients:%u\r\n"
             "client_recent_max_input_buffer:%zu\r\n"
             "client_recent_max_output_buffer:%zu\r\n"
             "blocked_clients:%d\r\n"
             "tracking_clients:%d\r\n"
             "clients_in_timeout_table:%llu\r\n",
             listLength(server.clients)-listLength(server.slaves),
+            getClusterConnectionsCount(),
+            server.maxclients,
             maxin, maxout,
             server.blocked_clients,
             server.tracking_clients,

From 639b73cd2ad906aad583ee404ac4b2f8b95a9c34 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Wed, 4 Nov 2020 03:00:18 -0500
Subject: [PATCH 0776/1098] redis-cli cluster import support source and target
 that need auth (#7994)

Make it possible for redis-cli cluster import to work with source and
target that require AUTH.

Adding two different flags --cluster-from-user, --cluster-from-pass
and --cluster-askpass for source node authentication.
Also for target authentication, using existing --user and --pass flag.

Example:

./redis-cli --cluster import 127.0.0.1:7000 --cluster-from 127.0.0.1:6379 --pass 1234 --user default --cluster-from-user default --cluster-from-pass 123456

./redis-cli --cluster import 127.0.0.1:7000 --cluster-from 127.0.0.1:6379 --askpass --cluster-from-user default --cluster-from-askpass
---
 src/redis-cli.c | 56 +++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 45 insertions(+), 11 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 50edc0c0e..8b5ab4136 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -189,6 +189,9 @@ typedef struct clusterManagerCommand {
     int pipeline;
     float threshold;
     char *backup_dir;
+    char *from_user;
+    char *from_pass;
+    int from_askpass;
 } clusterManagerCommand;
 
 static void createClusterManagerCommand(char *cmdname, int argc, char **argv);
@@ -753,14 +756,14 @@ static void freeHintsCallback(void *ptr) {
  *--------------------------------------------------------------------------- */
 
 /* Send AUTH command to the server */
-static int cliAuth(void) {
+static int cliAuth(redisContext *ctx, char *user, char *auth) {
     redisReply *reply;
-    if (config.auth == NULL) return REDIS_OK;
+    if (auth == NULL) return REDIS_OK;
 
-    if (config.user == NULL)
-        reply = redisCommand(context,"AUTH %s",config.auth);
+    if (user == NULL)
+        reply = redisCommand(ctx,"AUTH %s",auth);
     else
-        reply = redisCommand(context,"AUTH %s %s",config.user,config.auth);
+        reply = redisCommand(ctx,"AUTH %s %s",user,auth);
     if (reply != NULL) {
         if (reply->type == REDIS_REPLY_ERROR)
             fprintf(stderr,"Warning: AUTH failed\n");
@@ -849,7 +852,7 @@ static int cliConnect(int flags) {
         anetKeepAlive(NULL, context->fd, REDIS_CLI_KEEPALIVE_INTERVAL);
 
         /* Do AUTH, select the right DB, switch to RESP3 if needed. */
-        if (cliAuth() != REDIS_OK)
+        if (cliAuth(context, config.user, config.auth) != REDIS_OK)
             return REDIS_ERR;
         if (cliSelect() != REDIS_OK)
             return REDIS_ERR;
@@ -1574,6 +1577,12 @@ static int parseOptions(int argc, char **argv) {
             config.cluster_manager_command.from = argv[++i];
         } else if (!strcmp(argv[i],"--cluster-to") && !lastarg) {
             config.cluster_manager_command.to = argv[++i];
+        } else if (!strcmp(argv[i],"--cluster-from-user") && !lastarg) {
+            config.cluster_manager_command.from_user = argv[++i];
+        } else if (!strcmp(argv[i],"--cluster-from-pass") && !lastarg) {
+            config.cluster_manager_command.from_pass = argv[++i];
+        } else if (!strcmp(argv[i], "--cluster-from-askpass")) {
+            config.cluster_manager_command.from_askpass = 1;
         } else if (!strcmp(argv[i],"--cluster-weight") && !lastarg) {
             if (config.cluster_manager_command.weight != NULL) {
                 fprintf(stderr, "WARNING: you cannot use --cluster-weight "
@@ -2360,7 +2369,7 @@ clusterManagerCommandDef clusterManagerCommands[] = {
     {"set-timeout", clusterManagerCommandSetTimeout, 2,
      "host:port milliseconds", NULL},
     {"import", clusterManagerCommandImport, 1, "host:port",
-     "from <arg>,copy,replace"},
+     "from <arg>,from-user <arg>,from-pass <arg>,from-askpass,copy,replace"},
     {"backup", clusterManagerCommandBackup, 2,  "host:port backup_directory",
      NULL},
     {"help", clusterManagerCommandHelp, 0, NULL, NULL}
@@ -6303,6 +6312,7 @@ static int clusterManagerCommandImport(int argc, char **argv) {
     int port = 0, src_port = 0;
     char *ip = NULL, *src_ip = NULL;
     char *invalid_args_msg = NULL;
+    sds cmdfmt = NULL;
     if (!getClusterHostFromCmdArgs(argc, argv, &ip, &port)) {
         invalid_args_msg = CLUSTER_MANAGER_INVALID_HOST_ARG;
         goto invalid_args;
@@ -6334,6 +6344,14 @@ static int clusterManagerCommandImport(int argc, char **argv) {
                 src_port, src_ctx->errstr);
         goto cleanup;
     }
+    // Auth for the source node. 
+    char *from_user = config.cluster_manager_command.from_user;
+    char *from_pass = config.cluster_manager_command.from_pass;
+    if (cliAuth(src_ctx, from_user, from_pass) == REDIS_ERR) {
+        success = 0;
+        goto cleanup;
+    }
+
     src_reply = reconnectingRedisCommand(src_ctx, "INFO");
     if (!src_reply || src_reply->type == REDIS_REPLY_ERROR) {
         if (src_reply && src_reply->str) reply_err = src_reply->str;
@@ -6373,8 +6391,15 @@ static int clusterManagerCommandImport(int argc, char **argv) {
             }
         }
     }
+    cmdfmt = sdsnew("MIGRATE %s %d %s %d %d");
+    if (config.auth) {
+        if (config.user) {
+            cmdfmt = sdscatfmt(cmdfmt," AUTH2 %s %s", config.user, config.auth); 
+        } else {
+            cmdfmt = sdscatfmt(cmdfmt," AUTH %s", config.auth);
+        }
+    }
 
-    char cmdfmt[50] = "MIGRATE %s %d %s %d %d";
     if (config.cluster_manager_command.flags & CLUSTER_MANAGER_CMD_FLAG_COPY)
         strcat(cmdfmt, " %s");
     if (config.cluster_manager_command.flags & CLUSTER_MANAGER_CMD_FLAG_REPLACE)
@@ -6431,6 +6456,7 @@ cleanup:
                              src_ip, src_port, reply_err);
     if (src_ctx) redisFree(src_ctx);
     if (src_reply) freeReplyObject(src_reply);
+    if (cmdfmt) sdsfree(cmdfmt);
     return success;
 invalid_args:
     fprintf(stderr, "%s", invalid_args_msg);
@@ -8000,9 +8026,9 @@ static void intrinsicLatencyMode(void) {
     }
 }
 
-static sds askPassword() {
+static sds askPassword(const char *msg) {
     linenoiseMaskModeEnable();
-    sds auth = linenoise("Please input password: ");
+    sds auth = linenoise(msg);
     linenoiseMaskModeDisable();
     return auth;
 }
@@ -8060,6 +8086,9 @@ int main(int argc, char **argv) {
     config.cluster_manager_command.replicas = 0;
     config.cluster_manager_command.from = NULL;
     config.cluster_manager_command.to = NULL;
+    config.cluster_manager_command.from_user = NULL;
+    config.cluster_manager_command.from_pass = NULL;
+    config.cluster_manager_command.from_askpass = 0;
     config.cluster_manager_command.weight = NULL;
     config.cluster_manager_command.weight_argc = 0;
     config.cluster_manager_command.slots = 0;
@@ -8090,7 +8119,12 @@ int main(int argc, char **argv) {
     parseEnv();
 
     if (config.askpass) {
-        config.auth = askPassword();
+        config.auth = askPassword("Please input password: ");
+    }
+
+    if (config.cluster_manager_command.from_askpass) {
+        config.cluster_manager_command.from_pass = askPassword(
+            "Please input import source node password: ");
     }
 
 #ifdef USE_OPENSSL

From 762be79f0bc26366fbc3ac16e33d3baa854fbcf1 Mon Sep 17 00:00:00 2001
From: "Meir Shpilraien (Spielrein)" <meir@redislabs.com>
Date: Wed, 4 Nov 2020 13:13:55 +0200
Subject: [PATCH 0777/1098] Disable new SIGABRT test on valgrind (#8013)

The crash reports cause false-positive warnings when run with valgrind.
---
 tests/integration/logging.tcl | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/tests/integration/logging.tcl b/tests/integration/logging.tcl
index 3314f0f1d..fd9034644 100644
--- a/tests/integration/logging.tcl
+++ b/tests/integration/logging.tcl
@@ -1,7 +1,7 @@
-set server_path [tmpdir server.log]
 set system_name [string tolower [exec uname -s]]
 
 if {$system_name eq {linux} || $system_name eq {darwin}} {
+    set server_path [tmpdir server.log]
     start_server [list overrides [list dir $server_path]] {
         test "Server is able to generate a stack trace on selected systems" {
             r config set watchdog-period 200
@@ -21,15 +21,19 @@ if {$system_name eq {linux} || $system_name eq {darwin}} {
             }
         }
     }
-}
 
-set server_path [tmpdir server1.log]
-start_server [list overrides [list dir $server_path]] {
-    test "Crash report generated on SIGABRT" {
-        set pid [s process_id]
-        exec kill -SIGABRT $pid
-        set pattern "*STACK TRACE*"
-        set result [exec tail -1000 < [srv 0 stdout]]
-        assert {[string match $pattern $result]}
+    # Valgrind will complain that the process terminated by a signal, skip it.
+    if {!$::valgrind} {
+        set server_path [tmpdir server1.log]
+        start_server [list overrides [list dir $server_path]] {
+            test "Crash report generated on SIGABRT" {
+                set pid [s process_id]
+                exec kill -SIGABRT $pid
+                set pattern "*STACK TRACE*"
+                set result [exec tail -1000 < [srv 0 stdout]]
+                assert {[string match $pattern $result]}
+            }
+        }
     }
+
 }

From f4ca3d8757d6abb3536610ddb7b9ab3ad39e81df Mon Sep 17 00:00:00 2001
From: Egor Seredin <4819888+agmt@users.noreply.github.com>
Date: Wed, 4 Nov 2020 20:38:46 +0900
Subject: [PATCH 0778/1098] Allow '\0' inside of result of sdscatvprintf, and
 efficiency improvements (#6260)

This will allow to use: RedisModule_CreateStringPrintf(ctx, "%s %c %s", "string1", 0, "string2");

On large string, the previous code would incrementally retry to double the output buffer.
now it uses the the return value of snprintf and grows to the right size in one step.

and also avoids an excessive strlen in sdscat at the end.
---
 src/sds.c | 32 ++++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/src/sds.c b/src/sds.c
index 4cbbabfbb..f72eac921 100644
--- a/src/sds.c
+++ b/src/sds.c
@@ -545,6 +545,7 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
     va_list cpy;
     char staticbuf[1024], *buf = staticbuf, *t;
     size_t buflen = strlen(fmt)*2;
+    int bufstrlen;
 
     /* We try to start using a static buffer for speed.
      * If not possible we revert to heap allocation. */
@@ -555,16 +556,19 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
         buflen = sizeof(staticbuf);
     }
 
-    /* Try with buffers two times bigger every time we fail to
+    /* Alloc enough space for buffer and \0 after failing to
      * fit the string in the current buffer size. */
     while(1) {
-        buf[buflen-2] = '\0';
         va_copy(cpy,ap);
-        vsnprintf(buf, buflen, fmt, cpy);
+        bufstrlen = vsnprintf(buf, buflen, fmt, cpy);
         va_end(cpy);
-        if (buf[buflen-2] != '\0') {
+        if (bufstrlen < 0) {
             if (buf != staticbuf) s_free(buf);
-            buflen *= 2;
+            return NULL;
+        }
+        if (((size_t)bufstrlen) >= buflen) {
+            if (buf != staticbuf) s_free(buf);
+            buflen = ((size_t)bufstrlen) + 1;
             buf = s_malloc(buflen);
             if (buf == NULL) return NULL;
             continue;
@@ -573,7 +577,7 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
     }
 
     /* Finally concat the obtained string to the SDS string and return it. */
-    t = sdscat(s, buf);
+    t = sdscatlen(s, buf, bufstrlen);
     if (buf != staticbuf) s_free(buf);
     return t;
 }
@@ -1182,6 +1186,22 @@ int sdsTest(int argc, char **argv) {
         test_cond("sdscatprintf() seems working in the base case",
             sdslen(x) == 3 && memcmp(x,"123\0",4) == 0)
 
+        sdsfree(x);
+        x = sdscatprintf(sdsempty(),"a%cb",0);
+        test_cond("sdscatprintf() seems working with \\0 inside of result",
+            sdslen(x) == 3 && memcmp(x,"a\0""b\0",4) == 0)
+
+        {
+            sdsfree(x);
+            char etalon[1024*1024];
+            for (size_t i = 0; i < sizeof(etalon); i++) {
+                etalon[i] = '0';
+            }
+            x = sdscatprintf(sdsempty(),"%0*d",(int)sizeof(etalon),0);
+            test_cond("sdscatprintf() can print 1MB",
+                sdslen(x) == sizeof(etalon) && memcmp(x,etalon,sizeof(etalon)) == 0)
+        }
+
         sdsfree(x);
         x = sdsnew("--");
         x = sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX);

From 10b50069344120c1d46418838bfc2de38b65ca4a Mon Sep 17 00:00:00 2001
From: filipe oliveira <filipecosta.90@gmail.com>
Date: Wed, 4 Nov 2020 12:49:15 +0000
Subject: [PATCH 0779/1098] Enable specifying TLS ciphers(suites) in
 redis-cli/redis-benchmark (#8005)

Enable specifying the preferred ciphers and/or ciphersuites for redis-cli/redis-benchmark.

Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
---
 src/cli_common.c                      | 11 ++++++-
 src/cli_common.h                      |  4 +++
 src/redis-benchmark.c                 | 17 +++++++++++
 src/redis-cli.c                       | 15 ++++++++++
 tests/integration/redis-benchmark.tcl | 41 +++++++++++++++++++++++++++
 5 files changed, 87 insertions(+), 1 deletion(-)

diff --git a/src/cli_common.c b/src/cli_common.c
index c45e3de96..c2db9fffc 100644
--- a/src/cli_common.c
+++ b/src/cli_common.c
@@ -53,7 +53,6 @@ int cliSecureConnection(redisContext *c, cliSSLconfig config, const char **err)
             *err = "Failed to create SSL_CTX";
             goto error;
         }
-
         SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3);
         SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL);
 
@@ -78,6 +77,16 @@ int cliSecureConnection(redisContext *c, cliSSLconfig config, const char **err)
             *err = "Invalid private key";
             goto error;
         }
+        if (config.ciphers && !SSL_CTX_set_cipher_list(ssl_ctx, config.ciphers)) {
+            *err = "Error while configuring ciphers";
+            goto error;
+        }
+#ifdef TLS1_3_VERSION
+        if (config.ciphersuites && !SSL_CTX_set_ciphersuites(ssl_ctx, config.ciphersuites)) {
+            *err = "Error while setting cypher suites";
+            goto error;
+        }
+#endif
     }
 
     SSL *ssl = SSL_new(ssl_ctx);
diff --git a/src/cli_common.h b/src/cli_common.h
index 700a834ce..f3a91e9db 100644
--- a/src/cli_common.h
+++ b/src/cli_common.h
@@ -14,6 +14,10 @@ typedef struct cliSSLconfig {
     char *cert;
     /* Private key file to authenticate with, or NULL */
     char *key;
+    /* Prefered cipher list, or NULL (applies only to <= TLSv1.2) */
+    char* ciphers;
+    /* Prefered ciphersuites list, or NULL (applies only to TLSv1.3) */
+    char* ciphersuites;
 } cliSSLconfig;
 
 /* Wrapper around redisSecureConnection to avoid hiredis_ssl dependencies if
diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 9f27ad804..0ec9cb1eb 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -1520,6 +1520,14 @@ int parseOptions(int argc, const char **argv) {
         } else if (!strcmp(argv[i],"--key")) {
             if (lastarg) goto invalid;
             config.sslconfig.key = strdup(argv[++i]);
+        } else if (!strcmp(argv[i],"--tls-ciphers")) {
+            if (lastarg) goto invalid;
+            config.sslconfig.ciphers = strdup(argv[++i]);
+        #ifdef TLS1_3_VERSION
+        } else if (!strcmp(argv[i],"--tls-ciphersuites")) {
+            if (lastarg) goto invalid;
+            config.sslconfig.ciphersuites = strdup(argv[++i]);
+        #endif
         #endif
         } else {
             /* Assume the user meant to provide an option when the arg starts
@@ -1577,6 +1585,15 @@ usage:
 "                    system-wide trusted root certs configuration will apply.\n"
 " --cert <file>      Client certificate to authenticate with.\n"
 " --key <file>       Private key file to authenticate with.\n"
+" --tls-ciphers <list> Sets the list of prefered ciphers (TLSv1.2 and below)\n"
+"                    in order of preference from highest to lowest separated by colon (\":\").\n"
+"                    See the ciphers(1ssl) manpage for more information about the syntax of this string.\n"
+#ifdef TLS1_3_VERSION
+" --tls-ciphersuites <list> Sets the list of prefered ciphersuites (TLSv1.3)\n"
+"                    in order of preference from highest to lowest separated by colon (\":\").\n"
+"                    See the ciphers(1ssl) manpage for more information about the syntax of this string,\n"
+"                    and specifically for TLSv1.3 ciphersuites.\n"
+#endif
 #endif
 " --help             Output this help and exit.\n"
 " --version          Output version and exit.\n\n"
diff --git a/src/redis-cli.c b/src/redis-cli.c
index 8b5ab4136..9c8d0b063 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -1650,6 +1650,12 @@ static int parseOptions(int argc, char **argv) {
             config.sslconfig.cert = argv[++i];
         } else if (!strcmp(argv[i],"--key") && !lastarg) {
             config.sslconfig.key = argv[++i];
+        } else if (!strcmp(argv[i],"--tls-ciphers") && !lastarg) {
+            config.sslconfig.ciphers = argv[++i];
+        #ifdef TLS1_3_VERSION
+        } else if (!strcmp(argv[i],"--tls-ciphersuites") && !lastarg) {
+            config.sslconfig.ciphersuites = argv[++i];
+        #endif
 #endif
         } else if (!strcmp(argv[i],"-v") || !strcmp(argv[i], "--version")) {
             sds version = cliVersion();
@@ -1772,6 +1778,15 @@ static void usage(void) {
 "                     system-wide trusted root certs configuration will apply.\n"
 "  --cert <file>      Client certificate to authenticate with.\n"
 "  --key <file>       Private key file to authenticate with.\n"
+"  --tls-ciphers <list> Sets the list of prefered ciphers (TLSv1.2 and below)\n"
+"                     in order of preference from highest to lowest separated by colon (\":\").\n"
+"                     See the ciphers(1ssl) manpage for more information about the syntax of this string.\n"
+#ifdef TLS1_3_VERSION
+"  --tls-ciphersuites <list> Sets the list of prefered ciphersuites (TLSv1.3)\n"
+"                     in order of preference from highest to lowest separated by colon (\":\").\n"
+"                     See the ciphers(1ssl) manpage for more information about the syntax of this string,\n"
+"                     and specifically for TLSv1.3 ciphersuites.\n"
+#endif
 #endif
 "  --raw              Use raw formatting for replies (default when STDOUT is\n"
 "                     not a tty).\n"
diff --git a/tests/integration/redis-benchmark.tcl b/tests/integration/redis-benchmark.tcl
index 8abeabf0b..4a4be2ebc 100644
--- a/tests/integration/redis-benchmark.tcl
+++ b/tests/integration/redis-benchmark.tcl
@@ -123,5 +123,46 @@ start_server {tags {"benchmark"}} {
             # ensure the keyspace has the desired size
             assert_match  {50} [scan [regexp -inline {keys\=([\d]*)} [r info keyspace]] keys=%d]
         }
+
+        # tls specific tests
+        if {$::tls} {
+            test {benchmark: specific tls-ciphers} {
+                r flushall
+                r config resetstat
+                set cmd [redisbenchmark $master_host $master_port "-r 50 -t set -n 1000 --tls-ciphers \"DEFAULT:-AES128-SHA256\""]
+                if {[catch { exec {*}$cmd } error]} {
+                    set first_line [lindex [split $error "\n"] 0]
+                    puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
+                    fail "redis-benchmark non zero code. first line: $first_line"
+                }
+                assert_match  {*calls=1000,*} [cmdstat set]
+                # assert one of the non benchmarked commands is not present
+                assert_match  {} [cmdstat get]
+            }
+
+            test {benchmark: specific tls-ciphersuites} {
+                r flushall
+                r config resetstat
+                set ciphersuites_supported 1
+                set cmd [redisbenchmark $master_host $master_port "-r 50 -t set -n 1000 --tls-ciphersuites \"TLS_AES_128_GCM_SHA256\""]
+                if {[catch { exec {*}$cmd } error]} {
+                    set first_line [lindex [split $error "\n"] 0]
+                    if {[string match "*Invalid option*" $first_line]} {
+                        set ciphersuites_supported 0
+                        if {$::verbose} {
+                            puts "Skipping test, TLSv1.3 not supported."
+                        }
+                    } else {
+                        puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
+                        fail "redis-benchmark non zero code. first line: $first_line"
+                    }
+                }
+                if {$ciphersuites_supported} {
+                    assert_match  {*calls=1000,*} [cmdstat set]
+                    # assert one of the non benchmarked commands is not present
+                    assert_match  {} [cmdstat get]
+                }
+            }
+        }
     }
 }

From d8fbd3a8bafc9b430da6c3c13bdd04af4d75ad0c Mon Sep 17 00:00:00 2001
From: Tomasz Poradowski <tomasz@poradowski.net>
Date: Wed, 4 Nov 2020 15:19:18 +0100
Subject: [PATCH 0780/1098] rdb: corrected RedisModuleIO initialization point
 (#8014)

- rdbSaveSingleModuleAux() used RedisModuleIO's "bytes" field for
  tracking written bytes before calling moduleInitIOContext() which sets
  "bytes" to zero
- rdbSaveObject() re-initialized RedisModuleIO too late

This return value is not used at the moment since it's only tested
against -1, and the actual byte count isn't used yet.

Co-authored-by: Tomasz Poradowski <tomasz.poradowski@generiscorp.com>
---
 src/rdb.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/rdb.c b/src/rdb.c
index 3ea55fa67..e15984e44 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1015,10 +1015,10 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key) {
          * to call the right module during loading. */
         int retval = rdbSaveLen(rdb,mt->id);
         if (retval == -1) return -1;
+        moduleInitIOContext(io,mt,rdb,key);
         io.bytes += retval;
 
         /* Then write the module-specific representation + EOF marker. */
-        moduleInitIOContext(io,mt,rdb,key);
         mt->rdb_save(&io,mv->value);
         retval = rdbSaveLen(rdb,RDB_MODULE_OPCODE_EOF);
         if (retval == -1)
@@ -1147,6 +1147,7 @@ ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt) {
     RedisModuleIO io;
     int retval = rdbSaveType(rdb, RDB_OPCODE_MODULE_AUX);
     if (retval == -1) return -1;
+    moduleInitIOContext(io,mt,rdb,NULL);
     io.bytes += retval;
 
     /* Write the "module" identifier as prefix, so that we'll be able
@@ -1166,7 +1167,6 @@ ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt) {
     io.bytes += retval;
 
     /* Then write the module-specific representation + EOF marker. */
-    moduleInitIOContext(io,mt,rdb,NULL);
     mt->aux_save(&io,when);
     retval = rdbSaveLen(rdb,RDB_MODULE_OPCODE_EOF);
     if (retval == -1)

From 2faa0f19eb3030b019057e5db9e6226ee4c7d7de Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 4 Nov 2020 17:47:57 +0200
Subject: [PATCH 0781/1098] Fix test failure on slower systems.

Not disabling save, slower systems begun background save that did not
complete in time, resulting with SAVE failing with "ERR Background save
already in progress".
---
 tests/unit/memefficiency.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl
index a81d33d9b..6dee77bed 100644
--- a/tests/unit/memefficiency.tcl
+++ b/tests/unit/memefficiency.tcl
@@ -395,7 +395,7 @@ start_server {tags {"defrag"} overrides {appendonly yes auto-aof-rewrite-percent
             # if the current slab is lower in utilization the defragger would have ended up in stagnation,
             # keept running and not move any allocation.
             # this test is more consistent on a fresh server with no history
-            start_server {tags {"defrag"}} {
+            start_server {tags {"defrag"} overrides {save ""}} {
                 r flushdb
                 r config resetstat
                 r config set hz 100

From f6546eff452e9868569610c797765db8f56763e3 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 4 Nov 2020 17:50:31 +0200
Subject: [PATCH 0782/1098] Tests: fix filename reported in run_solo tests.

---
 tests/test_helper.tcl | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 37ab23d0c..6a38cc001 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -133,7 +133,8 @@ proc execute_test_file name {
 # as argument, and an associated name.
 # It will run the specified code and signal it to the test server when
 # finished.
-proc execute_test_code {name code} {
+proc execute_test_code {name filename code} {
+    set ::curfile $filename
     eval $code
     send_data_packet $::test_server_fd done "$name"
 }
@@ -238,7 +239,7 @@ proc run_solo {name code} {
         eval $code
         return
     }
-    send_data_packet $::test_server_fd run_solo [list $name $code]
+    send_data_packet $::test_server_fd run_solo [list $name $::curfile $code]
 }
 
 proc cleanup {} {
@@ -507,8 +508,8 @@ proc test_client_main server_port {
         if {$cmd eq {run}} {
             execute_test_file $data
         } elseif {$cmd eq {run_code}} {
-            foreach {name code} $data break
-            execute_test_code $name $code
+            foreach {name filename code} $data break
+            execute_test_code $name $filename $code
         } else {
             error "Unknown test client command: $cmd"
         }

From 1fd456f91a456aa5243973b4ba2fc77fbd3bf505 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 5 Nov 2020 10:51:26 +0200
Subject: [PATCH 0783/1098] Add RESET command. (#7982)

Perform full reset of all client connection states, is if the client was
disconnected and re-connected. This affects:

* MULTI state
* Watched keys
* MONITOR mode
* Pub/Sub subscription
* ACL/Authenticated state
* Client tracking state
* Cluster read-only/asking state
* RESP version (reset to 2)
* Selected database
* CLIENT REPLY state

The response is +RESET to make it easily distinguishable from other
responses.

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: Itamar Haber <itamar@redislabs.com>
---
 src/networking.c     | 60 ++++++++++++++++++++++++++++++++++++++++----
 src/server.c         | 17 +++++++++----
 src/server.h         |  1 +
 tests/unit/other.tcl | 51 +++++++++++++++++++++++++++++++++++++
 4 files changed, 119 insertions(+), 10 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 4f0e378da..8aca33e85 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -97,6 +97,16 @@ void linkClient(client *c) {
     raxInsert(server.clients_index,(unsigned char*)&id,sizeof(id),c,NULL);
 }
 
+/* Initialize client authentication state.
+ */
+static void clientSetDefaultAuth(client *c) {
+    /* If the default user does not require authentication, the user is
+     * directly authenticated. */
+    c->user = DefaultUser;
+    c->authenticated = (c->user->flags & USER_FLAG_NOPASS) &&
+                       !(c->user->flags & USER_FLAG_DISABLED);
+}
+
 client *createClient(connection *conn) {
     client *c = zmalloc(sizeof(client));
 
@@ -130,16 +140,12 @@ client *createClient(connection *conn) {
     c->argv = NULL;
     c->argv_len_sum = 0;
     c->cmd = c->lastcmd = NULL;
-    c->user = DefaultUser;
     c->multibulklen = 0;
     c->bulklen = -1;
     c->sentlen = 0;
     c->flags = 0;
     c->ctime = c->lastinteraction = server.unixtime;
-    /* If the default user does not require authentication, the user is
-     * directly authenticated. */
-    c->authenticated = (c->user->flags & USER_FLAG_NOPASS) &&
-                       !(c->user->flags & USER_FLAG_DISABLED);
+    clientSetDefaultAuth(c);
     c->replstate = REPL_STATE_NONE;
     c->repl_put_online_on_ack = 0;
     c->reploff = 0;
@@ -2257,6 +2263,50 @@ int clientSetNameOrReply(client *c, robj *name) {
     return C_OK;
 }
 
+/* Reset the client state to resemble a newly connected client.
+ */
+void resetCommand(client *c) {
+    listNode *ln;
+
+    /* MONITOR clients are also marked with CLIENT_SLAVE, we need to
+     * distinguish between the two.
+     */
+    if (c->flags & CLIENT_MONITOR) {
+        ln = listSearchKey(server.monitors,c);
+        serverAssert(ln != NULL);
+        listDelNode(server.monitors,ln);
+
+        c->flags &= ~(CLIENT_MONITOR|CLIENT_SLAVE);
+    }
+
+    if (c->flags & (CLIENT_SLAVE|CLIENT_MASTER|CLIENT_MODULE)) {
+        addReplyError(c,"can only reset normal client connections");
+        return;
+    }
+
+    if (c->flags & CLIENT_TRACKING) disableTracking(c);
+    selectDb(c,0);
+    c->resp = 2;
+
+    clientSetDefaultAuth(c);
+    moduleNotifyUserChanged(c);
+    discardTransaction(c);
+
+    pubsubUnsubscribeAllChannels(c,0);
+    pubsubUnsubscribeAllPatterns(c,0);
+
+    if (c->name) {
+        decrRefCount(c->name);
+        c->name = NULL;
+    }
+
+    /* Selectively clear state flags not covered above */
+    c->flags &= ~(CLIENT_ASKING|CLIENT_READONLY|CLIENT_PUBSUB|
+            CLIENT_REPLY_OFF|CLIENT_REPLY_SKIP_NEXT);
+
+    addReplyStatus(c,"RESET");
+}
+
 void clientCommand(client *c) {
     listNode *ln;
     listIter li;
diff --git a/src/server.c b/src/server.c
index a5aa64c77..9aab1f39b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1036,7 +1036,11 @@ struct redisCommand redisCommandTable[] = {
 
     {"stralgo",stralgoCommand,-2,
      "read-only @string",
-     0,lcsGetKeys,0,0,0,0,0,0}
+     0,lcsGetKeys,0,0,0,0,0,0},
+
+    {"reset",resetCommand,-1,
+     "no-script ok-stale ok-loading fast @connection",
+     0,NULL,0,0,0,0,0,0}
 };
 
 /*============================ Utility functions ============================ */
@@ -3740,7 +3744,8 @@ int processCommand(client *c) {
          * set. */
         if (c->flags & CLIENT_MULTI &&
             c->cmd->proc != execCommand &&
-            c->cmd->proc != discardCommand) {
+            c->cmd->proc != discardCommand &&
+            c->cmd->proc != resetCommand) {
             reject_cmd_on_oom = 1;
         }
 
@@ -3806,10 +3811,11 @@ int processCommand(client *c) {
         c->cmd->proc != subscribeCommand &&
         c->cmd->proc != unsubscribeCommand &&
         c->cmd->proc != psubscribeCommand &&
-        c->cmd->proc != punsubscribeCommand) {
+        c->cmd->proc != punsubscribeCommand &&
+        c->cmd->proc != resetCommand) {
         rejectCommandFormat(c,
             "Can't execute '%s': only (P)SUBSCRIBE / "
-            "(P)UNSUBSCRIBE / PING / QUIT are allowed in this context",
+            "(P)UNSUBSCRIBE / PING / QUIT / RESET are allowed in this context",
             c->cmd->name);
         return C_OK;
     }
@@ -3860,7 +3866,8 @@ int processCommand(client *c) {
     /* Exec the command */
     if (c->flags & CLIENT_MULTI &&
         c->cmd->proc != execCommand && c->cmd->proc != discardCommand &&
-        c->cmd->proc != multiCommand && c->cmd->proc != watchCommand)
+        c->cmd->proc != multiCommand && c->cmd->proc != watchCommand &&
+        c->cmd->proc != resetCommand)
     {
         queueMultiCommand(c);
         addReply(c,shared.queued);
diff --git a/src/server.h b/src/server.h
index b1e645b0c..7431c7a1e 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2493,6 +2493,7 @@ void xtrimCommand(client *c);
 void lolwutCommand(client *c);
 void aclCommand(client *c);
 void stralgoCommand(client *c);
+void resetCommand(client *c);
 
 #if defined(__GNUC__)
 void *calloc(size_t count, size_t size) __attribute__ ((deprecated));
diff --git a/tests/unit/other.tcl b/tests/unit/other.tcl
index 3ca70a77a..96e9ee3da 100644
--- a/tests/unit/other.tcl
+++ b/tests/unit/other.tcl
@@ -249,6 +249,57 @@ start_server {tags {"other"}} {
         waitForBgsave r
         r save
     } {OK}
+
+    test {RESET clears client state} {
+        r client setname test-client
+        r client tracking on
+
+        assert_equal [r reset] "RESET"
+        set client [r client list]
+        assert_match {*name= *} $client
+        assert_match {*flags=N *} $client
+    }
+
+    test {RESET clears MONITOR state} {
+        set rd [redis_deferring_client]
+        $rd monitor
+        assert_equal [$rd read] "OK"
+
+        $rd reset
+
+        # skip reset ouptut
+        $rd read
+        assert_equal [$rd read] "RESET"
+
+        assert_no_match {*flags=O*} [r client list]
+    }
+
+    test {RESET clears and discards MULTI state} {
+        r multi
+        r set key-a a
+
+        r reset
+        catch {r exec} err
+        assert_match {*EXEC without MULTI*} $err
+    }
+
+    test {RESET clears Pub/Sub state} {
+        r subscribe channel-1
+        r reset
+
+        # confirm we're not subscribed by executing another command
+        r set key val
+    }
+
+    test {RESET clears authenticated state} {
+        r acl setuser user1 on >secret +@all
+        r auth user1 secret
+        assert_equal [r acl whoami] user1
+
+        r reset
+
+        assert_equal [r acl whoami] default
+    }
 }
 
 start_server {tags {"other"}} {

From 7ace7231c6d306e3df92e6b4ec044b0e8cd8665b Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 5 Nov 2020 11:46:16 +0200
Subject: [PATCH 0784/1098] Better INFO fields to track diskless and disk-based
 replication progress (#7981)

Expose new `loading_rdb_used_mem` showing the used memory of the server
that saved the RDB file we're currently using.
This is useful in diskless replication when the total size of the rdb is
unkown, and can be used as a rought estimation of progres.

Use that new field to calculate the "user friendly"
`loading_loaded_perc` and `loading_eta_seconds`.

Expose `master_sync_total_bytes` and `master_sync_total_bytes` to complement
on the existing `master_sync_total_bytes` (which cannot be used on its own
to calculate progress).

Add "user friendly" field for `master_sync_perc`
---
 src/rdb.c    |  2 ++
 src/server.c | 35 +++++++++++++++++++++++++++--------
 src/server.h |  1 +
 3 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/src/rdb.c b/src/rdb.c
index e15984e44..e3f34b3f0 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2023,6 +2023,7 @@ void startLoading(size_t size, int rdbflags) {
     server.loading_start_time = time(NULL);
     server.loading_loaded_bytes = 0;
     server.loading_total_bytes = size;
+    server.loading_rdb_used_mem = 0;
     blockingOperationStarts();
 
     /* Fire the loading modules start event. */
@@ -2238,6 +2239,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
                 long long usedmem = strtoll(auxval->ptr,NULL,10);
                 serverLog(LL_NOTICE,"RDB memory usage when created %.2f Mb",
                     (double) usedmem / (1024*1024));
+                server.loading_rdb_used_mem = usedmem;
             } else if (!strcasecmp(auxkey->ptr,"aof-preamble")) {
                 long long haspreamble = strtoll(auxval->ptr,NULL,10);
                 if (haspreamble) serverLog(LL_NOTICE,"RDB has an AOF tail");
diff --git a/src/server.c b/src/server.c
index 9aab1f39b..883a81305 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2463,6 +2463,7 @@ void initServerConfig(void) {
     server.client_max_querybuf_len = PROTO_MAX_QUERYBUF_LEN;
     server.saveparams = NULL;
     server.loading = 0;
+    server.loading_rdb_used_mem = 0;
     server.logfile = zstrdup(CONFIG_DEFAULT_LOGFILE);
     server.aof_state = AOF_OFF;
     server.aof_rewrite_base_size = 0;
@@ -4501,13 +4502,20 @@ sds genRedisInfoString(const char *section) {
         }
 
         if (server.loading) {
-            double perc;
+            double perc = 0;
             time_t eta, elapsed;
-            off_t remaining_bytes = server.loading_total_bytes-
-                                    server.loading_loaded_bytes;
+            off_t remaining_bytes = 1;
 
-            perc = ((double)server.loading_loaded_bytes /
-                   (server.loading_total_bytes+1)) * 100;
+            if (server.loading_total_bytes) {
+                perc = ((double)server.loading_loaded_bytes / server.loading_total_bytes) * 100;
+                remaining_bytes = server.loading_total_bytes - server.loading_loaded_bytes;
+            } else if(server.loading_rdb_used_mem) {
+                perc = ((double)server.loading_loaded_bytes / server.loading_rdb_used_mem) * 100;
+                remaining_bytes = server.loading_rdb_used_mem - server.loading_loaded_bytes;
+                /* used mem is only a (bad) estimation of the rdb file size, avoid going over 100% */
+                if (perc > 99.99) perc = 99.99;
+                if (remaining_bytes < 1) remaining_bytes = 1;
+            }
 
             elapsed = time(NULL)-server.loading_start_time;
             if (elapsed == 0) {
@@ -4520,11 +4528,13 @@ sds genRedisInfoString(const char *section) {
             info = sdscatprintf(info,
                 "loading_start_time:%jd\r\n"
                 "loading_total_bytes:%llu\r\n"
+                "loading_rdb_used_mem:%llu\r\n"
                 "loading_loaded_bytes:%llu\r\n"
                 "loading_loaded_perc:%.2f\r\n"
                 "loading_eta_seconds:%jd\r\n",
                 (intmax_t) server.loading_start_time,
                 (unsigned long long) server.loading_total_bytes,
+                (unsigned long long) server.loading_rdb_used_mem,
                 (unsigned long long) server.loading_loaded_bytes,
                 perc,
                 (intmax_t)eta
@@ -4649,11 +4659,20 @@ sds genRedisInfoString(const char *section) {
             );
 
             if (server.repl_state == REPL_STATE_TRANSFER) {
+                double perc = 0;
+                if (server.repl_transfer_size) {
+                    perc = ((double)server.repl_transfer_read / server.repl_transfer_size) * 100;
+                }
                 info = sdscatprintf(info,
+                    "master_sync_total_bytes:%lld\r\n"
+                    "master_sync_read_bytes:%lld\r\n"
                     "master_sync_left_bytes:%lld\r\n"
-                    "master_sync_last_io_seconds_ago:%d\r\n"
-                    , (long long)
-                        (server.repl_transfer_size - server.repl_transfer_read),
+                    "master_sync_perc:%.2f\r\n"
+                    "master_sync_last_io_seconds_ago:%d\r\n",
+                    (long long) server.repl_transfer_size,
+                    (long long) server.repl_transfer_read,
+                    (long long) (server.repl_transfer_size - server.repl_transfer_read),
+                    perc,
                     (int)(server.unixtime-server.repl_transfer_lastio)
                 );
             }
diff --git a/src/server.h b/src/server.h
index 7431c7a1e..5a37ad65b 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1139,6 +1139,7 @@ struct redisServer {
     /* RDB / AOF loading information */
     volatile sig_atomic_t loading; /* We are loading data from disk if true */
     off_t loading_total_bytes;
+    off_t loading_rdb_used_mem;
     off_t loading_loaded_bytes;
     time_t loading_start_time;
     off_t loading_process_events_interval_bytes;

From 7e4325cbc9af96d55f0b61c7648233a4473e8c93 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 5 Nov 2020 15:43:53 +0200
Subject: [PATCH 0785/1098] Fix crash log output on ARM. (#8020)

---
 src/debug.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/debug.c b/src/debug.c
index e21440f84..30c51a1b0 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1221,7 +1221,7 @@ void logRegisters(ucontext_t *uc) {
 	      "R10:%016lx R9 :%016lx\nR8 :%016lx R7 :%016lx\n"
 	      "R6 :%016lx R5 :%016lx\nR4 :%016lx R3 :%016lx\n"
 	      "R2 :%016lx R1 :%016lx\nR0 :%016lx EC :%016lx\n"
-	      "fp: %016lx ip:%016lx\n",
+	      "fp: %016lx ip:%016lx\n"
 	      "pc:%016lx sp:%016lx\ncpsr:%016lx fault_address:%016lx\n",
 	      (unsigned long) uc->uc_mcontext.arm_r10,
 	      (unsigned long) uc->uc_mcontext.arm_r9,

From 254367788a32b5178c25257e5a8cbd77c4d4a31f Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Thu, 5 Nov 2020 12:58:54 -0500
Subject: [PATCH 0786/1098] Debug Populate: Avoid server crash when passing
 negative value for key and value size (#8018)

* Debug Populate: Add checks for count and keysize to avoid crash

* provide getRangeLongFromObjectOrReply and getPositiveLongFromObjectOrReply for range check
---
 src/debug.c  |  6 ++++--
 src/object.c | 17 +++++++++++++++++
 src/server.h |  2 ++
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/src/debug.c b/src/debug.c
index 30c51a1b0..36cef5625 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -600,12 +600,14 @@ NULL
         robj *key, *val;
         char buf[128];
 
-        if (getLongFromObjectOrReply(c, c->argv[2], &keys, NULL) != C_OK)
+        if (getPositiveLongFromObjectOrReply(c, c->argv[2], &keys, NULL) != C_OK)
             return;
+
         dictExpand(c->db->dict,keys);
         long valsize = 0;
-        if ( c->argc == 5 && getLongFromObjectOrReply(c, c->argv[4], &valsize, NULL) != C_OK ) 
+        if ( c->argc == 5 && getPositiveLongFromObjectOrReply(c, c->argv[4], &valsize, NULL) != C_OK ) 
             return;
+
         for (j = 0; j < keys; j++) {
             snprintf(buf,sizeof(buf),"%s:%lu",
                 (c->argc == 3) ? "key" : (char*)c->argv[3]->ptr, j);
diff --git a/src/object.c b/src/object.c
index 1b8852f18..eef5e6d9e 100644
--- a/src/object.c
+++ b/src/object.c
@@ -730,6 +730,23 @@ int getLongFromObjectOrReply(client *c, robj *o, long *target, const char *msg)
     return C_OK;
 }
 
+int getRangeLongFromObjectOrReply(client *c, robj *o, long min, long max, long *target, const char *msg) {
+    if (getLongFromObjectOrReply(c, o, target, msg) != C_OK) return C_ERR;
+    if (*target < min || *target > max) {
+        if (msg != NULL) {
+            addReplyError(c,(char*)msg);
+        } else {
+            addReplyErrorFormat(c,"value is out of range, value must between %ld and %ld", min, max);
+        }
+        return C_ERR;
+    }
+    return C_OK;
+}
+
+int getPositiveLongFromObjectOrReply(client *c, robj *o, long *target, const char *msg) {
+    return getRangeLongFromObjectOrReply(c, o, 0, LONG_MAX, target, msg);
+}
+
 char *strEncoding(int encoding) {
     switch(encoding) {
     case OBJ_ENCODING_RAW: return "raw";
diff --git a/src/server.h b/src/server.h
index 5a37ad65b..ea6eb5f04 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1838,6 +1838,8 @@ robj *createZsetZiplistObject(void);
 robj *createStreamObject(void);
 robj *createModuleObject(moduleType *mt, void *value);
 int getLongFromObjectOrReply(client *c, robj *o, long *target, const char *msg);
+int getPositiveLongFromObjectOrReply(client *c, robj *o, long *target, const char *msg);
+int getRangeLongFromObjectOrReply(client *c, robj *o, long min, long max, long *target, const char *msg);
 int checkType(client *c, robj *o, int type);
 int getLongLongFromObjectOrReply(client *c, robj *o, long long *target, const char *msg);
 int getDoubleFromObjectOrReply(client *c, robj *o, double *target, const char *msg);

From cd1c600548d184cb9019c0491c8fb5a5cf902706 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Sun, 8 Nov 2020 14:32:38 +0800
Subject: [PATCH 0787/1098] Typo fix: entires -> entries (#8031)

---
 redis.conf             | 2 +-
 src/acl.c              | 2 +-
 src/crcspeed.c         | 4 ++--
 src/help.h             | 2 +-
 tests/unit/slowlog.tcl | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/redis.conf b/redis.conf
index c96aebea4..a46668826 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1644,7 +1644,7 @@ hll-sparse-max-bytes 3000
 # maximum number of items it may contain before switching to a new node when
 # appending new stream entries. If any of the following settings are set to
 # zero, the limit is ignored, so for instance it is possible to set just a
-# max entires limit by setting max-bytes to 0 and max-entries to the desired
+# max entries limit by setting max-bytes to 0 and max-entries to the desired
 # value.
 stream-node-max-bytes 4096
 stream-node-max-entries 100
diff --git a/src/acl.c b/src/acl.c
index f0523b635..e8f9d3724 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1672,7 +1672,7 @@ void addACLLogEntry(client *c, int reason, int keypos, sds username) {
         le->cinfo = NULL;
         ACLFreeLogEntry(le);
     } else {
-        /* Add it to our list of entires. We'll have to trim the list
+        /* Add it to our list of entries. We'll have to trim the list
          * to its maximum size. */
         listAddNodeHead(ACLLog, le);
         while(listLength(ACLLog) > server.acllog_max_len) {
diff --git a/src/crcspeed.c b/src/crcspeed.c
index d2d97a8c7..d4955bfc9 100644
--- a/src/crcspeed.c
+++ b/src/crcspeed.c
@@ -84,7 +84,7 @@ static inline uint64_t rev8(uint64_t a) {
 /* This function is called once to initialize the CRC table for use on a
    big-endian architecture. */
 void crcspeed64big_init(crcfn64 fn, uint64_t big_table[8][256]) {
-    /* Create the little endian table then reverse all the entires. */
+    /* Create the little endian table then reverse all the entries. */
     crcspeed64little_init(fn, big_table);
     for (int k = 0; k < 8; k++) {
         for (int n = 0; n < 256; n++) {
@@ -94,7 +94,7 @@ void crcspeed64big_init(crcfn64 fn, uint64_t big_table[8][256]) {
 }
 
 void crcspeed16big_init(crcfn16 fn, uint16_t big_table[8][256]) {
-    /* Create the little endian table then reverse all the entires. */
+    /* Create the little endian table then reverse all the entries. */
     crcspeed16little_init(fn, big_table);
     for (int k = 0; k < 8; k++) {
         for (int n = 0; n < 256; n++) {
diff --git a/src/help.h b/src/help.h
index 1b1ac5e08..8a12458e1 100644
--- a/src/help.h
+++ b/src/help.h
@@ -1175,7 +1175,7 @@ struct commandHelp {
     "5.0.0" },
     { "XLEN",
     "key",
-    "Return the number of entires in a stream",
+    "Return the number of entries in a stream",
     14,
     "5.0.0" },
     { "XPENDING",
diff --git a/tests/unit/slowlog.tcl b/tests/unit/slowlog.tcl
index 22f088103..c2ef23cf7 100644
--- a/tests/unit/slowlog.tcl
+++ b/tests/unit/slowlog.tcl
@@ -70,7 +70,7 @@ start_server {tags {"slowlog"} overrides {slowlog-log-slower-than 1000000}} {
         assert_equal [lindex $e 3] {debug sleep 0.2}
     }
 
-    test {SLOWLOG - can clean older entires} {
+    test {SLOWLOG - can clean older entries} {
         r client setname lastentry_client
         r config set slowlog-max-len 1
         r debug sleep 0.2

From d428de590fb15f61304d84c0320f7b7eb415f12c Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Sun, 8 Nov 2020 07:16:14 +0000
Subject: [PATCH 0788/1098] DragonFlyBSD resident memory amount (almost)
 similar as FreeBSD. (#8023)

---
 src/zmalloc.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/zmalloc.c b/src/zmalloc.c
index 565376721..7425198fe 100644
--- a/src/zmalloc.c
+++ b/src/zmalloc.c
@@ -365,7 +365,7 @@ size_t zmalloc_get_rss(void) {
 
     return t_info.resident_size;
 }
-#elif defined(__FreeBSD__)
+#elif defined(__FreeBSD__) || defined(__DragonFly__)
 #include <sys/types.h>
 #include <sys/sysctl.h>
 #include <sys/user.h>
@@ -381,7 +381,11 @@ size_t zmalloc_get_rss(void) {
     mib[3] = getpid();
 
     if (sysctl(mib, 4, &info, &infolen, NULL, 0) == 0)
+#if defined(__FreeBSD__)
         return (size_t)info.ki_rssize;
+#else
+        return (size_t)info.kp_vm_rssize;
+#endif
 
     return 0L;
 }

From bcc46a2f251728e8a6348d0386903503f197e675 Mon Sep 17 00:00:00 2001
From: bugwz <cubegwz@gmail.com>
Date: Sun, 8 Nov 2020 15:26:25 +0800
Subject: [PATCH 0789/1098] Fix the init value for repl_id of rdbSaveInfo
 struct (#8026)

use 40 zeros rather than 30 zeros to match CONFIG_RUN_ID_SIZE.
this doesn't have any real implications.
---
 src/server.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.h b/src/server.h
index ea6eb5f04..587779844 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1014,7 +1014,7 @@ typedef struct rdbSaveInfo {
     long long repl_offset;                  /* Replication offset. */
 } rdbSaveInfo;
 
-#define RDB_SAVE_INFO_INIT {-1,0,"000000000000000000000000000000",-1}
+#define RDB_SAVE_INFO_INIT {-1,0,"0000000000000000000000000000000000000000",-1}
 
 struct malloc_stats {
     size_t zmalloc_used;

From 97d647a13920c230a27685992a23679b44b73550 Mon Sep 17 00:00:00 2001
From: "Meir Shpilraien (Spielrein)" <meir@redislabs.com>
Date: Mon, 9 Nov 2020 10:46:23 +0200
Subject: [PATCH 0790/1098] Moved RMAPI_FUNC_SUPPORTED location such that it
 will be visible to modules (#8037)

The RMAPI_FUNC_SUPPORTED was defined in the wrong place on redismodule.h
and was not visible to modules.
---
 src/redismodule.h    | 4 ++--
 tests/modules/fork.c | 6 ++++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/redismodule.h b/src/redismodule.h
index cb52bd79e..ca0a756c7 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -1020,13 +1020,13 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
 
 #define RedisModule_Assert(_e) ((_e)?(void)0 : (RedisModule__Assert(#_e,__FILE__,__LINE__),exit(1)))
 
+#define RMAPI_FUNC_SUPPORTED(func) (func != NULL)
+
 #else
 
 /* Things only defined for the modules core, not exported to modules
  * including this file. */
 #define RedisModuleString robj
 
-#define RMAPI_FUNC_SUPPORTED(func) (func != NULL)
-
 #endif /* REDISMODULE_CORE */
 #endif /* REDISMODULE_H */
diff --git a/tests/modules/fork.c b/tests/modules/fork.c
index 0443d9ef0..50889eef9 100644
--- a/tests/modules/fork.c
+++ b/tests/modules/fork.c
@@ -28,6 +28,12 @@ int fork_create(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
         RedisModule_WrongArity(ctx);
         return REDISMODULE_OK;
     }
+
+    if(!RMAPI_FUNC_SUPPORTED(RedisModule_Fork)){
+        RedisModule_ReplyWithError(ctx, "Fork api is not supported in the current redis version");
+        return REDISMODULE_OK;
+    }
+
     RedisModule_StringToLongLong(argv[1], &code_to_exit_with);
     exitted_with_code = -1;
     child_pid = RedisModule_Fork(done_handler, (void*)0xdeadbeef);

From 19c29b600746b575a7e902b30633b735bbf67c55 Mon Sep 17 00:00:00 2001
From: nitaicaro <42576749+nitaicaro@users.noreply.github.com>
Date: Mon, 9 Nov 2020 22:54:47 +0200
Subject: [PATCH 0791/1098] Extend client tracking tests (#7998)

Test support for the new map, null and push message types. Map objects are parsed as a list of lists of key value pairs.
for instance: user => john password => 123

will be parsed to the following TCL list:

{{user john} {password 123}}

Also added the following tests:

Redirection still works with RESP3

Able to use a RESP3 client as a redirection client

No duplicate invalidation messages when turning BCAST mode on after normal tracking

Server is able to evacuate enough keys when num of keys surpasses limit by more than defined initial effort

Different clients using different protocols can track the same key

OPTOUT tests

OPTIN tests

Clients can redirect to the same connection

tracking-redir-broken test

HELLO 3 checks

Invalidation messages still work when using RESP3, with and without redirection

Switching to RESP3 doesn't disturb previous tracked keys

Tracking info is correct

Flushall and flushdb produce invalidation messages

These tests achieve 100% line coverage for tracking.c using lcov.
---
 tests/support/redis.tcl |  27 ++++
 tests/unit/tracking.tcl | 339 +++++++++++++++++++++++++++++++++++-----
 2 files changed, 331 insertions(+), 35 deletions(-)

diff --git a/tests/support/redis.tcl b/tests/support/redis.tcl
index a90ac7f29..847f68e63 100644
--- a/tests/support/redis.tcl
+++ b/tests/support/redis.tcl
@@ -210,18 +210,45 @@ proc ::redis::redis_multi_bulk_read {id fd} {
     return $l
 }
 
+proc ::redis::redis_read_map {id fd} {
+    set count [redis_read_line $fd]
+    if {$count == -1} return {}
+    set l {}
+    set err {}
+    for {set i 0} {$i < $count} {incr i} {
+        if {[catch {
+            set t {}
+            lappend t [redis_read_reply $id $fd] ; # key
+            lappend t [redis_read_reply $id $fd] ; # value
+            lappend l $t
+        } e] && $err eq {}} {
+            set err $e
+        }
+    }
+    if {$err ne {}} {return -code error $err}
+    return $l
+}
+
 proc ::redis::redis_read_line fd {
     string trim [gets $fd]
 }
 
+proc ::redis::redis_read_null fd {
+    gets $fd
+    return {}
+}
+
 proc ::redis::redis_read_reply {id fd} {
     set type [read $fd 1]
     switch -exact -- $type {
+        _ {redis_read_null $fd}
         : -
         + {redis_read_line $fd}
         - {return -code error [redis_read_line $fd]}
         $ {redis_bulk_read $fd}
+        > -
         * {redis_multi_bulk_read $id $fd}
+        % {redis_read_map $id $fd}
         default {
             if {$type eq {}} {
                 set ::redis::fd($id) {}
diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index 839b894ea..fc2800791 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -1,26 +1,48 @@
 start_server {tags {"tracking"}} {
     # Create a deferred client we'll use to redirect invalidation
     # messages to.
-    set rd1 [redis_deferring_client]
-    $rd1 client id
-    set redir [$rd1 read]
-    $rd1 subscribe __redis__:invalidate
-    $rd1 read ; # Consume the SUBSCRIBE reply.
+    set rd_redirection [redis_deferring_client]
+    $rd_redirection client id
+    set redir_id [$rd_redirection read]
+    $rd_redirection subscribe __redis__:invalidate
+    $rd_redirection read ; # Consume the SUBSCRIBE reply.
 
-    # Create another client as well in order to test NOLOOP
-    set rd2 [redis_deferring_client]
+    # Create another client that's not used as a redirection client
+    # We should always keep this client's buffer clean
+    set rd [redis_deferring_client]
+
+    # Client to be used for SET and GET commands
+    # We don't read this client's buffer
+    set rd_sg [redis_client] 
+
+    proc clean_all {} {
+        uplevel {
+            $rd QUIT
+            $rd_redirection QUIT
+            set rd [redis_deferring_client]
+            set rd_redirection [redis_deferring_client]
+            $rd_redirection client id
+            set redir_id [$rd_redirection read]
+            $rd_redirection subscribe __redis__:invalidate
+            $rd_redirection read ; # Consume the SUBSCRIBE reply.
+            r FLUSHALL
+            r CLIENT TRACKING off
+            r HELLO 2
+            r config set tracking-table-max-keys 1000000
+        }
+    }
 
     test {Clients are able to enable tracking and redirect it} {
-        r CLIENT TRACKING on REDIRECT $redir
+        r CLIENT TRACKING on REDIRECT $redir_id
     } {*OK}
 
     test {The other connection is able to get invalidations} {
         r SET a 1
         r SET b 1
         r GET a
-        r INCR b ; # This key should not be notified, since it wasn't fetched. 
-        r INCR a 
-        set keys [lindex [$rd1 read] 2]
+        r INCR b ; # This key should not be notified, since it wasn't fetched.
+        r INCR a
+        set keys [lindex [$rd_redirection read] 2]
         assert {[llength $keys] == 1}
         assert {[lindex $keys 0] eq {a}}
     }
@@ -33,18 +55,18 @@ start_server {tags {"tracking"}} {
     }
 
     test {Clients can enable the BCAST mode with the empty prefix} {
-        r CLIENT TRACKING on BCAST REDIRECT $redir
+        r CLIENT TRACKING on BCAST REDIRECT $redir_id
     } {*OK*}
 
     test {The connection gets invalidation messages about all the keys} {
         r MSET a 1 b 2 c 3
-        set keys [lsort [lindex [$rd1 read] 2]]
+        set keys [lsort [lindex [$rd_redirection read] 2]]
         assert {$keys eq {a b c}}
     }
 
     test {Clients can enable the BCAST mode with prefixes} {
         r CLIENT TRACKING off
-        r CLIENT TRACKING on BCAST REDIRECT $redir PREFIX a: PREFIX b:
+        r CLIENT TRACKING on BCAST REDIRECT $redir_id PREFIX a: PREFIX b:
         r MULTI
         r INCR a:1
         r INCR a:2
@@ -53,64 +75,243 @@ start_server {tags {"tracking"}} {
         r EXEC
         # Because of the internals, we know we are going to receive
         # two separated notifications for the two different prefixes.
-        set keys1 [lsort [lindex [$rd1 read] 2]]
-        set keys2 [lsort [lindex [$rd1 read] 2]]
+        set keys1 [lsort [lindex [$rd_redirection read] 2]]
+        set keys2 [lsort [lindex [$rd_redirection read] 2]]
         set keys [lsort [list {*}$keys1 {*}$keys2]]
         assert {$keys eq {a:1 a:2 b:1 b:2}}
     }
-    
+
     test {Adding prefixes to BCAST mode works} {
-        r CLIENT TRACKING on BCAST REDIRECT $redir PREFIX c:
+        r CLIENT TRACKING on BCAST REDIRECT $redir_id PREFIX c:
         r INCR c:1234
-        set keys [lsort [lindex [$rd1 read] 2]]
+        set keys [lsort [lindex [$rd_redirection read] 2]]
         assert {$keys eq {c:1234}}
     }
 
     test {Tracking NOLOOP mode in standard mode works} {
         r CLIENT TRACKING off
-        r CLIENT TRACKING on REDIRECT $redir NOLOOP
+        r CLIENT TRACKING on REDIRECT $redir_id NOLOOP
         r MGET otherkey1 loopkey otherkey2
-        $rd2 SET otherkey1 1; # We should get this
+        $rd_sg SET otherkey1 1; # We should get this
         r SET loopkey 1 ; # We should not get this
-        $rd2 SET otherkey2 1; # We should get this
+        $rd_sg SET otherkey2 1; # We should get this
         # Because of the internals, we know we are going to receive
         # two separated notifications for the two different prefixes.
-        set keys1 [lsort [lindex [$rd1 read] 2]]
-        set keys2 [lsort [lindex [$rd1 read] 2]]
+        set keys1 [lsort [lindex [$rd_redirection read] 2]]
+        set keys2 [lsort [lindex [$rd_redirection read] 2]]
         set keys [lsort [list {*}$keys1 {*}$keys2]]
         assert {$keys eq {otherkey1 otherkey2}}
     }
 
     test {Tracking NOLOOP mode in BCAST mode works} {
         r CLIENT TRACKING off
-        r CLIENT TRACKING on BCAST REDIRECT $redir NOLOOP
-        $rd2 SET otherkey1 1; # We should get this
+        r CLIENT TRACKING on BCAST REDIRECT $redir_id NOLOOP
+        $rd_sg SET otherkey1 1; # We should get this
         r SET loopkey 1 ; # We should not get this
-        $rd2 SET otherkey2 1; # We should get this
+        $rd_sg SET otherkey2 1; # We should get this
         # Because of the internals, we know we are going to receive
         # two separated notifications for the two different prefixes.
-        set keys1 [lsort [lindex [$rd1 read] 2]]
-        set keys2 [lsort [lindex [$rd1 read] 2]]
+        set keys1 [lsort [lindex [$rd_redirection read] 2]]
+        set keys2 [lsort [lindex [$rd_redirection read] 2]]
         set keys [lsort [list {*}$keys1 {*}$keys2]]
         assert {$keys eq {otherkey1 otherkey2}}
     }
 
     test {Tracking gets notification of expired keys} {
         r CLIENT TRACKING off
-        r CLIENT TRACKING on BCAST REDIRECT $redir NOLOOP
+        r CLIENT TRACKING on BCAST REDIRECT $redir_id NOLOOP
         r SET mykey myval px 1
         r SET mykeyotherkey myval ; # We should not get it
         after 1000
         # Because of the internals, we know we are going to receive
         # two separated notifications for the two different prefixes.
-        set keys1 [lsort [lindex [$rd1 read] 2]]
+        set keys1 [lsort [lindex [$rd_redirection read] 2]]
         set keys [lsort [list {*}$keys1]]
         assert {$keys eq {mykey}}
     }
 
-    test {Tracking gets notification on tracking table key eviction} {
+    test {HELLO 3 reply is correct} {
+        set reply [r HELLO 3]
+        assert {[lindex $reply 2] eq {proto 3}}
+    }
+
+    test {RESP3 based basic invalidation} {
         r CLIENT TRACKING off
-        r CLIENT TRACKING on REDIRECT $redir NOLOOP
+        r CLIENT TRACKING on
+        $rd_sg SET key1 1
+        r GET key1
+        $rd_sg SET key1 2
+        r read
+    } {invalidate key1}
+
+    test {RESP3 tracking redirection} {
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on REDIRECT $redir_id
+        $rd_sg SET key1 1
+        r GET key1
+        $rd_sg SET key1 2
+        set res [lindex [$rd_redirection read] 2]
+        assert {$res eq {key1}}
+    }
+
+    test {Invalidations of previous keys can be redirected after switching to RESP3} {
+        $rd_sg SET key1 1
+        r GET key1
+        r HELLO 3
+        $rd_sg SET key1 2
+        set res [lindex [$rd_redirection read] 2]
+        assert {$res eq {key1}}
+    }
+
+    test {Invalidations of new keys can be redirected after switching to RESP3} {
+        r HELLO 3
+        $rd_sg SET key1 1
+        r GET key1
+        $rd_sg SET key1 2
+        set res [lindex [$rd_redirection read] 2]
+        assert {$res eq {key1}}
+    }
+
+    test {RESP3 Client gets tracking-redir-broken push message after cached key changed when rediretion client is terminated} {
+        r CLIENT TRACKING on REDIRECT $redir_id
+        $rd_sg SET key1 1
+        r GET key1
+        $rd_redirection QUIT
+        $rd_sg SET key1 2
+        set MAX_TRIES 100
+        for {set i 0} {$i <= $MAX_TRIES && $res <= 0} {incr i} {
+            set res [lsearch -exact [r PING] "tracking-redir-broken"]
+        }
+        assert {$res >= 0}
+    }
+
+    test {Different clients can redirect to the same connection} {
+        # Reinstantiating after QUIT
+        set rd_redirection [redis_deferring_client]
+        $rd_redirection CLIENT ID
+        set redir_id [$rd_redirection read]
+        $rd_redirection SUBSCRIBE __redis__:invalidate
+        $rd_redirection read ; # Consume the SUBSCRIBE reply
+        r CLIENT TRACKING on REDIRECT $redir_id
+        $rd CLIENT TRACKING on REDIRECT $redir_id 
+        $rd read ; # Consume the TRACKING reply
+        $rd_sg MSET key1 1 key2 1
+        r GET key1
+        $rd GET key2 
+        $rd read ; # Consume the GET reply
+        $rd_sg INCR key1
+        $rd_sg INCR key2
+        set res1 [lindex [$rd_redirection read] 2]
+        set res2 [lindex [$rd_redirection read] 2]
+        assert {$res1 eq {key1}}
+        assert {$res2 eq {key2}}
+    }
+
+    test {Different clients using different protocols can track the same key} {
+        $rd HELLO 3 
+        $rd read ; # Consume the HELLO reply
+        $rd CLIENT TRACKING on 
+        $rd read ; # Consume the TRACKING reply
+        $rd_sg set key1 1
+        r GET key1
+        $rd GET key1 
+        $rd read ; # Consume the GET reply
+        $rd_sg INCR key1
+        set res1 [lindex [$rd_redirection read] 2]
+        $rd PING ; # Non redirecting client has to talk to the server in order to get invalidation message
+        set res2 [lindex [split [$rd read] " "] 1] 
+        $rd read ; # Consume the PING reply, which comes together with the invalidation message
+        assert {$res1 eq {key1}}
+        assert {$res2 eq {key1}}
+    }
+
+    test {No invalidation message when using OPTIN option} {
+        r CLIENT TRACKING on OPTIN REDIRECT $redir_id
+        $rd_sg SET key1 1
+        r GET key1 ; # This key should not be notified, since OPTIN is on and CLIENT CACHING yes wasn't called
+        $rd_sg SET key1 2
+        # Preparing some message to consume on $rd_redirection so we don't get blocked
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on REDIRECT $redir_id
+        $rd_sg SET key2 1
+        r GET key2 ; # This key should be notified
+        $rd_sg SET key2 2
+        set res [lindex [$rd_redirection read] 2]
+        assert {$res eq {key2}}
+    }
+
+    test {Invalidation message sent when using OPTIN option with CLIENT CACHING yes} {
+        r CLIENT TRACKING on OPTIN REDIRECT $redir_id
+        $rd_sg SET key1 3
+        r CLIENT CACHING yes
+        r GET key1
+        $rd_sg SET key1 4
+        set res [lindex [$rd_redirection read] 2]
+        assert {$res eq {key1}}
+    }
+
+    test {Invalidation message sent when using OPTOUT option} {
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on OPTOUT REDIRECT $redir_id
+        $rd_sg SET key1 1
+        r GET key1 
+        $rd_sg SET key1 2
+        set res [lindex [$rd_redirection read] 2]
+        assert {$res eq {key1}}
+    }
+
+    test {No invalidation message when using OPTOUT option with CLIENT CACHING no} {
+        $rd_sg SET key1 1
+        r CLIENT CACHING no
+        r GET key1 ; # This key should not be notified, since OPTOUT is on and CLIENT CACHING no was called
+        $rd_sg SET key1 2
+        # Preparing some message to consume on $rd_redirection so we don't get blocked
+        $rd_sg SET key2 1
+        r GET key2 ; # This key should be notified
+        $rd_sg SET key2 2
+        set res [lindex [$rd_redirection read] 2]
+        assert {$res eq {key2}}
+    }
+
+    test {Able to redirect to a RESP3 client} {
+        $rd_redirection UNSUBSCRIBE __redis__:invalidate ; # Need to unsub first before we can do HELLO 3
+        $rd_redirection read ; # Consume the UNSUBSCRIBE reply
+        $rd_redirection HELLO 3
+        $rd_redirection read ; # Consume the HELLO reply
+        $rd_redirection SUBSCRIBE __redis__:invalidate
+        $rd_redirection read ; # Consume the SUBSCRIBE reply
+        r CLIENT TRACKING on REDIRECT $redir_id
+        $rd_sg SET key1 1
+        r GET key1
+        $rd_sg INCR key1
+        set res [lindex [$rd_redirection read] 1]
+        assert {$res eq {key1}}
+    }
+
+    test {After switching from normal tracking to BCAST mode, no invalidation message is produced for pre-BCAST keys} {
+        $rd HELLO 3
+        $rd read ; # Consume the HELLO reply
+        $rd CLIENT TRACKING on
+        $rd read ; # Consume the TRACKING reply
+        $rd_sg SET key1 1
+        $rd GET key1
+        $rd read ; # Consume the GET reply
+        $rd CLIENT TRACKING off 
+        $rd read ; # Consume the TRACKING reply
+        $rd CLIENT TRACKING on BCAST
+        $rd read ; # Consume the TRACKING reply
+        $rd_sg INCR key1
+        $rd PING
+        set inv_msg [$rd read]
+        set ping_reply [$rd read]
+        assert {$inv_msg eq {invalidate key1}}
+        assert {$ping_reply eq {PONG}}
+    } 
+
+    test {Tracking gets notification on tracking table key eviction} {
+        $rd_redirection HELLO 2
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on REDIRECT $redir_id NOLOOP
         r MSET key1 1 key2 2
         # Let the server track the two keys for us
         r MGET key1 key2
@@ -122,7 +323,7 @@ start_server {tags {"tracking"}} {
         # So we check that eventually we'll receive one or the other key,
         # otherwise the test will die for timeout.
         while 1 {
-            set keys [lindex [$rd1 read] 2]
+            set keys [lindex [$rd_redirection read] 2]
             if {$keys eq {key1} || $keys eq {key2}} break
         }
         # We should receive an expire notification for one of
@@ -130,5 +331,73 @@ start_server {tags {"tracking"}} {
         assert {$keys eq {key1} || $keys eq {key2}}
     }
 
-    $rd1 close
+    test {Invalidation message received for flushall} {
+        clean_all
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on REDIRECT $redir_id
+        $rd_sg SET key1 1
+        r GET key1
+        $rd_sg FLUSHALL
+        set msg [$rd_redirection read]
+        assert {[lindex msg 2] eq {} }
+    }
+
+    test {Invalidation message received for flushdb} {
+        clean_all
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on REDIRECT $redir_id
+        $rd_sg SET key1 1
+        r GET key1
+        $rd_sg FLUSHDB
+        set msg [$rd_redirection read]
+        assert {[lindex msg 2] eq {} }
+    }
+
+    # Keys are defined to be evicted 100 at a time by default.
+    # If after eviction the number of keys still surpasses the limit
+    # defined in tracking-table-max-keys, we increases eviction 
+    # effort to 200, and then 300, etc. 
+    # This test tests this effort incrementation. 
+    test {Server is able to evacuate enough keys when num of keys surpasses limit by more than defined initial effort} {
+        clean_all
+        set NUM_OF_KEYS_TO_TEST 250
+        set TRACKING_TABLE_MAX_KEYS 1
+        r CLIENT TRACKING on REDIRECT $redir_id
+        for {set i 0} {$i < $NUM_OF_KEYS_TO_TEST} {incr i} {
+            $rd_sg SET key$i $i
+            r GET key$i
+        }
+        r config set tracking-table-max-keys $TRACKING_TABLE_MAX_KEYS
+        # If not enough keys are evicted, we won't get enough invalidation
+        # messages, and "$rd_redirection read" will block.
+        # If too many keys are evicted, we will get too many invalidation
+        # messages, and the assert will fail.
+        for {set i 0} {$i < $NUM_OF_KEYS_TO_TEST - $TRACKING_TABLE_MAX_KEYS} {incr i} {
+            $rd_redirection read
+        }
+        $rd_redirection PING
+        assert {[$rd_redirection read] eq {pong {}}}
+    }
+
+    test {Tracking info is correct} {
+        clean_all
+        r CLIENT TRACKING on REDIRECT $redir_id
+        $rd_sg SET key1 1
+        $rd_sg SET key2 2
+        r GET key1 
+        r GET key2
+        $rd CLIENT TRACKING on BCAST PREFIX prefix:
+        $rd_sg SET prefix:key1 1 
+        $rd_sg SET prefix:key2 2
+        set info [r info]
+        regexp "\r\ntracking_total_items:(.*?)\r\n" $info _ total_items
+        regexp "\r\ntracking_total_keys:(.*?)\r\n" $info _ total_keys
+        regexp "\r\ntracking_total_prefixes:(.*?)\r\n" $info _ total_prefixes
+        assert {$total_items == 2}
+        assert {$total_keys == 2}
+        assert {$total_prefixes == 1}
+    }
+
+    $rd_redirection close
+    $rd close
 }

From c09b5941ce31be3cb08818533917afdeaf799575 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Mon, 9 Nov 2020 20:16:09 +0800
Subject: [PATCH 0792/1098] optimization src/adlist.c:listJoin()

---
 src/adlist.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/adlist.c b/src/adlist.c
index bc06ffc8f..7670b2c10 100644
--- a/src/adlist.c
+++ b/src/adlist.c
@@ -360,15 +360,16 @@ void listRotateHeadToTail(list *list) {
 /* Add all the elements of the list 'o' at the end of the
  * list 'l'. The list 'other' remains empty but otherwise valid. */
 void listJoin(list *l, list *o) {
-    if (o->head)
-        o->head->prev = l->tail;
+    if (o->len == 0) return;
+
+    o->head->prev = l->tail;
 
     if (l->tail)
         l->tail->next = o->head;
     else
         l->head = o->head;
 
-    if (o->tail) l->tail = o->tail;
+    l->tail = o->tail;
     l->len += o->len;
 
     /* Setup other as an empty list. */

From 5d0c6b08686d51d552bae7dbb640625cfc1d73db Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Tue, 10 Nov 2020 22:02:09 +0200
Subject: [PATCH 0793/1098] Adds help for '--cluster-yes' (#8035)

---
 src/redis-cli.c | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 9c8d0b063..44ea504c9 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -2390,6 +2390,15 @@ clusterManagerCommandDef clusterManagerCommands[] = {
     {"help", clusterManagerCommandHelp, 0, NULL, NULL}
 };
 
+typedef struct clusterManagerOptionDef {
+    char *name;
+    char *desc;
+} clusterManagerOptionDef;
+
+clusterManagerOptionDef clusterManagerOptions[] = {
+    {"--cluster-yes", "Automatic yes to cluster commands prompts"}
+};
+
 static void getRDB(clusterManagerNode *node);
 
 static void createClusterManagerCommand(char *cmdname, int argc, char **argv) {
@@ -6622,7 +6631,21 @@ static int clusterManagerCommandHelp(int argc, char **argv) {
     }
     fprintf(stderr, "\nFor check, fix, reshard, del-node, set-timeout you "
                     "can specify the host and port of any working node in "
-                    "the cluster.\n\n");
+                    "the cluster.\n");
+
+    int options_count = sizeof(clusterManagerOptions) /
+                        sizeof(clusterManagerOptionDef);
+    i = 0;
+    fprintf(stderr, "\nCluster Manager Options:\n");
+    for (; i < options_count; i++) {
+        clusterManagerOptionDef *def = &(clusterManagerOptions[i]);
+        int namelen = strlen(def->name), padlen = padding - namelen;
+        fprintf(stderr, "  %s", def->name);
+        for (j = 0; j < padlen; j++) fprintf(stderr, " ");
+        fprintf(stderr, "%s\n", def->desc);
+    }
+
+    fprintf(stderr, "\n");
     return 0;
 }
 

From 3feff7d78abdb5037ac14596205975bddeb6a931 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Tue, 10 Nov 2020 13:50:03 -0800
Subject: [PATCH 0794/1098] Rewritten commands are logged as their original
 command (#8006)

* Rewritten commands are logged as their original command

Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
---
 src/networking.c       | 45 ++++++++++++++++++++++++++++--------------
 src/server.c           |  7 ++++++-
 src/server.h           |  3 +++
 tests/unit/slowlog.tcl | 35 ++++++++++++++++++++++++++++++++
 4 files changed, 74 insertions(+), 16 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 8aca33e85..13793590b 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -139,6 +139,8 @@ client *createClient(connection *conn) {
     c->argc = 0;
     c->argv = NULL;
     c->argv_len_sum = 0;
+    c->original_argc = 0;
+    c->original_argv = NULL;
     c->cmd = c->lastcmd = NULL;
     c->multibulklen = 0;
     c->bulklen = -1;
@@ -1065,6 +1067,17 @@ void acceptUnixHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
     }
 }
 
+void freeClientOriginalArgv(client *c) {
+    /* We didn't rewrite this client */
+    if (!c->original_argv) return;
+
+    for (int j = 0; j < c->original_argc; j++)
+        decrRefCount(c->original_argv[j]);
+    zfree(c->original_argv);
+    c->original_argv = NULL;
+    c->original_argc = 0;
+}
+
 static void freeClientArgv(client *c) {
     int j;
     for (j = 0; j < c->argc; j++)
@@ -1246,6 +1259,7 @@ void freeClient(client *c) {
     /* Free data structures. */
     listRelease(c->reply);
     freeClientArgv(c);
+    freeClientOriginalArgv(c);
 
     /* Unlink the client: this will close the socket, remove the I/O
      * handlers, and remove references of the client from different
@@ -2765,6 +2779,19 @@ void securityWarningCommand(client *c) {
     freeClientAsync(c);
 }
 
+/* Keep track of the original command arguments so that we can generate
+ * an accurate slowlog entry after the command has been executed. */
+static void retainOriginalCommandVector(client *c) {
+    /* We already rewrote this command, so don't rewrite it again */
+    if (c->original_argv) return;
+    c->original_argc = c->argc;
+    c->original_argv = zmalloc(sizeof(robj*)*(c->argc));
+    for (int j = 0; j < c->argc; j++) {
+        c->original_argv[j] = c->argv[j];
+        incrRefCount(c->argv[j]);
+    }
+}
+
 /* Rewrite the command vector of the client. All the new objects ref count
  * is incremented. The old command vector is freed, and the old objects
  * ref count is decremented. */
@@ -2782,26 +2809,14 @@ void rewriteClientCommandVector(client *c, int argc, ...) {
         argv[j] = a;
         incrRefCount(a);
     }
-    /* We free the objects in the original vector at the end, so we are
-     * sure that if the same objects are reused in the new vector the
-     * refcount gets incremented before it gets decremented. */
-    for (j = 0; j < c->argc; j++) decrRefCount(c->argv[j]);
-    zfree(c->argv);
-    /* Replace argv and argc with our new versions. */
-    c->argv = argv;
-    c->argc = argc;
-    c->argv_len_sum = 0;
-    for (j = 0; j < c->argc; j++)
-        if (c->argv[j])
-            c->argv_len_sum += getStringObjectLen(c->argv[j]);
-    c->cmd = lookupCommandOrOriginal(c->argv[0]->ptr);
-    serverAssertWithInfo(c,NULL,c->cmd != NULL);
+    replaceClientCommandVector(c, argc, argv);
     va_end(ap);
 }
 
 /* Completely replace the client command vector with the provided one. */
 void replaceClientCommandVector(client *c, int argc, robj **argv) {
     int j;
+    retainOriginalCommandVector(c);
     freeClientArgv(c);
     zfree(c->argv);
     c->argv = argv;
@@ -2827,7 +2842,7 @@ void replaceClientCommandVector(client *c, int argc, robj **argv) {
  *    free the no longer used objects on c->argv. */
 void rewriteClientCommandArgument(client *c, int i, robj *newval) {
     robj *oldval;
-
+    retainOriginalCommandVector(c);
     if (i >= c->argc) {
         c->argv = zrealloc(c->argv,sizeof(robj*)*(i+1));
         c->argc = i+1;
diff --git a/src/server.c b/src/server.c
index 883a81305..0cc73dfe3 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3467,8 +3467,13 @@ void call(client *c, int flags) {
         char *latency_event = (c->cmd->flags & CMD_FAST) ?
                               "fast-command" : "command";
         latencyAddSampleIfNeeded(latency_event,duration/1000);
-        slowlogPushEntryIfNeeded(c,c->argv,c->argc,duration);
+        /* If command argument vector was rewritten, use the original
+         * arguments. */
+        robj **argv = c->original_argv ? c->original_argv : c->argv;
+        int argc = c->original_argv ? c->original_argc : c->argc;
+        slowlogPushEntryIfNeeded(c,argv,argc,duration);
     }
+    freeClientOriginalArgv(c);
 
     if (flags & CMD_CALL_STATS) {
         /* use the real command that was executed (cmd and lastamc) may be
diff --git a/src/server.h b/src/server.h
index 587779844..1ae96aee5 100644
--- a/src/server.h
+++ b/src/server.h
@@ -808,6 +808,8 @@ typedef struct client {
     size_t querybuf_peak;   /* Recent (100ms or more) peak of querybuf size. */
     int argc;               /* Num of arguments of current command. */
     robj **argv;            /* Arguments of current command. */
+    int original_argc;      /* Num of arguments of original command if arguments were rewritten. */
+    robj **original_argv;   /* Arguments of original command if arguments were rewritten. */
     size_t argv_len_sum;    /* Sum of lengths of objects in argv list. */
     struct redisCommand *cmd, *lastcmd;  /* Last command executed. */
     user *user;             /* User associated with this connection. If the
@@ -1668,6 +1670,7 @@ void closeTimedoutClients(void);
 void freeClient(client *c);
 void freeClientAsync(client *c);
 void resetClient(client *c);
+void freeClientOriginalArgv(client *c);
 void sendReplyToClient(connection *conn);
 void *addReplyDeferredLen(client *c);
 void setDeferredArrayLen(client *c, void *node, long length);
diff --git a/tests/unit/slowlog.tcl b/tests/unit/slowlog.tcl
index c2ef23cf7..e782682e4 100644
--- a/tests/unit/slowlog.tcl
+++ b/tests/unit/slowlog.tcl
@@ -41,6 +41,41 @@ start_server {tags {"slowlog"} overrides {slowlog-log-slower-than 1000000}} {
         assert_equal {foobar} [lindex $e 5]
     }
 
+    test {SLOWLOG - Rewritten commands are logged as their original command} {
+        r config set slowlog-log-slower-than 0
+
+        # Test rewriting client arguments
+        r sadd set a b c d e
+        r slowlog reset
+
+        # SPOP is rewritten as DEL when all keys are removed
+        r spop set 10
+        assert_equal {spop set 10} [lindex [lindex [r slowlog get] 0] 3]
+
+        # Test replacing client arguments
+        r slowlog reset
+
+        # GEOADD is replicated as ZADD
+        r geoadd cool-cities -122.33207 47.60621 Seattle
+        assert_equal {geoadd cool-cities -122.33207 47.60621 Seattle} [lindex [lindex [r slowlog get] 0] 3]
+
+        # Test replacing a single command argument
+        r set A 5
+        r slowlog reset
+        
+        # GETSET is replicated as SET
+        r getset a 5
+        assert_equal {getset a 5} [lindex [lindex [r slowlog get] 0] 3]
+
+        # INCRBYFLOAT calls rewrite multiple times, so it's a special case
+        r set A 0
+        r slowlog reset
+        
+        # INCRBYFLOAT is replicated as SET
+        r INCRBYFLOAT A 1.0
+        assert_equal {INCRBYFLOAT A 1.0} [lindex [lindex [r slowlog get] 0] 3]
+    }
+
     test {SLOWLOG - commands with too many arguments are trimmed} {
         r config set slowlog-log-slower-than 0
         r slowlog reset

From dd1f20edc5ecda7848c31601782c5e9d7bce4788 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Wed, 11 Nov 2020 01:22:17 -0500
Subject: [PATCH 0795/1098] add tracking bcast flag and client redirection in
 client list (#7995)

---
 src/networking.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 13793590b..626b117a1 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2169,6 +2169,7 @@ sds catClientInfoString(sds s, client *client) {
     if (client->flags & CLIENT_BLOCKED) *p++ = 'b';
     if (client->flags & CLIENT_TRACKING) *p++ = 't';
     if (client->flags & CLIENT_TRACKING_BROKEN_REDIR) *p++ = 'R';
+    if (client->flags & CLIENT_TRACKING_BCAST) *p++ = 'B';
     if (client->flags & CLIENT_DIRTY_CAS) *p++ = 'd';
     if (client->flags & CLIENT_CLOSE_AFTER_REPLY) *p++ = 'c';
     if (client->flags & CLIENT_UNBLOCKED) *p++ = 'u';
@@ -2198,7 +2199,7 @@ sds catClientInfoString(sds s, client *client) {
         total_mem += zmalloc_size(client->argv);
 
     return sdscatfmt(s,
-        "id=%U addr=%s laddr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U argv-mem=%U obl=%U oll=%U omem=%U tot-mem=%U events=%s cmd=%s user=%s",
+        "id=%U addr=%s laddr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U argv-mem=%U obl=%U oll=%U omem=%U tot-mem=%U events=%s cmd=%s user=%s redir=%I",
         (unsigned long long) client->id,
         getClientPeerId(client),
         getClientSockname(client),
@@ -2220,7 +2221,8 @@ sds catClientInfoString(sds s, client *client) {
         (unsigned long long) total_mem,
         events,
         client->lastcmd ? client->lastcmd->name : "NULL",
-        client->user ? client->user->name : "(superuser)");
+        client->user ? client->user->name : "(superuser)",
+        (client->flags & CLIENT_TRACKING) ? (long long) client->client_tracking_redirection : -1);
 }
 
 sds getAllClientsInfoString(int type) {

From d5059ba5e15c42def95c4c5054cdd8a91ed11efc Mon Sep 17 00:00:00 2001
From: tzongw <tzongw@users.noreply.github.com>
Date: Wed, 11 Nov 2020 18:10:54 +0800
Subject: [PATCH 0796/1098] Invalidate aeTimer when returning AE_NOMORE (#8022)

---
 src/module.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/module.c b/src/module.c
index b4a15acae..691a9d298 100644
--- a/src/module.c
+++ b/src/module.c
@@ -5453,7 +5453,12 @@ int moduleTimerHandler(struct aeEventLoop *eventLoop, long long id, void *client
 
     /* Reschedule the next timer or cancel it. */
     if (next_period <= 0) next_period = 1;
-    return (raxSize(Timers) > 0) ? next_period : AE_NOMORE;
+    if (raxSize(Timers) > 0) {
+        return next_period;
+    } else {
+        aeTimer = -1;
+        return AE_NOMORE;
+    }
 }
 
 /* Create a new timer that will fire after `period` milliseconds, and will call

From 48a9d63af3c288527c9abbf13206960c0055f62b Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 11 Nov 2020 22:57:33 +0200
Subject: [PATCH 0797/1098] Add timer module API tests. (#8041)

---
 tests/modules/Makefile         |   3 +-
 tests/modules/timer.c          | 103 +++++++++++++++++++++++++++++++++
 tests/unit/moduleapi/timer.tcl |  57 ++++++++++++++++++
 3 files changed, 162 insertions(+), 1 deletion(-)
 create mode 100644 tests/modules/timer.c
 create mode 100644 tests/unit/moduleapi/timer.tcl

diff --git a/tests/modules/Makefile b/tests/modules/Makefile
index a8b08ecf0..8df05ed01 100644
--- a/tests/modules/Makefile
+++ b/tests/modules/Makefile
@@ -25,7 +25,8 @@ TEST_MODULES = \
     auth.so \
     keyspace_events.so \
     blockedclient.so \
-    getkeys.so
+    getkeys.so \
+    timer.so
 
 
 .PHONY: all
diff --git a/tests/modules/timer.c b/tests/modules/timer.c
new file mode 100644
index 000000000..d2a8d8e74
--- /dev/null
+++ b/tests/modules/timer.c
@@ -0,0 +1,103 @@
+
+#define REDISMODULE_EXPERIMENTAL_API
+#include "redismodule.h"
+
+static void timer_callback(RedisModuleCtx *ctx, void *data)
+{
+    RedisModuleString *keyname = data;
+    RedisModuleCallReply *reply;
+
+    reply = RedisModule_Call(ctx, "INCR", "s", keyname);
+    if (reply != NULL)
+        RedisModule_FreeCallReply(reply);
+    RedisModule_FreeString(ctx, keyname);
+}
+
+int test_createtimer(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    if (argc != 3) {
+        RedisModule_WrongArity(ctx);
+        return REDISMODULE_OK;
+    }
+
+    long long period;
+    if (RedisModule_StringToLongLong(argv[1], &period) == REDISMODULE_ERR) {
+        RedisModule_ReplyWithError(ctx, "Invalid time specified.");
+        return REDISMODULE_OK;
+    }
+
+    RedisModuleString *keyname = argv[2];
+    RedisModule_RetainString(ctx, keyname);
+
+    RedisModuleTimerID id = RedisModule_CreateTimer(ctx, period, timer_callback, keyname);
+    RedisModule_ReplyWithLongLong(ctx, id);
+
+    return REDISMODULE_OK;
+}
+
+int test_gettimer(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    if (argc != 2) {
+        RedisModule_WrongArity(ctx);
+        return REDISMODULE_OK;
+    }
+
+    long long id;
+    if (RedisModule_StringToLongLong(argv[1], &id) == REDISMODULE_ERR) {
+        RedisModule_ReplyWithError(ctx, "Invalid id specified.");
+        return REDISMODULE_OK;
+    }
+
+    uint64_t remaining;
+    RedisModuleString *keyname;
+    if (RedisModule_GetTimerInfo(ctx, id, &remaining, (void **)&keyname) == REDISMODULE_ERR) {
+        RedisModule_ReplyWithNull(ctx);
+    } else {
+        RedisModule_ReplyWithArray(ctx, 2);
+        RedisModule_ReplyWithString(ctx, keyname);
+        RedisModule_ReplyWithLongLong(ctx, remaining);
+    }
+
+    return REDISMODULE_OK;
+}
+
+int test_stoptimer(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    if (argc != 2) {
+        RedisModule_WrongArity(ctx);
+        return REDISMODULE_OK;
+    }
+
+    long long id;
+    if (RedisModule_StringToLongLong(argv[1], &id) == REDISMODULE_ERR) {
+        RedisModule_ReplyWithError(ctx, "Invalid id specified.");
+        return REDISMODULE_OK;
+    }
+
+    int ret = 0;
+    RedisModuleString *keyname;
+    if (RedisModule_StopTimer(ctx, id, (void **) &keyname) == REDISMODULE_OK) {
+        RedisModule_FreeString(ctx, keyname);
+        ret = 1;
+    }
+
+    RedisModule_ReplyWithLongLong(ctx, ret);
+    return REDISMODULE_OK;
+}
+
+
+int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+    if (RedisModule_Init(ctx,"timer",1,REDISMODULE_APIVER_1)== REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx,"test.createtimer", test_createtimer,"",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+    if (RedisModule_CreateCommand(ctx,"test.gettimer", test_gettimer,"",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+    if (RedisModule_CreateCommand(ctx,"test.stoptimer", test_stoptimer,"",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    return REDISMODULE_OK;
+}
diff --git a/tests/unit/moduleapi/timer.tcl b/tests/unit/moduleapi/timer.tcl
new file mode 100644
index 000000000..10a902276
--- /dev/null
+++ b/tests/unit/moduleapi/timer.tcl
@@ -0,0 +1,57 @@
+set testmodule [file normalize tests/modules/timer.so]
+
+start_server {tags {"modules"}} {
+    r module load $testmodule
+
+    test {RM_CreateTimer: a sequence of timers work} {
+        # We can't guarantee same-ms but we try using MULTI/EXEC
+        r multi
+        for {set i 0} {$i < 20} {incr i} {
+            r test.createtimer 10 timer-incr-key
+        }
+        r exec
+
+        after 500
+        assert_equal 20 [r get timer-incr-key]
+    }
+
+    test {RM_GetTimer: basic sanity} {
+        # Getting non-existing timer
+        assert_equal {} [r test.gettimer 0]
+
+        # Getting a real timer
+        set id [r test.createtimer 10000 timer-incr-key]
+        set info [r test.gettimer $id]
+
+        assert_equal "timer-incr-key" [lindex $info 0]
+        set remaining [lindex $info 1]
+        assert {$remaining < 10000 && $remaining > 1}
+    }
+
+    test {RM_StopTimer: basic sanity} {
+        r set "timer-incr-key" 0
+        set id [r test.createtimer 1000 timer-incr-key]
+
+        assert_equal 1 [r test.stoptimer $id]
+
+        # Wait to be sure timer doesn't execute
+        after 2000
+        assert_equal 0 [r get timer-incr-key]
+
+        # Stop non-existing timer
+        assert_equal 0 [r test.stoptimer $id]
+    }
+
+    test {Timer appears non-existing after it fires} {
+        r set "timer-incr-key" 0
+        set id [r test.createtimer 10 timer-incr-key]
+
+        # verify timer fired
+        after 500
+        assert_equal 1 [r get timer-incr-key]
+
+        # verify id does not exist
+        assert_equal {} [r test.gettimer $id]
+    }
+}
+

From a0576bdeea5210e80c070338b245d7548e97a6aa Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Wed, 11 Nov 2020 13:58:05 -0800
Subject: [PATCH 0798/1098] Initialize original client argv for aof (#8042)

(this is a fixup for #8006)

Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
---
 src/aof.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/aof.c b/src/aof.c
index f3176934f..5cb78d077 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -669,6 +669,8 @@ struct client *createAOFClient(void) {
     c->querybuf_peak = 0;
     c->argc = 0;
     c->argv = NULL;
+    c->original_argc = 0;
+    c->original_argv = NULL;
     c->argv_len_sum = 0;
     c->bufpos = 0;
     c->flags = 0;
@@ -704,6 +706,7 @@ void freeFakeClient(struct client *c) {
     listRelease(c->reply);
     listRelease(c->watched_keys);
     freeClientMultiState(c);
+    freeClientOriginalArgv(c);
     zfree(c);
 }
 

From c170365dcf4698080aeac32e4209ebb2ef7c6c6c Mon Sep 17 00:00:00 2001
From: Yash Ladha <201551061@iiitvadodara.ac.in>
Date: Thu, 12 Nov 2020 14:25:51 +0530
Subject: [PATCH 0799/1098] cleanup: move list pop logic to single function
 (#7997)

BLPOP when there are elements in the list works in the same way as LPOP
does. Due to this they also does the same repetitive action and logic
for the same is written at two different places. This is a bad code
practice as the one needs the context to change the BLPOP list pop code
as well when the LPOP code gets changed.

Separated the generic logic from LPOP to a function that is being used
by the BLPOP code as well.
---
 src/server.h |  1 +
 src/t_list.c | 43 +++++++++++++++++++------------------------
 2 files changed, 20 insertions(+), 24 deletions(-)

diff --git a/src/server.h b/src/server.h
index 1ae96aee5..8c179293a 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1793,6 +1793,7 @@ void listTypeDelete(listTypeIterator *iter, listTypeEntry *entry);
 void listTypeConvert(robj *subject, int enc);
 void unblockClientWaitingData(client *c);
 void popGenericCommand(client *c, int where);
+void listElementsRemoved(client *c, robj *key, int where, robj *o);
 
 /* MULTI/EXEC/WATCH... */
 void unwatchAllKeys(client *c);
diff --git a/src/t_list.c b/src/t_list.c
index e7a69b1e9..25dda3178 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -363,26 +363,30 @@ void lsetCommand(client *c) {
     }
 }
 
-void popGenericCommand(client *c, int where) {
-    robj *o = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp]);
-    if (o == NULL || checkType(c,o,OBJ_LIST)) return;
+void listElementsRemoved(client *c, robj *key, int where, robj *o) {
+    char *event = (where == LIST_HEAD) ? "lpop" : "rpop";
 
-    robj *value = listTypePop(o,where);
+    notifyKeyspaceEvent(NOTIFY_LIST, event, key, c->db->id);
+    if (listTypeLength(o) == 0) {
+        notifyKeyspaceEvent(NOTIFY_GENERIC, "del", key, c->db->id);
+        dbDelete(c->db, key);
+    }
+    signalModifiedKey(c, c->db, key);
+    server.dirty++;
+}
+
+void popGenericCommand(client *c, int where) {
+    robj *o = lookupKeyWriteOrReply(c, c->argv[1], shared.null[c->resp]);
+    if (o == NULL || checkType(c, o, OBJ_LIST))
+        return;
+
+    robj *value = listTypePop(o, where);
     if (value == NULL) {
         addReplyNull(c);
     } else {
-        char *event = (where == LIST_HEAD) ? "lpop" : "rpop";
-
         addReplyBulk(c,value);
         decrRefCount(value);
-        notifyKeyspaceEvent(NOTIFY_LIST,event,c->argv[1],c->db->id);
-        if (listTypeLength(o) == 0) {
-            notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
-                                c->argv[1],c->db->id);
-            dbDelete(c->db,c->argv[1]);
-        }
-        signalModifiedKey(c,c->db,c->argv[1]);
-        server.dirty++;
+        listElementsRemoved(c,c->argv[1],where,o);
     }
 }
 
@@ -859,7 +863,6 @@ void blockingPopGenericCommand(client *c, int where) {
             } else {
                 if (listTypeLength(o) != 0) {
                     /* Non empty list, this is like a normal [LR]POP. */
-                    char *event = (where == LIST_HEAD) ? "lpop" : "rpop";
                     robj *value = listTypePop(o,where);
                     serverAssert(value != NULL);
 
@@ -867,15 +870,7 @@ void blockingPopGenericCommand(client *c, int where) {
                     addReplyBulk(c,c->argv[j]);
                     addReplyBulk(c,value);
                     decrRefCount(value);
-                    notifyKeyspaceEvent(NOTIFY_LIST,event,
-                                        c->argv[j],c->db->id);
-                    if (listTypeLength(o) == 0) {
-                        dbDelete(c->db,c->argv[j]);
-                        notifyKeyspaceEvent(NOTIFY_GENERIC,"del",
-                                            c->argv[j],c->db->id);
-                    }
-                    signalModifiedKey(c,c->db,c->argv[j]);
-                    server.dirty++;
+                    listElementsRemoved(c,c->argv[j],where,o);
 
                     /* Replicate it as an [LR]POP instead of B[LR]POP. */
                     rewriteClientCommandVector(c,2,

From 92ec5925202db79944e267f3bc367ec3d7bdd6fe Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Thu, 12 Nov 2020 19:19:18 +0200
Subject: [PATCH 0800/1098] Adds user parsing to redis-cli URIs (#8048)

---
 src/redis-cli.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 44ea504c9..c09a6b798 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -411,7 +411,7 @@ static void parseRedisUri(const char *uri) {
     /* Extract user info. */
     if ((userinfo = strchr(curr,'@'))) {
         if ((username = strchr(curr, ':')) && username < userinfo) {
-            /* If provided, username is ignored. */
+            config.user = percentDecode(curr, username - curr);
             curr = username + 1;
         }
 

From 204a14b8cffba6a23e4e313e248bed7ef5cd8260 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Fri, 13 Nov 2020 21:16:40 +0800
Subject: [PATCH 0801/1098] Fix make warning in hellohook,c and testmodule.c
 (#8044)

* Fix build warning in hellohook,c and testmodule.c

* Change ci->id type to (unsigned long long)
---
 src/modules/hellohook.c  | 2 +-
 src/modules/testmodule.c | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/modules/hellohook.c b/src/modules/hellohook.c
index 7ab78ed07..b451bd975 100644
--- a/src/modules/hellohook.c
+++ b/src/modules/hellohook.c
@@ -47,7 +47,7 @@ void clientChangeCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub,
     printf("Client %s event for client #%llu %s:%d\n",
         (sub == REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED) ?
             "connection" : "disconnection",
-        ci->id,ci->addr,ci->port);
+        (unsigned long long)ci->id,ci->addr,ci->port);
 }
 
 void flushdbCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data)
diff --git a/src/modules/testmodule.c b/src/modules/testmodule.c
index 5381380e5..5634530dd 100644
--- a/src/modules/testmodule.c
+++ b/src/modules/testmodule.c
@@ -220,7 +220,7 @@ int TestNotifications(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     } else {
         rep = RedisModule_CallReplyStringPtr(r, &sz);
         if (sz != 1 || *rep != '1') {
-            FAIL("Got reply '%.*s'. expected '1'", sz, rep);
+            FAIL("Got reply '%.*s'. expected '1'", (int)sz, rep);
         }
     }
     /* For l we expect nothing since we didn't subscribe to list events */
@@ -235,7 +235,7 @@ int TestNotifications(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     } else {
         rep = RedisModule_CallReplyStringPtr(r, &sz);
         if (sz != 1 || *rep != '2') {
-            FAIL("Got reply '%.*s'. expected '2'", sz, rep);
+            FAIL("Got reply '%.*s'. expected '2'", (int)sz, rep);
         }
     }
 

From d8fd48c4363dd5265bb944c6ec881368d67f4afd Mon Sep 17 00:00:00 2001
From: Felipe Machado <462154+felipou@users.noreply.github.com>
Date: Sun, 15 Nov 2020 09:14:25 -0300
Subject: [PATCH 0802/1098] Add new commands ZDIFF and ZDIFFSTORE (#7961)

- Add ZDIFF and ZDIFFSTORE which work similarly to SDIFF and SDIFFSTORE
- Make sure the new WITHSCORES argument that was added for ZUNION isn't considered valid for ZUNIONSTORE

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 redis.conf               |  14 +-
 src/db.c                 |   4 +-
 src/server.c             |  16 ++-
 src/server.h             |   6 +-
 src/t_zset.c             | 278 ++++++++++++++++++++++++++++++++++-----
 tests/unit/type/zset.tcl |  89 ++++++++++++-
 6 files changed, 352 insertions(+), 55 deletions(-)

diff --git a/redis.conf b/redis.conf
index a46668826..d28c17b64 100644
--- a/redis.conf
+++ b/redis.conf
@@ -901,14 +901,12 @@ acllog-max-len 128
 # Both LRU, LFU and volatile-ttl are implemented using approximated
 # randomized algorithms.
 #
-# Note: with any of the above policies, Redis will return an error on write
-#       operations, when there are no suitable keys for eviction.
-#
-#       At the date of writing these commands are: set setnx setex append
-#       incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd
-#       sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby
-#       zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby
-#       getset mset msetnx exec sort
+# Note: with any of the above policies, when there are no suitable keys for
+# eviction, Redis will return an error on write operations that require
+# more memory. These are usually commands that create new keys, add data or
+# modify existing keys. A few examples are: SET, INCR, HSET, LPUSH, SUNIONSTORE,
+# SORT (due to the STORE argument), and EXEC (if the transaction includes any
+# command that requires memory).
 #
 # The default is:
 #
diff --git a/src/db.c b/src/db.c
index 0f37c7441..1ca2fe086 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1441,12 +1441,12 @@ int genericGetKeys(int storeKeyOfs, int keyCountOfs, int firstKeyOfs, int keySte
     return result->numkeys;
 }
 
-int zunionInterStoreGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
+int zunionInterDiffStoreGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
     UNUSED(cmd);
     return genericGetKeys(1, 2, 3, 1, argv, argc, result);
 }
 
-int zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
+int zunionInterDiffGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
     UNUSED(cmd);
     return genericGetKeys(0, 1, 2, 1, argv, argc, result);
 }
diff --git a/src/server.c b/src/server.c
index 0cc73dfe3..04b6d98ba 100644
--- a/src/server.c
+++ b/src/server.c
@@ -436,19 +436,27 @@ struct redisCommand redisCommandTable[] = {
 
     {"zunionstore",zunionstoreCommand,-4,
      "write use-memory @sortedset",
-     0,zunionInterStoreGetKeys,0,0,0,0,0,0},
+     0,zunionInterDiffStoreGetKeys,0,0,0,0,0,0},
 
     {"zinterstore",zinterstoreCommand,-4,
      "write use-memory @sortedset",
-     0,zunionInterStoreGetKeys,0,0,0,0,0,0},
+     0,zunionInterDiffStoreGetKeys,0,0,0,0,0,0},
+
+    {"zdiffstore",zdiffstoreCommand,-4,
+     "write use-memory @sortedset",
+     0,zunionInterDiffStoreGetKeys,0,0,0,0,0,0},
 
     {"zunion",zunionCommand,-3,
      "read-only @sortedset",
-     0,zunionInterGetKeys,0,0,0,0,0,0},
+     0,zunionInterDiffGetKeys,0,0,0,0,0,0},
 
     {"zinter",zinterCommand,-3,
      "read-only @sortedset",
-     0,zunionInterGetKeys,0,0,0,0,0,0},
+     0,zunionInterDiffGetKeys,0,0,0,0,0,0},
+
+    {"zdiff",zdiffCommand,-3,
+     "read-only @sortedset",
+     0,zunionInterDiffGetKeys,0,0,0,0,0,0},
 
     {"zrange",zrangeCommand,-4,
      "read-only @sortedset",
diff --git a/src/server.h b/src/server.h
index 8c179293a..c6d5f987a 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2207,8 +2207,8 @@ void freeObjAsync(robj *o);
 int *getKeysPrepareResult(getKeysResult *result, int numkeys);
 int getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
 void getKeysFreeResult(getKeysResult *result);
-int zunionInterGetKeys(struct redisCommand *cmd,robj **argv, int argc, getKeysResult *result);
-int zunionInterStoreGetKeys(struct redisCommand *cmd,robj **argv, int argc, getKeysResult *result);
+int zunionInterDiffGetKeys(struct redisCommand *cmd,robj **argv, int argc, getKeysResult *result);
+int zunionInterDiffStoreGetKeys(struct redisCommand *cmd,robj **argv, int argc, getKeysResult *result);
 int evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
 int sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
 int migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
@@ -2427,8 +2427,10 @@ void hstrlenCommand(client *c);
 void zremrangebyrankCommand(client *c);
 void zunionstoreCommand(client *c);
 void zinterstoreCommand(client *c);
+void zdiffstoreCommand(client *c);
 void zunionCommand(client *c);
 void zinterCommand(client *c);
+void zdiffCommand(client *c);
 void zscanCommand(client *c);
 void hkeysCommand(client *c);
 void hvalsCommand(client *c);
diff --git a/src/t_zset.c b/src/t_zset.c
index 5fc7643c3..858085a05 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -1435,6 +1435,36 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) {
     return 0; /* Never reached. */
 }
 
+/* Deletes the element 'ele' from the sorted set encoded as a skiplist+dict,
+ * returning 1 if the element existed and was deleted, 0 otherwise (the
+ * element was not there). It does not resize the dict after deleting the
+ * element. */
+static int zsetRemoveFromSkiplist(zset *zs, sds ele) {
+    dictEntry *de;
+    double score;
+
+    de = dictUnlink(zs->dict,ele);
+    if (de != NULL) {
+        /* Get the score in order to delete from the skiplist later. */
+        score = *(double*)dictGetVal(de);
+
+        /* Delete from the hash table and later from the skiplist.
+         * Note that the order is important: deleting from the skiplist
+         * actually releases the SDS string representing the element,
+         * which is shared between the skiplist and the hash table, so
+         * we need to delete from the skiplist as the final step. */
+        dictFreeUnlinkedEntry(zs->dict,de);
+
+        /* Delete from skiplist. */
+        int retval = zslDelete(zs->zsl,score,ele,NULL);
+        serverAssert(retval);
+
+        return 1;
+    }
+
+    return 0;
+}
+
 /* Delete the element 'ele' from the sorted set, returning 1 if the element
  * existed and was deleted, 0 otherwise (the element was not there). */
 int zsetDel(robj *zobj, sds ele) {
@@ -1447,25 +1477,7 @@ int zsetDel(robj *zobj, sds ele) {
         }
     } else if (zobj->encoding == OBJ_ENCODING_SKIPLIST) {
         zset *zs = zobj->ptr;
-        dictEntry *de;
-        double score;
-
-        de = dictUnlink(zs->dict,ele);
-        if (de != NULL) {
-            /* Get the score in order to delete from the skiplist later. */
-            score = *(double*)dictGetVal(de);
-
-            /* Delete from the hash table and later from the skiplist.
-             * Note that the order is important: deleting from the skiplist
-             * actually releases the SDS string representing the element,
-             * which is shared between the skiplist and the hash table, so
-             * we need to delete from the skiplist as the final step. */
-            dictFreeUnlinkedEntry(zs->dict,de);
-
-            /* Delete from skiplist. */
-            int retval = zslDelete(zs->zsl,score,ele,NULL);
-            serverAssert(retval);
-
+        if (zsetRemoveFromSkiplist(zs, ele)) {
             if (htNeedsResize(zs->dict)) dictResize(zs->dict);
             return 1;
         }
@@ -2162,6 +2174,10 @@ int zuiCompareByCardinality(const void *s1, const void *s2) {
     return 0;
 }
 
+static int zuiCompareByRevCardinality(const void *s1, const void *s2) {
+    return zuiCompareByCardinality(s1, s2) * -1;
+}
+
 #define REDIS_AGGR_SUM 1
 #define REDIS_AGGR_MIN 2
 #define REDIS_AGGR_MAX 3
@@ -2184,6 +2200,180 @@ inline static void zunionInterAggregate(double *target, double val, int aggregat
     }
 }
 
+static int zsetDictGetMaxElementLength(dict *d) {
+    dictIterator *di;
+    dictEntry *de;
+    size_t maxelelen = 0;
+
+    di = dictGetIterator(d);
+
+    while((de = dictNext(di)) != NULL) {
+        sds ele = dictGetKey(de);
+        if (sdslen(ele) > maxelelen) maxelelen = sdslen(ele);
+    }
+
+    dictReleaseIterator(di);
+
+    return maxelelen;
+}
+
+static void zdiffAlgorithm1(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen) {
+    /* DIFF Algorithm 1:
+     *
+     * We perform the diff by iterating all the elements of the first set,
+     * and only adding it to the target set if the element does not exist
+     * into all the other sets.
+     *
+     * This way we perform at max N*M operations, where N is the size of
+     * the first set, and M the number of sets.
+     *
+     * There is also a O(K*log(K)) cost for adding the resulting elements
+     * to the target set, where K is the final size of the target set.
+     *
+     * The final complexity of this algorithm is O(N*M + K*log(K)). */
+    int j;
+    zsetopval zval;
+    zskiplistNode *znode;
+    sds tmp;
+
+    /* With algorithm 1 it is better to order the sets to subtract
+     * by decreasing size, so that we are more likely to find
+     * duplicated elements ASAP. */
+    qsort(src+1,setnum-1,sizeof(zsetopsrc),zuiCompareByRevCardinality);
+
+    memset(&zval, 0, sizeof(zval));
+    zuiInitIterator(&src[0]);
+    while (zuiNext(&src[0],&zval)) {
+        double value;
+        int exists = 0;
+
+        for (j = 1; j < setnum; j++) {
+            /* It is not safe to access the zset we are
+             * iterating, so explicitly check for equal object.
+             * This check isn't really needed anymore since we already
+             * check for a duplicate set in the zsetChooseDiffAlgorithm
+             * function, but we're leaving it for future-proofing. */
+            if (src[j].subject == src[0].subject ||
+                zuiFind(&src[j],&zval,&value)) {
+                exists = 1;
+                break;
+            }
+        }
+
+        if (!exists) {
+            tmp = zuiNewSdsFromValue(&zval);
+            znode = zslInsert(dstzset->zsl,zval.score,tmp);
+            dictAdd(dstzset->dict,tmp,&znode->score);
+            if (sdslen(tmp) > *maxelelen) *maxelelen = sdslen(tmp);
+        }
+    }
+}
+
+
+static void zdiffAlgorithm2(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen) {
+    /* DIFF Algorithm 2:
+     *
+     * Add all the elements of the first set to the auxiliary set.
+     * Then remove all the elements of all the next sets from it.
+     *
+
+     * This is O(L + (N-K)log(N)) where L is the sum of all the elements in every
+     * set, N is the size of the first set, and K is the size of the result set.
+     *
+     * Note that from the (L-N) dict searches, (N-K) got to the zsetRemoveFromSkiplist
+     * which costs log(N)
+     *
+     * There is also a O(K) cost at the end for finding the largest element
+     * size, but this doesn't change the algorithm complexity since K < L, and
+     * O(2L) is the same as O(L). */
+    int j;
+    int cardinality = 0;
+    zsetopval zval;
+    zskiplistNode *znode;
+    sds tmp;
+
+    for (j = 0; j < setnum; j++) {
+        if (zuiLength(&src[j]) == 0) continue;
+
+        memset(&zval, 0, sizeof(zval));
+        zuiInitIterator(&src[j]);
+        while (zuiNext(&src[j],&zval)) {
+            if (j == 0) {
+                tmp = zuiNewSdsFromValue(&zval);
+                znode = zslInsert(dstzset->zsl,zval.score,tmp);
+                dictAdd(dstzset->dict,tmp,&znode->score);
+                cardinality++;
+            } else {
+                tmp = zuiNewSdsFromValue(&zval);
+                if (zsetRemoveFromSkiplist(dstzset, tmp)) {
+                    cardinality--;
+                }
+            }
+
+            /* Exit if result set is empty as any additional removal
+                * of elements will have no effect. */
+            if (cardinality == 0) break;
+        }
+
+        if (cardinality == 0) break;
+    }
+
+    /* Redize dict if needed after removing multiple elements */
+    if (htNeedsResize(dstzset->dict)) dictResize(dstzset->dict);
+
+    /* Using this algorithm, we can't calculate the max element as we go,
+     * we have to iterate through all elements to find the max one after. */
+    *maxelelen = zsetDictGetMaxElementLength(dstzset->dict);
+}
+
+static int zsetChooseDiffAlgorithm(zsetopsrc *src, long setnum) {
+    int j;
+
+    /* Select what DIFF algorithm to use.
+     *
+     * Algorithm 1 is O(N*M + K*log(K)) where N is the size of the
+     * first set, M the total number of sets, and K is the size of the
+     * result set.
+     *
+     * Algorithm 2 is O(L + (N-K)log(N)) where L is the total number of elements
+     * in all the sets, N is the size of the first set, and K is the size of the
+     * result set.
+     *
+     * We compute what is the best bet with the current input here. */
+    long long algo_one_work = 0;
+    long long algo_two_work = 0;
+
+    for (j = 0; j < setnum; j++) {
+        /* If any other set is equal to the first set, there is nothing to be
+         * done, since we would remove all elements anyway. */
+        if (j > 0 && src[0].subject == src[j].subject) {
+            return 0;
+        }
+
+        algo_one_work += zuiLength(&src[0]);
+        algo_two_work += zuiLength(&src[j]);
+    }
+
+    /* Algorithm 1 has better constant times and performs less operations
+     * if there are elements in common. Give it some advantage. */
+    algo_one_work /= 2;
+    return (algo_one_work <= algo_two_work) ? 1 : 2;
+}
+
+static void zdiff(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen) {
+    /* Skip everything if the smallest input is empty. */
+    if (zuiLength(&src[0]) > 0) {
+        int diff_algo = zsetChooseDiffAlgorithm(src, setnum);
+        if (diff_algo == 1) {
+            zdiffAlgorithm1(src, setnum, dstzset, maxelelen);
+        } else if (diff_algo == 2) {
+            zdiffAlgorithm2(src, setnum, dstzset, maxelelen);
+        } else if (diff_algo != 0) {
+            serverPanic("Unknown algorithm");
+        }
+    }
+}
+
 uint64_t dictSdsHash(const void *key);
 int dictSdsKeyCompare(void *privdata, const void *key1, const void *key2);
 
@@ -2196,15 +2386,15 @@ dictType setAccumulatorDictType = {
     NULL                       /* val destructor */
 };
 
-/* The zunionInterGenericCommand() function is called in order to implement the
- * following commands: ZUNION, ZINTER, ZUNIONSTORE, ZINTERSTORE.
+/* The zunionInterDiffGenericCommand() function is called in order to implement the
+ * following commands: ZUNION, ZINTER, ZDIFF, ZUNIONSTORE, ZINTERSTORE, ZDIFFSTORE.
  *
- * 'numkeysIndex' parameter position of key number. for ZUNION/ZINTER command, this
- * value is 1, for ZUNIONSTORE/ZINTERSTORE command, this value is 2.
+ * 'numkeysIndex' parameter position of key number. for ZUNION/ZINTER/ZDIFF command,
+ * this value is 1, for ZUNIONSTORE/ZINTERSTORE/ZDIFFSTORE command, this value is 2.
  *
- * 'op' SET_OP_INTER or SET_OP_UNION.
+ * 'op' SET_OP_INTER, SET_OP_UNION or SET_OP_DIFF.
  */
-void zunionInterGenericCommand(client *c, robj *dstkey, int numkeysIndex, int op) {
+void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, int op) {
     int i, j;
     long setnum;
     int aggregate = REDIS_AGGR_SUM;
@@ -2223,7 +2413,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int numkeysIndex, int op
 
     if (setnum < 1) {
         addReplyError(c,
-            "at least 1 input key is needed for ZUNIONSTORE/ZINTERSTORE");
+            "at least 1 input key is needed for ZUNIONSTORE/ZINTERSTORE/ZDIFFSTORE");
         return;
     }
 
@@ -2260,7 +2450,8 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int numkeysIndex, int op
         int remaining = c->argc - j;
 
         while (remaining) {
-            if (remaining >= (setnum + 1) &&
+            if (op != SET_OP_DIFF &&
+                remaining >= (setnum + 1) &&
                 !strcasecmp(c->argv[j]->ptr,"weights"))
             {
                 j++; remaining--;
@@ -2272,7 +2463,8 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int numkeysIndex, int op
                         return;
                     }
                 }
-            } else if (remaining >= 2 &&
+            } else if (op != SET_OP_DIFF &&
+                       remaining >= 2 &&
                        !strcasecmp(c->argv[j]->ptr,"aggregate"))
             {
                 j++; remaining--;
@@ -2289,6 +2481,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int numkeysIndex, int op
                 }
                 j++; remaining--;
             } else if (remaining >= 1 &&
+                       !dstkey &&
                        !strcasecmp(c->argv[j]->ptr,"withscores"))
             {
                 j++; remaining--;
@@ -2301,9 +2494,11 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int numkeysIndex, int op
         }
     }
 
-    /* sort sets from the smallest to largest, this will improve our
-     * algorithm's performance */
-    qsort(src,setnum,sizeof(zsetopsrc),zuiCompareByCardinality);
+    if (op != SET_OP_DIFF) {
+        /* sort sets from the smallest to largest, this will improve our
+        * algorithm's performance */
+        qsort(src,setnum,sizeof(zsetopsrc),zuiCompareByCardinality);
+    }
 
     dstobj = createZsetObject();
     dstzset = dstobj->ptr;
@@ -2409,6 +2604,8 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int numkeysIndex, int op
         }
         dictReleaseIterator(di);
         dictRelease(accumulator);
+    } else if (op == SET_OP_DIFF) {
+        zdiff(src, setnum, dstzset, &maxelelen);
     } else {
         serverPanic("Unknown operator");
     }
@@ -2419,7 +2616,8 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int numkeysIndex, int op
             setKey(c, c->db, dstkey, dstobj);
             addReplyLongLong(c, zsetLength(dstobj));
             notifyKeyspaceEvent(NOTIFY_ZSET,
-                                (op == SET_OP_UNION) ? "zunionstore" : "zinterstore",
+                                (op == SET_OP_UNION) ? "zunionstore" :
+                                    (op == SET_OP_INTER ? "zinterstore" : "zdiffstore"),
                                 dstkey, c->db->id);
             server.dirty++;
         } else {
@@ -2451,19 +2649,27 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int numkeysIndex, int op
 }
 
 void zunionstoreCommand(client *c) {
-    zunionInterGenericCommand(c, c->argv[1], 2, SET_OP_UNION);
+    zunionInterDiffGenericCommand(c, c->argv[1], 2, SET_OP_UNION);
 }
 
 void zinterstoreCommand(client *c) {
-    zunionInterGenericCommand(c, c->argv[1], 2, SET_OP_INTER);
+    zunionInterDiffGenericCommand(c, c->argv[1], 2, SET_OP_INTER);
+}
+
+void zdiffstoreCommand(client *c) {
+    zunionInterDiffGenericCommand(c, c->argv[1], 2, SET_OP_DIFF);
 }
 
 void zunionCommand(client *c) {
-    zunionInterGenericCommand(c, NULL, 1, SET_OP_UNION);
+    zunionInterDiffGenericCommand(c, NULL, 1, SET_OP_UNION);
 }
 
 void zinterCommand(client *c) {
-    zunionInterGenericCommand(c, NULL, 1, SET_OP_INTER);
+    zunionInterDiffGenericCommand(c, NULL, 1, SET_OP_INTER);
+}
+
+void zdiffCommand(client *c) {
+    zunionInterDiffGenericCommand(c, NULL, 1, SET_OP_DIFF);
 }
 
 void zrangeGenericCommand(client *c, int reverse) {
diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl
index 2f9eb823f..d8fccd33c 100644
--- a/tests/unit/type/zset.tcl
+++ b/tests/unit/type/zset.tcl
@@ -612,10 +612,11 @@ start_server {tags {"zset"}} {
             assert_equal 0 [r exists dst_key]
         }
 
-        test "ZUNION/ZINTER against non-existing key - $encoding" {
+        test "ZUNION/ZINTER/ZDIFF against non-existing key - $encoding" {
             r del zseta
             assert_equal {} [r zunion 1 zseta]
             assert_equal {} [r zinter 1 zseta]
+            assert_equal {} [r zdiff 1 zseta]
         }
 
         test "ZUNIONSTORE with empty set - $encoding" {
@@ -626,12 +627,13 @@ start_server {tags {"zset"}} {
             r zrange zsetc 0 -1 withscores
         } {a 1 b 2}
 
-        test "ZUNION/ZINTER with empty set - $encoding" {
+        test "ZUNION/ZINTER/ZDIFF with empty set - $encoding" {
             r del zseta zsetb
             r zadd zseta 1 a
             r zadd zseta 2 b
             assert_equal {a 1 b 2} [r zunion 2 zseta zsetb withscores]
             assert_equal {} [r zinter 2 zseta zsetb withscores]
+            assert_equal {a 1 b 2} [r zdiff 2 zseta zsetb withscores]
         }
 
         test "ZUNIONSTORE basics - $encoding" {
@@ -647,7 +649,7 @@ start_server {tags {"zset"}} {
             assert_equal {a 1 b 3 d 3 c 5} [r zrange zsetc 0 -1 withscores]
         }
 
-        test "ZUNION/ZINTER with integer members - $encoding" {
+        test "ZUNION/ZINTER/ZDIFF with integer members - $encoding" {
             r del zsetd zsetf
             r zadd zsetd 1 1
             r zadd zsetd 2 2
@@ -658,6 +660,7 @@ start_server {tags {"zset"}} {
 
             assert_equal {1 2 2 2 4 4 3 6} [r zunion 2 zsetd zsetf withscores]
             assert_equal {1 2 3 6} [r zinter 2 zsetd zsetf withscores]
+            assert_equal {2 2} [r zdiff 2 zsetd zsetf withscores]
         }
 
         test "ZUNIONSTORE with weights - $encoding" {
@@ -773,6 +776,80 @@ start_server {tags {"zset"}} {
             }
         }
 
+        test "ZDIFFSTORE basics - $encoding" {
+            assert_equal 1 [r zdiffstore zsetc 2 zseta zsetb]
+            assert_equal {a 1} [r zrange zsetc 0 -1 withscores]
+        }
+
+        test "ZDIFF basics - $encoding" {
+            assert_equal {a 1} [r zdiff 2 zseta zsetb withscores]
+        }
+
+        test "ZDIFFSTORE with a regular set - $encoding" {
+            r del seta
+            r sadd seta a
+            r sadd seta b
+            r sadd seta c
+            assert_equal 1 [r zdiffstore zsetc 2 seta zsetb]
+            assert_equal {a 1} [r zrange zsetc 0 -1 withscores]
+        }
+
+        test "ZDIFF subtracting set from itself - $encoding" {
+            assert_equal 0 [r zdiffstore zsetc 2 zseta zseta]
+            assert_equal {} [r zrange zsetc 0 -1 withscores]
+        }
+
+        test "ZDIFF algorithm 1 - $encoding" {
+            r del zseta zsetb zsetc
+            r zadd zseta 1 a
+            r zadd zseta 2 b
+            r zadd zseta 3 c
+            r zadd zsetb 1 b
+            r zadd zsetb 2 c
+            r zadd zsetb 3 d
+            assert_equal 1 [r zdiffstore zsetc 2 zseta zsetb]
+            assert_equal {a 1} [r zrange zsetc 0 -1 withscores]
+        }
+
+        test "ZDIFF algorithm 2 - $encoding" {
+            r del zseta zsetb zsetc zsetd zsete
+            r zadd zseta 1 a
+            r zadd zseta 2 b
+            r zadd zseta 3 c
+            r zadd zseta 5 e
+            r zadd zsetb 1 b
+            r zadd zsetc 1 c
+            r zadd zsetd 1 d
+            assert_equal 2 [r zdiffstore zsete 4 zseta zsetb zsetc zsetd]
+            assert_equal {a 1 e 5} [r zrange zsete 0 -1 withscores]
+        }
+
+        test "ZDIFF fuzzing" {
+            for {set j 0} {$j < 100} {incr j} {
+                unset -nocomplain s
+                array set s {}
+                set args {}
+                set num_sets [expr {[randomInt 10]+1}]
+                for {set i 0} {$i < $num_sets} {incr i} {
+                    set num_elements [randomInt 100]
+                    r del zset_$i
+                    lappend args zset_$i
+                    while {$num_elements} {
+                        set ele [randomValue]
+                        r zadd zset_$i [randomInt 100] $ele
+                        if {$i == 0} {
+                            set s($ele) x
+                        } else {
+                            unset -nocomplain s($ele)
+                        }
+                        incr num_elements -1
+                    }
+                }
+                set result [lsort [r zdiff [llength $args] {*}$args]]
+                assert_equal $result [lsort [array names s]]
+            }
+        }
+
         test "Basic ZPOP with a single key - $encoding" {
             r del zset
             assert_equal {} [r zpopmin zset]
@@ -888,6 +965,12 @@ start_server {tags {"zset"}} {
             set oldscore $score
         }
     }
+
+    test "ZUNIONSTORE/ZINTERSTORE/ZDIFFSTORE error if using WITHSCORES " {
+        assert_error "*ERR*syntax*" {r zunionstore foo 2 zsetd zsetf withscores}
+        assert_error "*ERR*syntax*" {r zinterstore foo 2 zsetd zsetf withscores}
+        assert_error "*ERR*syntax*" {r zdiffstore foo 2 zsetd zsetf withscores}
+    }
     
     test {ZMSCORE retrieve} {
         r del zmscoretest

From c1aaad06d85c89ab7abebd5cefab026bdcb086ab Mon Sep 17 00:00:00 2001
From: chenyangyang <chenyang8094@users.noreply.github.com>
Date: Mon, 16 Nov 2020 16:34:04 +0800
Subject: [PATCH 0803/1098] Modules callbacks for lazy free effort, and unlink
 (#7912)

Add two optional callbacks to the RedisModuleTypeMethods structure, which is `free_effort`
and `unlink`. the `free_effort` callback indicates the effort required to free a module memory.
Currently, if the effort exceeds LAZYFREE_THRESHOLD, the module memory may be released
asynchronously. the `unlink` callback indicates the key has been removed from the DB by redis, and
may soon be freed by a background thread.

Add `lazyfreed_objects` info field, which represents the number of objects that have been
lazyfreed since redis was started.

Add `RM_GetTypeMethodVersion` API, which return the current redis-server runtime value of
`REDISMODULE_TYPE_METHOD_VERSION`. You can use that when calling `RM_CreateDataType` to know
which fields of RedisModuleTypeMethods are gonna be supported and which will be ignored.
---
 runtest-moduleapi                      |   1 +
 src/db.c                               |  13 +-
 src/lazyfree.c                         |  40 ++++-
 src/module.c                           |  42 +++++-
 src/redismodule.h                      |  11 +-
 src/server.c                           |   6 +-
 src/server.h                           |   8 +-
 src/t_set.c                            |   2 +-
 tests/modules/Makefile                 |   3 +-
 tests/modules/test_lazyfree.c          | 196 +++++++++++++++++++++++++
 tests/unit/moduleapi/test_lazyfree.tcl |  32 ++++
 11 files changed, 338 insertions(+), 16 deletions(-)
 create mode 100644 tests/modules/test_lazyfree.c
 create mode 100644 tests/unit/moduleapi/test_lazyfree.tcl

diff --git a/runtest-moduleapi b/runtest-moduleapi
index f881dfd3f..fdf60886f 100755
--- a/runtest-moduleapi
+++ b/runtest-moduleapi
@@ -28,4 +28,5 @@ $TCLSH tests/test_helper.tcl \
 --single unit/moduleapi/keyspace_events \
 --single unit/moduleapi/blockedclient \
 --single unit/moduleapi/getkeys \
+--single unit/moduleapi/test_lazyfree \
 "${@}"
diff --git a/src/db.c b/src/db.c
index 1ca2fe086..f9570dc75 100644
--- a/src/db.c
+++ b/src/db.c
@@ -217,10 +217,14 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) {
     if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) {
         val->lru = old->lru;
     }
+    /* Although the key is not really deleted from the database, we regard 
+    overwrite as two steps of unlink+add, so we still need to call the unlink 
+    callback of the module. */
+    moduleNotifyKeyUnlink(key,val);
     dictSetVal(db->dict, de, val);
 
     if (server.lazyfree_lazy_server_del) {
-        freeObjAsync(old);
+        freeObjAsync(key,old);
         dictSetVal(db->dict, &auxentry, NULL);
     }
 
@@ -298,7 +302,12 @@ int dbSyncDelete(redisDb *db, robj *key) {
     /* Deleting an entry from the expires dict will not free the sds of
      * the key, because it is shared with the main dictionary. */
     if (dictSize(db->expires) > 0) dictDelete(db->expires,key->ptr);
-    if (dictDelete(db->dict,key->ptr) == DICT_OK) {
+    dictEntry *de = dictUnlink(db->dict,key->ptr);
+    if (de) {
+        robj *val = dictGetVal(de);
+        /* Tells the module that the key has been unlinked from the database. */
+        moduleNotifyKeyUnlink(key,val);
+        dictFreeUnlinkedEntry(db->dict,de);
         if (server.cluster_enabled) slotToKeyDel(key->ptr);
         return 1;
     } else {
diff --git a/src/lazyfree.c b/src/lazyfree.c
index b0fc26fcf..5a78d5a55 100644
--- a/src/lazyfree.c
+++ b/src/lazyfree.c
@@ -4,6 +4,7 @@
 #include "cluster.h"
 
 static redisAtomic size_t lazyfree_objects = 0;
+static redisAtomic size_t lazyfreed_objects = 0;
 
 /* Return the number of currently pending objects to free. */
 size_t lazyfreeGetPendingObjectsCount(void) {
@@ -12,6 +13,13 @@ size_t lazyfreeGetPendingObjectsCount(void) {
     return aux;
 }
 
+/* Return the number of objects that have been freed. */
+size_t lazyfreeGetFreedObjectsCount(void) {
+    size_t aux;
+    atomicGet(lazyfreed_objects,aux);
+    return aux;
+}
+
 /* Return the amount of work needed in order to free an object.
  * The return value is not always the actual number of allocations the
  * object is composed of, but a number proportional to it.
@@ -27,7 +35,7 @@ size_t lazyfreeGetPendingObjectsCount(void) {
  *
  * For lists the function returns the number of elements in the quicklist
  * representing the list. */
-size_t lazyfreeGetFreeEffort(robj *obj) {
+size_t lazyfreeGetFreeEffort(robj *key, robj *obj) {
     if (obj->type == OBJ_LIST) {
         quicklist *ql = obj->ptr;
         return ql->len;
@@ -64,6 +72,17 @@ size_t lazyfreeGetFreeEffort(robj *obj) {
             raxStop(&ri);
         }
         return effort;
+    } else if (obj->type == OBJ_MODULE) {
+        moduleValue *mv = obj->ptr;
+        moduleType *mt = mv->type;
+        if (mt->free_effort != NULL) {
+            size_t effort  = mt->free_effort(key,mv->value);
+            /* If the module's free_effort returns 0, it will use asynchronous free
+             memory by default */
+            return effort == 0 ? ULONG_MAX : effort;
+        } else {
+            return 1;
+        }
     } else {
         return 1; /* Everything else is a single allocation. */
     }
@@ -85,7 +104,11 @@ int dbAsyncDelete(redisDb *db, robj *key) {
     dictEntry *de = dictUnlink(db->dict,key->ptr);
     if (de) {
         robj *val = dictGetVal(de);
-        size_t free_effort = lazyfreeGetFreeEffort(val);
+
+        /* Tells the module that the key has been unlinked from the database. */
+        moduleNotifyKeyUnlink(key,val);
+
+        size_t free_effort = lazyfreeGetFreeEffort(key,val);
 
         /* If releasing the object is too much work, do it in the background
          * by adding the object to the lazy free list.
@@ -114,13 +137,13 @@ int dbAsyncDelete(redisDb *db, robj *key) {
 }
 
 /* Free an object, if the object is huge enough, free it in async way. */
-void freeObjAsync(robj *o) {
-    size_t free_effort = lazyfreeGetFreeEffort(o);
-    if (free_effort > LAZYFREE_THRESHOLD && o->refcount == 1) {
+void freeObjAsync(robj *key, robj *obj) {
+    size_t free_effort = lazyfreeGetFreeEffort(key,obj);
+    if (free_effort > LAZYFREE_THRESHOLD && obj->refcount == 1) {
         atomicIncr(lazyfree_objects,1);
-        bioCreateBackgroundJob(BIO_LAZY_FREE,o,NULL,NULL);
+        bioCreateBackgroundJob(BIO_LAZY_FREE,obj,NULL,NULL);
     } else {
-        decrRefCount(o);
+        decrRefCount(obj);
     }
 }
 
@@ -152,6 +175,7 @@ void slotToKeyFlushAsync(void) {
 void lazyfreeFreeObjectFromBioThread(robj *o) {
     decrRefCount(o);
     atomicDecr(lazyfree_objects,1);
+    atomicIncr(lazyfreed_objects,1);
 }
 
 /* Release a database from the lazyfree thread. The 'db' pointer is the
@@ -164,6 +188,7 @@ void lazyfreeFreeDatabaseFromBioThread(dict *ht1, dict *ht2) {
     dictRelease(ht1);
     dictRelease(ht2);
     atomicDecr(lazyfree_objects,numkeys);
+    atomicIncr(lazyfreed_objects,numkeys);
 }
 
 /* Release the skiplist mapping Redis Cluster keys to slots in the
@@ -172,4 +197,5 @@ void lazyfreeFreeSlotsMapFromBioThread(rax *rt) {
     size_t len = rt->numele;
     raxFree(rt);
     atomicDecr(lazyfree_objects,len);
+    atomicIncr(lazyfreed_objects,len);
 }
diff --git a/src/module.c b/src/module.c
index 691a9d298..03fde12a0 100644
--- a/src/module.c
+++ b/src/module.c
@@ -3666,6 +3666,8 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) {
  *          .mem_usage = myType_MemUsageCallBack,
  *          .aux_load = myType_AuxRDBLoadCallBack,
  *          .aux_save = myType_AuxRDBSaveCallBack,
+ *          .free_effort = myType_FreeEffortCallBack
+ *          .unlink = myType_UnlinkCallBack
  *      }
  *
  * * **rdb_load**: A callback function pointer that loads data from RDB files.
@@ -3677,7 +3679,15 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) {
  *   'when' argument is either REDISMODULE_AUX_BEFORE_RDB or REDISMODULE_AUX_AFTER_RDB.
  * * **aux_load**: A callback function pointer that loads out of keyspace data from RDB files.
  *   Similar to aux_save, returns REDISMODULE_OK on success, and ERR otherwise.
- *
+ * * **free_effort**: A callback function pointer that used to determine whether the module's
+ *   memory needs to be lazy reclaimed. The module should return the complexity involved by
+ *   freeing the value. for example: how many pointers are gonna be freed. Note that if it 
+ *   returns 0, we'll always do an async free.
+ * * **unlink**: A callback function pointer that used to notifies the module that the key has 
+ *   been removed from the DB by redis, and may soon be freed by a background thread. Note that 
+ *   it won't be called on FLUSHALL/FLUSHDB (both sync and async), and the module can use the 
+ *   RedisModuleEvent_FlushDB to hook into that.
+ * 
  * The **digest** and **mem_usage** methods should currently be omitted since
  * they are not yet implemented inside the Redis modules core.
  *
@@ -3720,6 +3730,10 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver,
             moduleTypeAuxSaveFunc aux_save;
             int aux_save_triggers;
         } v2;
+        struct {
+            moduleTypeFreeEffortFunc free_effort;
+            moduleTypeUnlinkFunc unlink;
+        } v3;
     } *tms = (struct typemethods*) typemethods_ptr;
 
     moduleType *mt = zcalloc(sizeof(*mt));
@@ -3736,6 +3750,10 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver,
         mt->aux_save = tms->v2.aux_save;
         mt->aux_save_triggers = tms->v2.aux_save_triggers;
     }
+    if (tms->version >= 3) {
+        mt->free_effort = tms->v3.free_effort;
+        mt->unlink = tms->v3.unlink;
+    }
     memcpy(mt->name,name,sizeof(mt->name));
     listAddNodeTail(ctx->module->types,mt);
     return mt;
@@ -7489,6 +7507,18 @@ void processModuleLoadingProgressEvent(int is_aof) {
     }
 }
 
+/* When a module key is deleted (in dbAsyncDelete/dbSyncDelete/dbOverwrite), it 
+*  will be called to tell the module which key is about to be released. */
+void moduleNotifyKeyUnlink(robj *key, robj *val) {
+    if (val->type == OBJ_MODULE) {
+        moduleValue *mv = val->ptr;
+        moduleType *mt = mv->type;
+        if (mt->unlink != NULL) {
+            mt->unlink(key,mv->value);
+        } 
+    }
+}
+
 /* --------------------------------------------------------------------------
  * Modules API internals
  * -------------------------------------------------------------------------- */
@@ -7990,6 +8020,15 @@ int RM_GetServerVersion() {
     return REDIS_VERSION_NUM;
 }
 
+/**
+ * Return the current redis-server runtime value of REDISMODULE_TYPE_METHOD_VERSION.
+ * You can use that when calling RM_CreateDataType to know which fields of
+ * RedisModuleTypeMethods are gonna be supported and which will be ignored.
+ */
+int RM_GetTypeMethodVersion() {
+    return REDISMODULE_TYPE_METHOD_VERSION;
+}
+
 /* Replace the value assigned to a module type.
  *
  * The key must be open for writing, have an existing value, and have a moduleType
@@ -8329,4 +8368,5 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(GetServerVersion);
     REGISTER_API(GetClientCertificate);
     REGISTER_API(GetCommandKeys);
+    REGISTER_API(GetTypeMethodVersion);
 }
diff --git a/src/redismodule.h b/src/redismodule.h
index ca0a756c7..0f7134549 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -14,6 +14,10 @@
 /* API versions. */
 #define REDISMODULE_APIVER_1 1
 
+/* Version of the RedisModuleTypeMethods structure. Once the RedisModuleTypeMethods 
+ * structure is changed, this version number needs to be changed synchronistically. */
+#define REDISMODULE_TYPE_METHOD_VERSION 3
+
 /* API flags and constants */
 #define REDISMODULE_READ (1<<0)
 #define REDISMODULE_WRITE (1<<1)
@@ -485,6 +489,8 @@ typedef void (*RedisModuleTypeRewriteFunc)(RedisModuleIO *aof, RedisModuleString
 typedef size_t (*RedisModuleTypeMemUsageFunc)(const void *value);
 typedef void (*RedisModuleTypeDigestFunc)(RedisModuleDigest *digest, void *value);
 typedef void (*RedisModuleTypeFreeFunc)(void *value);
+typedef size_t (*RedisModuleTypeFreeEffortFunc)(RedisModuleString *key, const void *value);
+typedef void (*RedisModuleTypeUnlinkFunc)(RedisModuleString *key, const void *value);
 typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len);
 typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data);
 typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter);
@@ -494,7 +500,6 @@ typedef void (*RedisModuleScanCB)(RedisModuleCtx *ctx, RedisModuleString *keynam
 typedef void (*RedisModuleScanKeyCB)(RedisModuleKey *key, RedisModuleString *field, RedisModuleString *value, void *privdata);
 typedef void (*RedisModuleUserChangedFunc) (uint64_t client_id, void *privdata);
 
-#define REDISMODULE_TYPE_METHOD_VERSION 2
 typedef struct RedisModuleTypeMethods {
     uint64_t version;
     RedisModuleTypeLoadFunc rdb_load;
@@ -506,6 +511,8 @@ typedef struct RedisModuleTypeMethods {
     RedisModuleTypeAuxLoadFunc aux_load;
     RedisModuleTypeAuxSaveFunc aux_save;
     int aux_save_triggers;
+    RedisModuleTypeFreeEffortFunc free_effort;
+    RedisModuleTypeUnlinkFunc unlink;
 } RedisModuleTypeMethods;
 
 #define REDISMODULE_GET_API(name) \
@@ -706,6 +713,7 @@ REDISMODULE_API int (*RedisModule_GetContextFlagsAll)() REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_GetKeyspaceNotificationFlagsAll)() REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_IsSubEventSupported)(RedisModuleEvent event, uint64_t subevent) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_GetServerVersion)() REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_GetTypeMethodVersion)() REDISMODULE_ATTR;
 
 /* Experimental APIs */
 #ifdef REDISMODULE_EXPERIMENTAL_API
@@ -956,6 +964,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(GetKeyspaceNotificationFlagsAll);
     REDISMODULE_GET_API(IsSubEventSupported);
     REDISMODULE_GET_API(GetServerVersion);
+    REDISMODULE_GET_API(GetTypeMethodVersion);
 
 #ifdef REDISMODULE_EXPERIMENTAL_API
     REDISMODULE_GET_API(GetThreadSafeContext);
diff --git a/src/server.c b/src/server.c
index 04b6d98ba..685910f19 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4404,7 +4404,8 @@ sds genRedisInfoString(const char *section) {
             "mem_aof_buffer:%zu\r\n"
             "mem_allocator:%s\r\n"
             "active_defrag_running:%d\r\n"
-            "lazyfree_pending_objects:%zu\r\n",
+            "lazyfree_pending_objects:%zu\r\n"
+            "lazyfreed_objects:%zu\r\n",
             zmalloc_used,
             hmem,
             server.cron_malloc_stats.process_rss,
@@ -4447,7 +4448,8 @@ sds genRedisInfoString(const char *section) {
             mh->aof_buffer,
             ZMALLOC_LIB,
             server.active_defrag_running,
-            lazyfreeGetPendingObjectsCount()
+            lazyfreeGetPendingObjectsCount(),
+            lazyfreeGetFreedObjectsCount()
         );
         freeMemoryOverheadData(mh);
     }
diff --git a/src/server.h b/src/server.h
index c6d5f987a..c7ddc9e84 100644
--- a/src/server.h
+++ b/src/server.h
@@ -513,6 +513,8 @@ typedef void (*moduleTypeRewriteFunc)(struct RedisModuleIO *io, struct redisObje
 typedef void (*moduleTypeDigestFunc)(struct RedisModuleDigest *digest, void *value);
 typedef size_t (*moduleTypeMemUsageFunc)(const void *value);
 typedef void (*moduleTypeFreeFunc)(void *value);
+typedef size_t (*moduleTypeFreeEffortFunc)(struct redisObject *key, const void *value);
+typedef void (*moduleTypeUnlinkFunc)(struct redisObject *key, void *value);
 
 /* This callback type is called by moduleNotifyUserChanged() every time
  * a user authenticated via the module API is associated with a different
@@ -532,6 +534,8 @@ typedef struct RedisModuleType {
     moduleTypeMemUsageFunc mem_usage;
     moduleTypeDigestFunc digest;
     moduleTypeFreeFunc free;
+    moduleTypeFreeEffortFunc free_effort;
+    moduleTypeUnlinkFunc unlink;
     moduleTypeAuxLoadFunc aux_load;
     moduleTypeAuxSaveFunc aux_save;
     int aux_save_triggers;
@@ -1651,6 +1655,7 @@ int moduleTryServeClientBlockedOnKey(client *c, robj *key);
 void moduleUnblockClient(client *c);
 int moduleClientIsBlockedOnKeys(client *c);
 void moduleNotifyUserChanged(client *c);
+void moduleNotifyKeyUnlink(robj *key, robj *val);
 
 /* Utils */
 long long ustime(void);
@@ -2201,7 +2206,8 @@ int dbAsyncDelete(redisDb *db, robj *key);
 void emptyDbAsync(redisDb *db);
 void slotToKeyFlushAsync(void);
 size_t lazyfreeGetPendingObjectsCount(void);
-void freeObjAsync(robj *o);
+size_t lazyfreeGetFreedObjectsCount(void);
+void freeObjAsync(robj *key, robj *obj);
 
 /* API to get key arguments from commands */
 int *getKeysPrepareResult(getKeysResult *result, int numkeys);
diff --git a/src/t_set.c b/src/t_set.c
index c1e7fddef..2cca15ae2 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -1080,7 +1080,7 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
             sdsfree(ele);
         }
         setTypeReleaseIterator(si);
-        server.lazyfree_lazy_server_del ? freeObjAsync(dstset) :
+        server.lazyfree_lazy_server_del ? freeObjAsync(NULL, dstset) :
                                           decrRefCount(dstset);
     } else {
         /* If we have a target key where to store the resulting set
diff --git a/tests/modules/Makefile b/tests/modules/Makefile
index 8df05ed01..36222fc1f 100644
--- a/tests/modules/Makefile
+++ b/tests/modules/Makefile
@@ -26,7 +26,8 @@ TEST_MODULES = \
     keyspace_events.so \
     blockedclient.so \
     getkeys.so \
-    timer.so
+    test_lazyfree.so \
+    timer.so \
 
 
 .PHONY: all
diff --git a/tests/modules/test_lazyfree.c b/tests/modules/test_lazyfree.c
new file mode 100644
index 000000000..144dab9b3
--- /dev/null
+++ b/tests/modules/test_lazyfree.c
@@ -0,0 +1,196 @@
+/* This module emulates a linked list for lazyfree testing of modules, which
+ is a simplified version of 'hellotype.c'
+ */
+#include "redismodule.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+#include <stdint.h>
+
+static RedisModuleType *LazyFreeLinkType;
+
+struct LazyFreeLinkNode {
+    int64_t value;
+    struct LazyFreeLinkNode *next;
+};
+
+struct LazyFreeLinkObject {
+    struct LazyFreeLinkNode *head;
+    size_t len; /* Number of elements added. */
+};
+
+struct LazyFreeLinkObject *createLazyFreeLinkObject(void) {
+    struct LazyFreeLinkObject *o;
+    o = RedisModule_Alloc(sizeof(*o));
+    o->head = NULL;
+    o->len = 0;
+    return o;
+}
+
+void LazyFreeLinkInsert(struct LazyFreeLinkObject *o, int64_t ele) {
+    struct LazyFreeLinkNode *next = o->head, *newnode, *prev = NULL;
+
+    while(next && next->value < ele) {
+        prev = next;
+        next = next->next;
+    }
+    newnode = RedisModule_Alloc(sizeof(*newnode));
+    newnode->value = ele;
+    newnode->next = next;
+    if (prev) {
+        prev->next = newnode;
+    } else {
+        o->head = newnode;
+    }
+    o->len++;
+}
+
+void LazyFreeLinkReleaseObject(struct LazyFreeLinkObject *o) {
+    struct LazyFreeLinkNode *cur, *next;
+    cur = o->head;
+    while(cur) {
+        next = cur->next;
+        RedisModule_Free(cur);
+        cur = next;
+    }
+    RedisModule_Free(o);
+}
+
+/* LAZYFREELINK.INSERT key value */
+int LazyFreeLinkInsert_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    RedisModule_AutoMemory(ctx); /* Use automatic memory management. */
+
+    if (argc != 3) return RedisModule_WrongArity(ctx);
+    RedisModuleKey *key = RedisModule_OpenKey(ctx,argv[1],
+        REDISMODULE_READ|REDISMODULE_WRITE);
+    int type = RedisModule_KeyType(key);
+    if (type != REDISMODULE_KEYTYPE_EMPTY &&
+        RedisModule_ModuleTypeGetType(key) != LazyFreeLinkType)
+    {
+        return RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE);
+    }
+
+    long long value;
+    if ((RedisModule_StringToLongLong(argv[2],&value) != REDISMODULE_OK)) {
+        return RedisModule_ReplyWithError(ctx,"ERR invalid value: must be a signed 64 bit integer");
+    }
+
+    struct LazyFreeLinkObject *hto;
+    if (type == REDISMODULE_KEYTYPE_EMPTY) {
+        hto = createLazyFreeLinkObject();
+        RedisModule_ModuleTypeSetValue(key,LazyFreeLinkType,hto);
+    } else {
+        hto = RedisModule_ModuleTypeGetValue(key);
+    }
+
+    LazyFreeLinkInsert(hto,value);
+    RedisModule_SignalKeyAsReady(ctx,argv[1]);
+
+    RedisModule_ReplyWithLongLong(ctx,hto->len);
+    RedisModule_ReplicateVerbatim(ctx);
+    return REDISMODULE_OK;
+}
+
+/* LAZYFREELINK.LEN key */
+int LazyFreeLinkLen_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    RedisModule_AutoMemory(ctx); /* Use automatic memory management. */
+
+    if (argc != 2) return RedisModule_WrongArity(ctx);
+    RedisModuleKey *key = RedisModule_OpenKey(ctx,argv[1],
+        REDISMODULE_READ|REDISMODULE_WRITE);
+    int type = RedisModule_KeyType(key);
+    if (type != REDISMODULE_KEYTYPE_EMPTY &&
+        RedisModule_ModuleTypeGetType(key) != LazyFreeLinkType)
+    {
+        return RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE);
+    }
+
+    struct LazyFreeLinkObject *hto = RedisModule_ModuleTypeGetValue(key);
+    RedisModule_ReplyWithLongLong(ctx,hto ? hto->len : 0);
+    return REDISMODULE_OK;
+}
+
+void *LazyFreeLinkRdbLoad(RedisModuleIO *rdb, int encver) {
+    if (encver != 0) {
+        return NULL;
+    }
+    uint64_t elements = RedisModule_LoadUnsigned(rdb);
+    struct LazyFreeLinkObject *hto = createLazyFreeLinkObject();
+    while(elements--) {
+        int64_t ele = RedisModule_LoadSigned(rdb);
+        LazyFreeLinkInsert(hto,ele);
+    }
+    return hto;
+}
+
+void LazyFreeLinkRdbSave(RedisModuleIO *rdb, void *value) {
+    struct LazyFreeLinkObject *hto = value;
+    struct LazyFreeLinkNode *node = hto->head;
+    RedisModule_SaveUnsigned(rdb,hto->len);
+    while(node) {
+        RedisModule_SaveSigned(rdb,node->value);
+        node = node->next;
+    }
+}
+
+void LazyFreeLinkAofRewrite(RedisModuleIO *aof, RedisModuleString *key, void *value) {
+    struct LazyFreeLinkObject *hto = value;
+    struct LazyFreeLinkNode *node = hto->head;
+    while(node) {
+        RedisModule_EmitAOF(aof,"LAZYFREELINK.INSERT","sl",key,node->value);
+        node = node->next;
+    }
+}
+
+void LazyFreeLinkFree(void *value) {
+    LazyFreeLinkReleaseObject(value);
+}
+
+size_t LazyFreeLinkFreeEffort(RedisModuleString *key, const void *value) {
+    REDISMODULE_NOT_USED(key);
+    const struct LazyFreeLinkObject *hto = value;
+    return hto->len;
+}
+
+void LazyFreeLinkUnlink(RedisModuleString *key, const void *value) {
+    REDISMODULE_NOT_USED(key);
+    REDISMODULE_NOT_USED(value);
+    /* Here you can know which key and value is about to be freed. */
+}
+
+int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+
+    if (RedisModule_Init(ctx,"lazyfreetest",1,REDISMODULE_APIVER_1)
+        == REDISMODULE_ERR) return REDISMODULE_ERR;
+
+    /* We only allow our module to be loaded when the redis core version is greater than the version of my module */
+    if (RedisModule_GetTypeMethodVersion() < REDISMODULE_TYPE_METHOD_VERSION) {
+        return REDISMODULE_ERR;
+    }
+
+    RedisModuleTypeMethods tm = {
+        .version = REDISMODULE_TYPE_METHOD_VERSION,
+        .rdb_load = LazyFreeLinkRdbLoad,
+        .rdb_save = LazyFreeLinkRdbSave,
+        .aof_rewrite = LazyFreeLinkAofRewrite,
+        .free = LazyFreeLinkFree,
+        .free_effort = LazyFreeLinkFreeEffort,
+        .unlink = LazyFreeLinkUnlink,
+    };
+
+    LazyFreeLinkType = RedisModule_CreateDataType(ctx,"test_lazy",0,&tm);
+    if (LazyFreeLinkType == NULL) return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx,"lazyfreelink.insert",
+        LazyFreeLinkInsert_RedisCommand,"write deny-oom",1,1,1) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx,"lazyfreelink.len",
+        LazyFreeLinkLen_RedisCommand,"readonly",1,1,1) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    return REDISMODULE_OK;
+}
diff --git a/tests/unit/moduleapi/test_lazyfree.tcl b/tests/unit/moduleapi/test_lazyfree.tcl
new file mode 100644
index 000000000..9f482769c
--- /dev/null
+++ b/tests/unit/moduleapi/test_lazyfree.tcl
@@ -0,0 +1,32 @@
+set testmodule [file normalize tests/modules/test_lazyfree.so]
+
+start_server {tags {"modules"}} {
+    r module load $testmodule
+
+    test "modules allocated memory can be reclaimed in the background" {
+        set orig_mem [s used_memory]
+        set rd [redis_deferring_client]
+
+        # LAZYFREE_THRESHOLD is 64
+        for {set i 0} {$i < 10000} {incr i} {
+            $rd lazyfreelink.insert lazykey $i
+        }
+
+        for {set j 0} {$j < 10000} {incr j} {
+            $rd read 
+        }
+
+        assert {[r lazyfreelink.len lazykey] == 10000}
+
+        set peak_mem [s used_memory]
+        assert {[r unlink lazykey] == 1}
+        assert {$peak_mem > $orig_mem+10000}
+        wait_for_condition 50 100 {
+            [s used_memory] < $peak_mem &&
+            [s used_memory] < $orig_mem*2 &&
+            [string match {*lazyfreed_objects:1*} [r info Memory]]
+        } else {
+            fail "Module memory is not reclaimed by UNLINK"
+        }
+    }
+}
\ No newline at end of file

From 9812e88959a37ed4aa95bb4d3fb55c56dc49a768 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 16 Nov 2020 16:37:15 +0200
Subject: [PATCH 0804/1098] Fix memory leaks in newly added ZDIFF (#8056)

---
 src/t_zset.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/t_zset.c b/src/t_zset.c
index 858085a05..5b848a1b9 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -2267,6 +2267,7 @@ static void zdiffAlgorithm1(zsetopsrc *src, long setnum, zset *dstzset, size_t *
             if (sdslen(tmp) > *maxelelen) *maxelelen = sdslen(tmp);
         }
     }
+    zuiClearIterator(&src[0]);
 }
 
 
@@ -2304,7 +2305,7 @@ static void zdiffAlgorithm2(zsetopsrc *src, long setnum, zset *dstzset, size_t *
                 dictAdd(dstzset->dict,tmp,&znode->score);
                 cardinality++;
             } else {
-                tmp = zuiNewSdsFromValue(&zval);
+                tmp = zuiSdsFromValue(&zval);
                 if (zsetRemoveFromSkiplist(dstzset, tmp)) {
                     cardinality--;
                 }
@@ -2314,6 +2315,7 @@ static void zdiffAlgorithm2(zsetopsrc *src, long setnum, zset *dstzset, size_t *
                 * of elements will have no effect. */
             if (cardinality == 0) break;
         }
+        zuiClearIterator(&src[j]);
 
         if (cardinality == 0) break;
     }

From ea7cf737a1ff65524994ce6731eb9320891aa946 Mon Sep 17 00:00:00 2001
From: swamp0407 <swamp0407@gmail.com>
Date: Tue, 17 Nov 2020 19:03:05 +0900
Subject: [PATCH 0805/1098] Add COPY command (#7953)

Syntax:
COPY <key> <new-key> [DB <dest-db>] [REPLACE]

No support for module keys yet.

Co-authored-by: tmgauss
Co-authored-by: Itamar Haber <itamar@redislabs.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/db.c                | 102 ++++++++++++++++++++
 src/server.c            |   4 +
 src/server.h            |   5 +
 src/stream.h            |   1 +
 src/t_hash.c            |  54 +++++++++++
 src/t_list.c            |  23 +++++
 src/t_set.c             |  46 +++++++++
 src/t_stream.c          | 104 ++++++++++++++++++++
 src/t_zset.c            |  62 ++++++++++++
 tests/unit/keyspace.tcl | 207 ++++++++++++++++++++++++++++++++++++++++
 10 files changed, 608 insertions(+)

diff --git a/src/db.c b/src/db.c
index f9570dc75..b8abb6877 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1079,6 +1079,108 @@ void moveCommand(client *c) {
     addReply(c,shared.cone);
 }
 
+void copyCommand(client *c) {
+    robj *o;
+    redisDb *src, *dst;
+    int srcid;
+    long long dbid, expire;
+    int j, replace = 0, delete = 0;
+
+    /* Obtain source and target DB pointers 
+     * Default target DB is the same as the source DB 
+     * Parse the REPLACE option and targetDB option. */
+    src = c->db;
+    dst = c->db;
+    srcid = c->db->id;
+    dbid = c->db->id;
+    for (j = 3; j < c->argc; j++) {
+        int additional = c->argc - j - 1;
+        if (!strcasecmp(c->argv[j]->ptr,"replace")) {
+            replace = 1;
+        } else if (!strcasecmp(c->argv[j]->ptr, "db") && additional >= 1) {
+            if (getLongLongFromObject(c->argv[j+1], &dbid) == C_ERR ||
+                dbid < INT_MIN || dbid > INT_MAX ||
+                selectDb(c, dbid) == C_ERR)
+            {
+                addReplyError(c,"invalid DB index");
+                return;
+            }
+            dst = c->db;
+            selectDb(c,srcid); /* Back to the source DB */
+            j++; /* Consume additional arg. */
+        } else {
+            addReply(c, shared.syntaxerr);
+            return;
+        }
+    }
+
+    if ((server.cluster_enabled == 1) && (srcid != 0 || dbid != 0)) {
+        addReplyError(c,"Copying to another database is not allowed in cluster mode");
+        return;
+    }
+
+    /* If the user select the same DB as
+     * the source DB and using newkey as the same key
+     * it is probably an error. */
+    robj *key = c->argv[1];
+    robj *newkey = c->argv[2];
+    if (src == dst && (sdscmp(key->ptr, newkey->ptr) == 0)) {
+        addReply(c,shared.sameobjecterr);
+        return;
+    }
+
+    /* Check if the element exists and get a reference */
+    o = lookupKeyWrite(c->db, key);
+    if (!o) {
+        addReply(c,shared.czero);
+        return;
+    }
+    expire = getExpire(c->db,key);
+
+    /* Return zero if the key already exists in the target DB. 
+     * If REPLACE option is selected, delete newkey from targetDB. */
+    if (lookupKeyWrite(dst,newkey) != NULL) {
+        if (replace) {
+            delete = 1;
+        } else {
+            addReply(c,shared.czero);
+            return;
+        }
+    }
+
+    /* Duplicate object according to object's type. */
+    robj *newobj;
+    switch(o->type) {
+        case OBJ_STRING: newobj = dupStringObject(o); break;
+        case OBJ_LIST: newobj = listTypeDup(o); break;
+        case OBJ_SET: newobj = setTypeDup(o); break;
+        case OBJ_ZSET: newobj = zsetDup(o); break;
+        case OBJ_HASH: newobj = hashTypeDup(o); break;
+        case OBJ_STREAM: newobj = streamDup(o); break;
+        case OBJ_MODULE:
+            addReplyError(c, "Copying module type object is not supported");
+            return;
+        default: {
+            addReplyError(c, "unknown type object");
+            return;
+        };
+    }
+
+    if (delete) {
+        dbDelete(dst,newkey);
+    }
+
+    dbAdd(dst,newkey,newobj);
+    if (expire != -1) setExpire(c, dst, newkey, expire);
+
+    /* OK! key copied */
+    signalModifiedKey(c,dst,c->argv[2]);
+    notifyKeyspaceEvent(NOTIFY_GENERIC,"copy_to",c->argv[2],dst->id);
+
+    server.dirty++;
+    addReply(c,shared.cone);
+}
+
 /* Helper function for dbSwapDatabases(): scans the list of keys that have
  * one or more blocked clients for B[LR]POP or other blocking commands
  * and signal the keys as ready if they are of the right type. See the comment
diff --git a/src/server.c b/src/server.c
index 685910f19..cef9f7185 100644
--- a/src/server.c
+++ b/src/server.c
@@ -630,6 +630,10 @@ struct redisCommand redisCommandTable[] = {
      "write fast @keyspace",
      0,NULL,1,1,1,0,0,0},
 
+    {"copy",copyCommand,-3,
+     "write use-memory @keyspace",
+     0,NULL,1,2,1,0,0,0},
+
     /* Like for SET, we can't mark rename as a fast command because
      * overwriting the target key may result in an implicit slow DEL. */
     {"rename",renameCommand,3,
diff --git a/src/server.h b/src/server.h
index c7ddc9e84..66ee066a1 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1796,6 +1796,7 @@ void listTypeInsert(listTypeEntry *entry, robj *value, int where);
 int listTypeEqual(listTypeEntry *entry, robj *o);
 void listTypeDelete(listTypeIterator *iter, listTypeEntry *entry);
 void listTypeConvert(robj *subject, int enc);
+robj *listTypeDup(robj *o);
 void unblockClientWaitingData(client *c);
 void popGenericCommand(client *c, int where);
 void listElementsRemoved(client *c, robj *key, int where, robj *o);
@@ -2027,6 +2028,7 @@ unsigned long zslGetRank(zskiplist *zsl, double score, sds o);
 int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore);
 long zsetRank(robj *zobj, sds ele, int reverse);
 int zsetDel(robj *zobj, sds ele);
+robj *zsetDup(robj *o);
 void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey, robj *countarg);
 sds ziplistGetObject(unsigned char *sptr);
 int zslValueGteMin(double value, zrangespec *spec);
@@ -2105,6 +2107,7 @@ int setTypeRandomElement(robj *setobj, sds *sdsele, int64_t *llele);
 unsigned long setTypeRandomElements(robj *set, unsigned long count, robj *aux_set);
 unsigned long setTypeSize(const robj *subject);
 void setTypeConvert(robj *subject, int enc);
+robj *setTypeDup(robj *o);
 
 /* Hash data type */
 #define HASH_SET_TAKE_FIELD (1<<0)
@@ -2129,6 +2132,7 @@ sds hashTypeCurrentObjectNewSds(hashTypeIterator *hi, int what);
 robj *hashTypeLookupWriteOrCreate(client *c, robj *key);
 robj *hashTypeGetValueObject(robj *o, sds field);
 int hashTypeSet(robj *o, sds field, sds value, int flags);
+robj *hashTypeDup(robj *o);
 
 /* Pub / Sub */
 int pubsubUnsubscribeAllChannels(client *c, int notify);
@@ -2336,6 +2340,7 @@ void bgsaveCommand(client *c);
 void bgrewriteaofCommand(client *c);
 void shutdownCommand(client *c);
 void moveCommand(client *c);
+void copyCommand(client *c);
 void renameCommand(client *c);
 void renamenxCommand(client *c);
 void lpushCommand(client *c);
diff --git a/src/stream.h b/src/stream.h
index 3d692cd9d..7e1a24417 100644
--- a/src/stream.h
+++ b/src/stream.h
@@ -118,5 +118,6 @@ int streamCompareID(streamID *a, streamID *b);
 void streamFreeNACK(streamNACK *na);
 void streamIncrID(streamID *id);
 void streamPropagateConsumerCreation(client *c, robj *key, robj *groupname, sds consumername);
+robj *streamDup(robj *o);
 
 #endif
diff --git a/src/t_hash.c b/src/t_hash.c
index 89193a41e..5c844ce30 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -504,6 +504,60 @@ void hashTypeConvert(robj *o, int enc) {
     }
 }
 
+/* This is a helper function for the COPY command.
+ * Duplicate a hash object, with the guarantee that the returned object
+ * has the same encoding as the original one.
+ *
+ * The resulting object always has refcount set to 1 */
+robj *hashTypeDup(robj *o) {
+    robj *hobj;
+    hashTypeIterator *hi;
+
+    serverAssert(o->type == OBJ_HASH);
+
+    switch (o->encoding) {
+        case OBJ_ENCODING_ZIPLIST:
+            hobj = createHashObject();
+            break;
+        case OBJ_ENCODING_HT:
+            hobj = createHashObject();
+            hashTypeConvert(hobj, OBJ_ENCODING_HT);
+            dict *d = o->ptr;
+            dictExpand(hobj->ptr, dictSize(d));
+            break;
+        default:
+            serverPanic("Wrong encoding.");
+            break;
+    }
+
+    if(o->encoding == OBJ_ENCODING_ZIPLIST){
+        unsigned char *zl = o->ptr;
+        size_t sz = ziplistBlobLen(zl);
+        unsigned char *new_zl = zmalloc(sz);
+        memcpy(new_zl, zl, sz);
+        zfree(hobj->ptr);
+        hobj->ptr = new_zl;
+    } else if(o->encoding == OBJ_ENCODING_HT){
+        hi = hashTypeInitIterator(o);
+        while (hashTypeNext(hi) != C_ERR) {
+            sds field, value;
+            sds newfield, newvalue;
+            /* Extract a field-value pair from an original hash object.*/
+            field = hashTypeCurrentFromHashTable(hi, OBJ_HASH_KEY);
+            value = hashTypeCurrentFromHashTable(hi, OBJ_HASH_VALUE);
+            newfield = sdsdup(field);
+            newvalue = sdsdup(value);
+
+            /* Add a field-value pair to a new hash object. */
+            dictAdd(hobj->ptr,newfield,newvalue);
+        }
+        hashTypeReleaseIterator(hi);
+    } else {
+        serverPanic("Unknown hash encoding");
+    }
+    return hobj;
+}
+
 /*-----------------------------------------------------------------------------
  * Hash type commands
  *----------------------------------------------------------------------------*/
diff --git a/src/t_list.c b/src/t_list.c
index 25dda3178..cdfc0ff51 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -190,6 +190,29 @@ void listTypeConvert(robj *subject, int enc) {
     }
 }
 
+/* This is a helper function for the COPY command.
+ * Duplicate a list object, with the guarantee that the returned object
+ * has the same encoding as the original one.
+ *
+ * The resulting object always has refcount set to 1 */
+robj *listTypeDup(robj *o) {
+    robj *lobj;
+
+    serverAssert(o->type == OBJ_LIST);
+
+    switch (o->encoding) {
+        case OBJ_ENCODING_QUICKLIST:
+            lobj = createQuicklistObject();
+            break;
+        default:
+            serverPanic("Wrong encoding.");
+            break;
+    }
+    zfree(lobj->ptr);
+    lobj->ptr = quicklistDup(o->ptr);
+    return lobj;
+}
+
 /*-----------------------------------------------------------------------------
  * List Commands
  *----------------------------------------------------------------------------*/
diff --git a/src/t_set.c b/src/t_set.c
index 2cca15ae2..cb4c5002f 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -261,6 +261,52 @@ void setTypeConvert(robj *setobj, int enc) {
     }
 }
 
+/* This is a helper function for the COPY command.
+ * Duplicate a set object, with the guarantee that the returned object
+ * has the same encoding as the original one.
+ *
+ * The resulting object always has refcount set to 1 */
+robj *setTypeDup(robj *o) {
+    robj *set;
+    setTypeIterator *si;
+    sds elesds;
+    int64_t intobj;
+
+    serverAssert(o->type == OBJ_SET);
+
+    /* Create a new set object that have the same encoding as the original object's encoding */
+    switch (o->encoding) {
+        case OBJ_ENCODING_INTSET:
+            set = createIntsetObject();
+            break;
+        case OBJ_ENCODING_HT:
+            set = createSetObject();
+            dict *d = o->ptr;
+            dictExpand(set->ptr, dictSize(d));
+            break;
+        default:
+            serverPanic("Wrong encoding.");
+            break;
+    }
+    if (set->encoding == OBJ_ENCODING_INTSET) {
+        intset *is = o->ptr;
+        size_t size = intsetBlobLen(is);
+        intset *newis = zmalloc(size);
+        memcpy(newis,is,size);
+        zfree(set->ptr);
+        set->ptr = newis;
+    } else if (set->encoding == OBJ_ENCODING_HT) {
+        si = setTypeInitIterator(o);
+        while (setTypeNext(si, &elesds, &intobj) != -1) {
+            setTypeAdd(set, elesds);
+        }
+        setTypeReleaseIterator(si);
+    } else {
+        serverPanic("Unknown set encoding");
+    }
+    return set;
+}
+
 void saddCommand(client *c) {
     robj *set;
     int j, added = 0;
diff --git a/src/t_stream.c b/src/t_stream.c
index d6f6a3011..02e4c3242 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -106,6 +106,110 @@ void streamNextID(streamID *last_id, streamID *new_id) {
     }
 }
 
+/* This is a helper function for the COPY command.
+ * Duplicate a Stream object, with the guarantee that the returned object
+ * has the same encoding as the original one.
+ *
+ * The resulting object always has refcount set to 1 */
+robj *streamDup(robj *o) {
+    robj *sobj;
+
+    serverAssert(o->type == OBJ_STREAM);
+
+    switch (o->encoding) {
+        case OBJ_ENCODING_STREAM:
+            sobj = createStreamObject();
+            break;
+        default:
+            serverPanic("Wrong encoding.");
+            break;
+    }
+
+    stream *s;
+    stream *new_s;
+    s = o->ptr;
+    new_s = sobj->ptr;
+
+    raxIterator ri;
+    uint64_t rax_key[2];
+    raxStart(&ri, s->rax);
+    raxSeek(&ri, "^", NULL, 0);
+    size_t lp_bytes = 0;      /* Total bytes in the listpack. */
+    unsigned char *lp = NULL; /* listpack pointer. */
+    /* Get a reference to the listpack node. */
+    while (raxNext(&ri)) {
+        lp = ri.data;
+        lp_bytes = lpBytes(lp);
+        unsigned char *new_lp = zmalloc(lp_bytes);
+        memcpy(new_lp, lp, lp_bytes);
+        memcpy(rax_key, ri.key, sizeof(rax_key));
+        raxInsert(new_s->rax, (unsigned char *)&rax_key, sizeof(rax_key),
+                  new_lp, NULL);
+    }
+    new_s->length = s->length;
+    new_s->last_id = s->last_id;
+    raxStop(&ri);
+
+    if (s->cgroups == NULL) return sobj;
+
+    /* Consumer Groups */
+    raxIterator ri_cgroups;
+    raxStart(&ri_cgroups, s->cgroups);
+    raxSeek(&ri_cgroups, "^", NULL, 0);
+    while (raxNext(&ri_cgroups)) {
+        streamCG *cg = ri_cgroups.data;
+        streamCG *new_cg = streamCreateCG(new_s, (char *)ri_cgroups.key,
+                                          ri_cgroups.key_len, &cg->last_id);
+
+        serverAssert(new_cg != NULL);
+
+        /* Consumer Group PEL */
+        raxIterator ri_cg_pel;
+        raxStart(&ri_cg_pel,cg->pel);
+        raxSeek(&ri_cg_pel,"^",NULL,0);
+        while(raxNext(&ri_cg_pel)){
+            streamNACK *nack = ri_cg_pel.data;
+            streamNACK *new_nack = streamCreateNACK(NULL);
+            new_nack->delivery_time = nack->delivery_time;
+            new_nack->delivery_count = nack->delivery_count;
+            raxInsert(new_cg->pel, ri_cg_pel.key, sizeof(streamID), new_nack, NULL);
+        }
+        raxStop(&ri_cg_pel);
+
+        /* Consumers */
+        raxIterator ri_consumers;
+        raxStart(&ri_consumers, cg->consumers);
+        raxSeek(&ri_consumers, "^", NULL, 0);
+        while (raxNext(&ri_consumers)) {
+            streamConsumer *consumer = ri_consumers.data;
+            streamConsumer *new_consumer;
+            new_consumer = zmalloc(sizeof(*new_consumer));
+            new_consumer->name = sdsdup(consumer->name);
+            new_consumer->pel = raxNew();
+            raxInsert(new_cg->consumers,(unsigned char *)new_consumer->name,
+                        sdslen(new_consumer->name), new_consumer, NULL);
+            new_consumer->seen_time = consumer->seen_time;
+
+            /* Consumer PEL */
+            raxIterator ri_cpel;
+            raxStart(&ri_cpel, consumer->pel);
+            raxSeek(&ri_cpel, "^", NULL, 0);
+            while (raxNext(&ri_cpel)) {
+                streamNACK *new_nack = raxFind(new_cg->pel,ri_cpel.key,sizeof(streamID));
+
+                serverAssert(new_nack != raxNotFound);
+
+                new_nack->consumer = new_consumer;
+                raxInsert(new_consumer->pel,ri_cpel.key,sizeof(streamID),new_nack,NULL);
+            }
+            raxStop(&ri_cpel);
+        }
+        raxStop(&ri_consumers);
+    }
+    raxStop(&ri_cgroups);
+    return sobj;
+}
+
 /* This is just a wrapper for lpAppend() to directly use a 64 bit integer
  * instead of a string. */
 unsigned char *lpAppendInteger(unsigned char *lp, int64_t value) {
diff --git a/src/t_zset.c b/src/t_zset.c
index 5b848a1b9..07581673e 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -1553,6 +1553,68 @@ long zsetRank(robj *zobj, sds ele, int reverse) {
     }
 }
 
+/* This is a helper function for the COPY command.
+ * Duplicate a sorted set object, with the guarantee that the returned object
+ * has the same encoding as the original one.
+ *
+ * The resulting object always has refcount set to 1 */
+robj *zsetDup(robj *o) {
+    robj *zobj;
+    zset *zs;
+    zset *new_zs;
+
+    serverAssert(o->type == OBJ_ZSET);
+
+    /* Create a new sorted set object that have the same encoding as the original object's encoding */
+    switch (o->encoding) {
+        case OBJ_ENCODING_ZIPLIST:
+            zobj = createZsetZiplistObject();
+            break;
+        case OBJ_ENCODING_SKIPLIST:
+            zobj = createZsetObject();
+            zs = o->ptr;
+            new_zs = zobj->ptr;
+            dictExpand(new_zs->dict,dictSize(zs->dict));
+            break;
+        default:
+            serverPanic("Wrong encoding.");
+            break;
+    }
+    if (zobj->encoding == OBJ_ENCODING_ZIPLIST) {
+        unsigned char *zl = o->ptr;
+        size_t sz = ziplistBlobLen(zl);
+        unsigned char *new_zl = zmalloc(sz);
+        memcpy(new_zl, zl, sz);
+        zfree(zobj->ptr);
+        zobj->ptr = new_zl;
+    } else if (zobj->encoding == OBJ_ENCODING_SKIPLIST) {
+        zs = o->ptr;
+        new_zs = zobj->ptr;
+        zskiplist *zsl = zs->zsl;
+        zskiplistNode *ln;
+        sds ele;
+        long llen = zsetLength(o);
+
+        /* We copy the skiplist elements from the greatest to the
+         * smallest (that's trivial since the elements are already ordered in
+         * the skiplist): this improves the load process, since the next loaded
+         * element will always be the smaller, so adding to the skiplist
+         * will always immediately stop at the head, making the insertion
+         * O(1) instead of O(log(N)). */
+        ln = zsl->tail;
+        while (llen--) {
+            ele = ln->ele;
+            sds new_ele = sdsdup(ele);
+            zskiplistNode *znode = zslInsert(new_zs->zsl,ln->score,new_ele);
+            dictAdd(new_zs->dict,new_ele,&znode->score);
+            ln = ln->backward;
+        }
+    } else {
+        serverPanic("Unknown sorted set encoding");
+    }
+    return zobj;
+}
+
 /*-----------------------------------------------------------------------------
  * Sorted set commands
  *----------------------------------------------------------------------------*/
diff --git a/tests/unit/keyspace.tcl b/tests/unit/keyspace.tcl
index d4e7bf51c..a74651db9 100644
--- a/tests/unit/keyspace.tcl
+++ b/tests/unit/keyspace.tcl
@@ -169,6 +169,213 @@ start_server {tags {"keyspace"}} {
         format $res
     } {0}
 
+    test {COPY basic usage for string} {
+        r set mykey foobar
+        set res {}
+        r copy mykey mynewkey
+        lappend res [r get mynewkey]
+        lappend res [r dbsize]
+        r copy mykey mynewkey DB 10
+        r select 10
+        lappend res [r get mynewkey]
+        lappend res [r dbsize]
+        r select 9
+        format $res
+    } [list foobar 2 foobar 1]
+
+    test {COPY for string does not replace an existing key without REPLACE option} {
+        r set mykey2 hello
+        catch {r copy mykey2 mynewkey DB 10} e
+        set e
+    } {0}
+
+    test {COPY for string can replace an existing key with REPLACE option} {
+        r copy mykey2 mynewkey DB 10 REPLACE
+        r select 10
+        r get mynewkey
+    } {hello}
+
+    test {COPY for string ensures that copied data is independent of copying data} {
+        r flushdb
+        r select 9
+        r set mykey foobar
+        set res {}
+        r copy mykey mynewkey DB 10
+        r select 10
+        lappend res [r get mynewkey]
+        r set mynewkey hoge
+        lappend res [r get mynewkey]
+        r select 9
+        lappend res [r get mykey]
+        r select 10
+        r flushdb
+        r select 9
+        format $res
+    } [list foobar hoge foobar]
+
+    test {COPY for string does not copy data to no-integer DB} {
+        r set mykey foobar
+        catch {r copy mykey mynewkey DB notanumber} e
+        set e
+    } {*ERR*invalid DB index}
+
+    test {COPY can copy key expire metadata as well} {
+        r set mykey foobar ex 100
+        r copy mykey mynewkey REPLACE
+        assert {[r ttl mynewkey] > 0 && [r ttl mynewkey] <= 100}
+        assert {[r get mynewkey] eq "foobar"}
+    }
+
+    test {COPY does not create an expire if it does not exist} {
+        r set mykey foobar
+        assert {[r ttl mykey] == -1}
+        r copy mykey mynewkey REPLACE
+        assert {[r ttl mynewkey] == -1}
+        assert {[r get mynewkey] eq "foobar"}
+    }
+
+    test {COPY basic usage for list} {
+        r del mylist mynewlist
+        r lpush mylist a b c d
+        r copy mylist mynewlist
+        set digest [r debug digest-value mylist]
+        assert_equal $digest [r debug digest-value mynewlist]
+        assert_equal 1 [r object refcount mylist]
+        assert_equal 1 [r object refcount mynewlist]
+        r del mylist
+        assert_equal $digest [r debug digest-value mynewlist]
+    }
+
+    test {COPY basic usage for intset set} {
+        r del set1 newset1 
+        r sadd set1 1 2 3
+        assert_encoding intset set1
+        r copy set1 newset1
+        set digest [r debug digest-value set1]
+        assert_equal $digest [r debug digest-value newset1]
+        assert_equal 1 [r object refcount set1]
+        assert_equal 1 [r object refcount newset1]
+        r del set1
+        assert_equal $digest [r debug digest-value newset1]
+    }
+
+    test {COPY basic usage for hashtable set} {
+        r del set2 newset2
+        r sadd set2 1 2 3 a
+        assert_encoding hashtable set2
+        r copy set2 newset2
+        set digest [r debug digest-value set2]
+        assert_equal $digest [r debug digest-value newset2]
+        assert_equal 1 [r object refcount set2]
+        assert_equal 1 [r object refcount newset2]
+        r del set2
+        assert_equal $digest [r debug digest-value newset2]
+    }
+
+    test {COPY basic usage for ziplist sorted set} {
+        r del zset1 newzset1
+        r zadd zset1 123 foobar
+        assert_encoding ziplist zset1
+        r copy zset1 newzset1
+        set digest [r debug digest-value zset1]
+        assert_equal $digest [r debug digest-value newzset1]
+        assert_equal 1 [r object refcount zset1]
+        assert_equal 1 [r object refcount newzset1]
+        r del zset1
+        assert_equal $digest [r debug digest-value newzset1]
+    }
+
+     test {COPY basic usage for skiplist sorted set} {
+        r del zset2 newzset2
+        set original_max [lindex [r config get zset-max-ziplist-entries] 1]
+        r config set zset-max-ziplist-entries 0
+        for {set j 0} {$j < 130} {incr j} {
+            r zadd zset2 [randomInt 50] ele-[randomInt 10]
+        }
+        assert_encoding skiplist zset2
+        r copy zset2 newzset2
+        set digest [r debug digest-value zset2]
+        assert_equal $digest [r debug digest-value newzset2]
+        assert_equal 1 [r object refcount zset2]
+        assert_equal 1 [r object refcount newzset2]
+        r del zset2
+        assert_equal $digest [r debug digest-value newzset2]
+        r config set zset-max-ziplist-entries $original_max
+    }
+
+    test {COPY basic usage for ziplist hash} {
+        r del hash1 newhash1
+        r hset hash1 tmp 17179869184
+        assert_encoding ziplist hash1
+        r copy hash1 newhash1
+        set digest [r debug digest-value hash1]
+        assert_equal $digest [r debug digest-value newhash1]
+        assert_equal 1 [r object refcount hash1]
+        assert_equal 1 [r object refcount newhash1]
+        r del hash1
+        assert_equal $digest [r debug digest-value newhash1]
+    }
+
+    test {COPY basic usage for hashtable hash} {
+        r del hash2 newhash2
+        set original_max [lindex [r config get hash-max-ziplist-entries] 1]
+        r config set hash-max-ziplist-entries 0
+        for {set i 0} {$i < 64} {incr i} {
+            r hset hash2 [randomValue] [randomValue]
+        }
+        assert_encoding hashtable hash2
+        r copy hash2 newhash2
+        set digest [r debug digest-value hash2]
+        assert_equal $digest [r debug digest-value newhash2]
+        assert_equal 1 [r object refcount hash2]
+        assert_equal 1 [r object refcount newhash2]
+        r del hash2
+        assert_equal $digest [r debug digest-value newhash2]
+        r config set hash-max-ziplist-entries $original_max
+    }
+
+    test {COPY basic usage for stream} {
+        r del mystream mynewstream
+        for {set i 0} {$i < 1000} {incr i} {
+            r XADD mystream * item 2 value b
+        }
+        r copy mystream mynewstream
+        set digest [r debug digest-value mystream]
+        assert_equal $digest [r debug digest-value mynewstream]
+        assert_equal 1 [r object refcount mystream]
+        assert_equal 1 [r object refcount mynewstream]
+        r del mystream
+        assert_equal $digest [r debug digest-value mynewstream]
+    }
+
+    test {COPY basic usage for stream-cgroups} {
+        r del x
+        r XADD x 100 a 1
+        set id [r XADD x 101 b 1]
+        r XADD x 102 c 1
+        r XADD x 103 e 1
+        r XADD x 104 f 1
+        r XADD x 105 g 1
+        r XGROUP CREATE x g1 0
+        r XGROUP CREATE x g2 0
+        r XREADGROUP GROUP g1 Alice COUNT 1 STREAMS x >
+        r XREADGROUP GROUP g1 Bob COUNT 1 STREAMS x >
+        r XREADGROUP GROUP g1 Bob NOACK COUNT 1 STREAMS x >
+        r XREADGROUP GROUP g2 Charlie COUNT 4 STREAMS x >
+        r XGROUP SETID x g1 $id
+        r XREADGROUP GROUP g1 Dave COUNT 3 STREAMS x >
+        r XDEL x 103
+
+        r copy x newx
+        set info [r xinfo stream x full]
+        assert_equal $info [r xinfo stream newx full]
+        assert_equal 1 [r object refcount x]
+        assert_equal 1 [r object refcount newx]
+        r del x
+        assert_equal $info [r xinfo stream newx full]
+        r flushdb
+    }
+
     test {MOVE basic usage} {
         r set mykey foobar
         r move mykey 10

From d638b058341c0cc09871848848ed0f59a2a23fc8 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 17 Nov 2020 12:52:49 +0200
Subject: [PATCH 0806/1098] Improve and clean up supervised process support.
 (#8036)

* Configuration file default should now be "auto".
* Expose "process_supervised" as an info field.
* Log messages improvements (clarify required systemd config, report
  auto-detected supervision mode, etc.)
* Set server.supervised properly, so it can take precedence of
  "daemonize" configuration.
* Produce clear warning if systemd is detected/requested but executable
  is compiled without support for it, instead of silently ignoring.
* Handle systemd notification error on startup, and turn off supervised
  mode if it failed.
---
 redis.conf   |   9 ++++-
 src/server.c | 101 +++++++++++++++++++++++++++++++++------------------
 2 files changed, 74 insertions(+), 36 deletions(-)

diff --git a/redis.conf b/redis.conf
index d28c17b64..dcb5f6241 100644
--- a/redis.conf
+++ b/redis.conf
@@ -228,6 +228,7 @@ tcp-keepalive 300
 
 # By default Redis does not run as a daemon. Use 'yes' if you need it.
 # Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
+# When Redis is supervised by upstart or systemd, this parameter has no impact.
 daemonize no
 
 # If you run Redis from upstart or systemd, Redis can interact with your
@@ -236,11 +237,17 @@ daemonize no
 #   supervised upstart - signal upstart by putting Redis into SIGSTOP mode
 #                        requires "expect stop" in your upstart job config
 #   supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET
+#                        on startup, and updating Redis status on a regular
+#                        basis.
 #   supervised auto    - detect upstart or systemd method based on
 #                        UPSTART_JOB or NOTIFY_SOCKET environment variables
 # Note: these supervision methods only signal "process is ready."
 #       They do not enable continuous pings back to your supervisor.
-supervised no
+#
+# The default is "no". To run under upstart/systemd, you can simply uncomment
+# the line below:
+#
+# supervised auto
 
 # If a pid file is specified, Redis writes it where specified at startup
 # and removes it at exit.
diff --git a/src/server.c b/src/server.c
index cef9f7185..1ce6b68bd 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4247,11 +4247,20 @@ sds genRedisInfoString(const char *section) {
         static int call_uname = 1;
         static struct utsname name;
         char *mode;
+        char *supervised;
 
         if (server.cluster_enabled) mode = "cluster";
         else if (server.sentinel_mode) mode = "sentinel";
         else mode = "standalone";
 
+        if (server.supervised) {
+            if (server.supervised_mode == SUPERVISED_UPSTART) supervised = "upstart";
+            else if (server.supervised_mode == SUPERVISED_SYSTEMD) supervised = "systemd";
+            else supervised = "unknown";
+        } else {
+            supervised = "no";
+        }
+
         if (sections++) info = sdscat(info,"\r\n");
 
         if (call_uname) {
@@ -4275,6 +4284,7 @@ sds genRedisInfoString(const char *section) {
             "atomicvar_api:%s\r\n"
             "gcc_version:%i.%i.%i\r\n"
             "process_id:%I\r\n"
+            "process_supervised:%s\r\n"
             "run_id:%s\r\n"
             "tcp_port:%i\r\n"
             "uptime_in_seconds:%I\r\n"
@@ -4300,6 +4310,7 @@ sds genRedisInfoString(const char *section) {
             0,0,0,
 #endif
             (int64_t) getpid(),
+            supervised,
             server.runid,
             server.port ? server.port : server.tls_port,
             (int64_t)uptime,
@@ -5224,62 +5235,82 @@ void redisSetCpuAffinity(const char *cpulist) {
 #endif
 }
 
-/*
- * Check whether systemd or upstart have been used to start redis.
- */
+/* Send a notify message to systemd. Returns sd_notify return code which is
+ * a positive number on success. */
+int redisCommunicateSystemd(const char *sd_notify_msg) {
+#ifdef HAVE_LIBSYSTEMD
+    int ret = sd_notify(0, sd_notify_msg);
 
-int redisSupervisedUpstart(void) {
+    if (ret == 0)
+        serverLog(LL_WARNING, "systemd supervision error: NOTIFY_SOCKET not found!");
+    else if (ret < 0)
+        serverLog(LL_WARNING, "systemd supervision error: sd_notify: %d", ret);
+    return ret;
+#else
+    UNUSED(sd_notify_msg);
+    return 0;
+#endif
+}
+
+/* Attempt to set up upstart supervision. Returns 1 if successful. */
+static int redisSupervisedUpstart(void) {
     const char *upstart_job = getenv("UPSTART_JOB");
 
     if (!upstart_job) {
         serverLog(LL_WARNING,
-                "upstart supervision requested, but UPSTART_JOB not found");
+                "upstart supervision requested, but UPSTART_JOB not found!");
         return 0;
     }
 
-    serverLog(LL_NOTICE, "supervised by upstart, will stop to signal readiness");
+    serverLog(LL_NOTICE, "supervised by upstart, will stop to signal readiness.");
     raise(SIGSTOP);
     unsetenv("UPSTART_JOB");
     return 1;
 }
 
-int redisCommunicateSystemd(const char *sd_notify_msg) {
-    const char *notify_socket = getenv("NOTIFY_SOCKET");
-    if (!notify_socket) {
-        serverLog(LL_WARNING,
-                "systemd supervision requested, but NOTIFY_SOCKET not found");
-    }
-
-    #ifdef HAVE_LIBSYSTEMD
-    (void) sd_notify(0, sd_notify_msg);
-    #else
-    UNUSED(sd_notify_msg);
-    #endif
+/* Attempt to set up systemd supervision. Returns 1 if successful. */
+static int redisSupervisedSystemd(void) {
+#ifndef HAVE_LIBSYSTEMD
+    serverLog(LL_WARNING,
+            "systemd supervision requested or auto-detected, but Redis is compiled without libsystemd support!");
     return 0;
+#else
+    if (redisCommunicateSystemd("STATUS=Redis is loading...\n") <= 0)
+        return 0;
+    serverLog(LL_NOTICE,
+        "Supervised by systemd. Please make sure you set appropriate values for TimeoutStartSec and TimeoutStopSec in your service unit.");
+    return 1;
+#endif
 }
 
 int redisIsSupervised(int mode) {
-    if (mode == SUPERVISED_AUTODETECT) {
-        const char *upstart_job = getenv("UPSTART_JOB");
-        const char *notify_socket = getenv("NOTIFY_SOCKET");
+    int ret = 0;
 
-        if (upstart_job) {
-            redisSupervisedUpstart();
-        } else if (notify_socket) {
-            server.supervised_mode = SUPERVISED_SYSTEMD;
-            serverLog(LL_WARNING,
-                "WARNING auto-supervised by systemd - you MUST set appropriate values for TimeoutStartSec and TimeoutStopSec in your service unit.");
-            return redisCommunicateSystemd("STATUS=Redis is loading...\n");
+    if (mode == SUPERVISED_AUTODETECT) {
+        if (getenv("UPSTART_JOB")) {
+            serverLog(LL_VERBOSE, "Upstart supervision detected.");
+            mode = SUPERVISED_UPSTART;
+        } else if (getenv("NOTIFY_SOCKET")) {
+            serverLog(LL_VERBOSE, "Systemd supervision detected.");
+            mode = SUPERVISED_SYSTEMD;
         }
-    } else if (mode == SUPERVISED_UPSTART) {
-        return redisSupervisedUpstart();
-    } else if (mode == SUPERVISED_SYSTEMD) {
-        serverLog(LL_WARNING,
-            "WARNING supervised by systemd - you MUST set appropriate values for TimeoutStartSec and TimeoutStopSec in your service unit.");
-        return redisCommunicateSystemd("STATUS=Redis is loading...\n");
     }
 
-    return 0;
+    switch (mode) {
+        case SUPERVISED_UPSTART:
+            ret = redisSupervisedUpstart();
+            break;
+        case SUPERVISED_SYSTEMD:
+            ret = redisSupervisedSystemd();
+            break;
+        default:
+            break;
+    }
+
+    if (ret)
+        server.supervised_mode = mode;
+
+    return ret;
 }
 
 int iAmMaster(void) {

From 39f716a121dd66d9a13b14797ef4bdcff531e27b Mon Sep 17 00:00:00 2001
From: thomaston <411598110@qq.com>
Date: Wed, 18 Nov 2020 00:35:56 +0800
Subject: [PATCH 0807/1098] ZREVRANGEBYSCORE Optimization for out of range
 offset (#5773)

ZREVRANGEBYSCORE key max min [WITHSCORES] [LIMIT offset count]
When the offset is too large, the query is very slow. Especially when the offset is greater than the length of zset it is easy to determine whether the offset is greater than the length of zset at first, and If it exceed the length of zset, then return directly.

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/t_zset.c             | 6 ++++++
 tests/unit/type/zset.tcl | 1 +
 2 files changed, 7 insertions(+)

diff --git a/src/t_zset.c b/src/t_zset.c
index 07581673e..6147674ff 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -2906,6 +2906,12 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
     if ((zobj = lookupKeyReadOrReply(c,key,shared.emptyarray)) == NULL ||
         checkType(c,zobj,OBJ_ZSET)) return;
 
+    /* For invalid offset, return directly. */
+    if (offset > 0 && offset >= (long)zsetLength(zobj)) {
+        addReply(c,shared.emptyarray);
+        return;
+    }
+
     if (zobj->encoding == OBJ_ENCODING_ZIPLIST) {
         unsigned char *zl = zobj->ptr;
         unsigned char *eptr, *sptr;
diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl
index d8fccd33c..c1406797b 100644
--- a/tests/unit/type/zset.tcl
+++ b/tests/unit/type/zset.tcl
@@ -432,6 +432,7 @@ start_server {tags {"zset"}} {
             create_default_zset
             assert_equal {e 4 f 5} [r zrangebyscore zset 2 5 LIMIT 2 3 WITHSCORES]
             assert_equal {d 3 c 2} [r zrevrangebyscore zset 5 2 LIMIT 2 3 WITHSCORES]
+            assert_equal {} [r zrangebyscore zset 2 5 LIMIT 12 13 WITHSCORES]
         }
 
         test "ZRANGEBYSCORE with non-value min or max" {

From d87a0d02868df3f8739b6e2e1cd8c2a1a9cc14c3 Mon Sep 17 00:00:00 2001
From: "Meir Shpilraien (Spielrein)" <meir@redislabs.com>
Date: Tue, 17 Nov 2020 18:58:55 +0200
Subject: [PATCH 0808/1098] Unified MULTI, LUA, and RM_Call with respect to
 blocking commands (#8025)

Blocking command should not be used with MULTI, LUA, and RM_Call. This is because,
the caller, who executes the command in this context, expects a reply.

Today, LUA and MULTI have a special (and different) treatment to blocking commands:

LUA   - Most commands are marked with no-script flag which are checked when executing
and command from LUA, commands that are not marked (like XREAD) verify that their
blocking mode is not used inside LUA (by checking the CLIENT_LUA client flag).
MULTI - Command that is going to block, first verify that the client is not inside
multi (by checking the CLIENT_MULTI client flag). If the client is inside multi, they
return a result which is a match to the empty key with no timeout (for example blpop
inside MULTI will act as lpop)
For modules that perform RM_Call with blocking command, the returned results type is
REDISMODULE_REPLY_UNKNOWN and the caller can not really know what happened.

Disadvantages of the current state are:

No unified approach, LUA, MULTI, and RM_Call, each has a different treatment
Module can not safely execute blocking command (and get reply or error).
Though It is true that modules are not like LUA or MULTI and should be smarter not
to execute blocking commands on RM_Call, sometimes you want to execute a command base
on client input (for example if you create a module that provides a new scripting
language like javascript or python).
While modules (on modules command) can check for REDISMODULE_CTX_FLAGS_LUA or
REDISMODULE_CTX_FLAGS_MULTI to know not to block the client, there is no way to
check if the command came from another module using RM_Call. So there is no way
for a module to know not to block another module RM_Call execution.

This commit adds a way to unify the treatment for blocking clients by introducing
a new CLIENT_DENY_BLOCKING client flag. On LUA, MULTI, and RM_Call the new flag
turned on to signify that the client should not be blocked. A blocking command
verifies that the flag is turned off before blocking. If a blocking command sees
that the CLIENT_DENY_BLOCKING flag is on, it's not blocking and return results
which are matches to empty key with no timeout (as MULTI does today).

The new flag is checked on the following commands:

List blocking commands: BLPOP, BRPOP, BRPOPLPUSH, BLMOVE,
Zset blocking commands: BZPOPMIN, BZPOPMAX
Stream blocking commands: XREAD, XREADGROUP
SUBSCRIBE, PSUBSCRIBE, MONITOR
In addition, the new flag is turned on inside the AOF client, we do not want to
block the AOF client to prevent deadlocks and commands ordering issues (and there
is also an existing assert in the code that verifies it).

To keep backward compatibility on LUA, all the no-script flags on existing commands
were kept untouched. In addition, a LUA special treatment on XREAD and XREADGROUP was kept.

To keep backward compatibility on MULTI (which today allows SUBSCRIBE, and PSUBSCRIBE).
We added a special treatment on those commands to allow executing them on MULTI.

The only backward compatibility issue that this PR introduces is that now MONITOR
is not allowed inside MULTI.

Tests were added to verify blocking commands are not blocking the client on LUA, MULTI,
or RM_Call. Tests were added to verify the module can check for CLIENT_DENY_BLOCKING flag.

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: Itamar Haber <itamar@redislabs.com>
---
 src/aof.c                              | 13 +++++-
 src/module.c                           |  7 ++++
 src/multi.c                            | 12 ++++++
 src/pubsub.c                           | 24 +++++++++++
 src/redismodule.h                      |  5 ++-
 src/replication.c                      | 13 ++++++
 src/scripting.c                        |  5 ++-
 src/server.c                           |  8 ++++
 src/server.h                           |  3 ++
 src/t_list.c                           |  8 ++--
 src/t_stream.c                         |  9 ++--
 src/t_zset.c                           |  4 +-
 tests/modules/blockedclient.c          | 31 ++++++++++++++
 tests/unit/moduleapi/blockedclient.tcl | 58 ++++++++++++++++++++++++++
 tests/unit/multi.tcl                   | 17 ++++++++
 tests/unit/scripting.tcl               | 32 +++++++++++++-
 16 files changed, 236 insertions(+), 13 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index 5cb78d077..94dc1bdfa 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -673,7 +673,18 @@ struct client *createAOFClient(void) {
     c->original_argv = NULL;
     c->argv_len_sum = 0;
     c->bufpos = 0;
-    c->flags = 0;
+
+    /*
+     * The AOF client should never be blocked (unlike master
+     * replication connection).
+     * This is because blocking the AOF client might cause
+     * deadlock (because potentially no one will unblock it).
+     * Also, if the AOF client will be blocked just for
+     * background processing there is a chance that the
+     * command execution order will be violated.
+     */
+    c->flags = CLIENT_DENY_BLOCKING;
+
     c->btype = BLOCKED_NONE;
     /* We set the fake client as a slave waiting for the synchronization
      * so that Redis will not try to send replies to this client. */
diff --git a/src/module.c b/src/module.c
index 03fde12a0..15ab927cb 100644
--- a/src/module.c
+++ b/src/module.c
@@ -1947,6 +1947,8 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) {
              flags |= REDISMODULE_CTX_FLAGS_LUA;
             if (ctx->client->flags & CLIENT_MULTI)
              flags |= REDISMODULE_CTX_FLAGS_MULTI;
+            if (ctx->client->flags & CLIENT_DENY_BLOCKING)
+             flags |= REDISMODULE_CTX_FLAGS_DENY_BLOCKING;
             /* Module command received from MASTER, is replicated. */
             if (ctx->client->flags & CLIENT_MASTER)
              flags |= REDISMODULE_CTX_FLAGS_REPLICATED;
@@ -3392,6 +3394,9 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
 
     /* Setup our fake client for command execution. */
     c->flags |= CLIENT_MODULE;
+
+    /* We do not want to allow block, the module do not expect it */
+    c->flags |= CLIENT_DENY_BLOCKING;
     c->db = ctx->client->db;
     c->argv = argv;
     c->argc = argc;
@@ -3460,6 +3465,8 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
     }
     call(c,call_flags);
 
+    serverAssert((c->flags & CLIENT_BLOCKED) == 0);
+
     /* Convert the result of the Redis command into a module reply. */
     sds proto = sdsnewlen(c->buf,c->bufpos);
     c->bufpos = 0;
diff --git a/src/multi.c b/src/multi.c
index 3ce6d60ec..7790b0223 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -100,6 +100,7 @@ void multiCommand(client *c) {
         return;
     }
     c->flags |= CLIENT_MULTI;
+
     addReply(c,shared.ok);
 }
 
@@ -168,6 +169,11 @@ void execCommand(client *c) {
         goto handle_monitor;
     }
 
+    uint64_t old_flags = c->flags;
+
+    /* we do not want to allow blocking commands inside multi */
+    c->flags |= CLIENT_DENY_BLOCKING;
+
     /* Exec all the queued commands */
     unwatchAllKeys(c); /* Unwatch ASAP otherwise we'll waste CPU cycles */
     orig_argv = c->argv;
@@ -206,6 +212,7 @@ void execCommand(client *c) {
                 "no permission to touch the specified keys");
         } else {
             call(c,server.loading ? CMD_CALL_NONE : CMD_CALL_FULL);
+            serverAssert((c->flags & CLIENT_BLOCKED) == 0);
         }
 
         /* Commands may alter argc/argv, restore mstate. */
@@ -213,6 +220,11 @@ void execCommand(client *c) {
         c->mstate.commands[j].argv = c->argv;
         c->mstate.commands[j].cmd = c->cmd;
     }
+
+    // restore old DENY_BLOCKING value
+    if (!(old_flags & CLIENT_DENY_BLOCKING))
+        c->flags &= ~CLIENT_DENY_BLOCKING;
+
     c->argv = orig_argv;
     c->argc = orig_argc;
     c->cmd = orig_cmd;
diff --git a/src/pubsub.c b/src/pubsub.c
index 6fa397704..b7f7d9673 100644
--- a/src/pubsub.c
+++ b/src/pubsub.c
@@ -360,6 +360,18 @@ int pubsubPublishMessage(robj *channel, robj *message) {
 void subscribeCommand(client *c) {
     int j;
 
+    if ((c->flags & CLIENT_DENY_BLOCKING) && !(c->flags & CLIENT_MULTI)) {
+        /**
+         * A client that has CLIENT_DENY_BLOCKING flag on
+         * expect a reply per command and so can not execute subscribe.
+         *
+         * Notice that we have a special treatment for multi because of
+         * backword compatibility
+         */
+        addReplyError(c, "subscribe is not allow on DENY BLOCKING client");
+        return;
+    }
+
     for (j = 1; j < c->argc; j++)
         pubsubSubscribeChannel(c,c->argv[j]);
     c->flags |= CLIENT_PUBSUB;
@@ -380,6 +392,18 @@ void unsubscribeCommand(client *c) {
 void psubscribeCommand(client *c) {
     int j;
 
+    if ((c->flags & CLIENT_DENY_BLOCKING) && !(c->flags & CLIENT_MULTI)) {
+        /**
+         * A client that has CLIENT_DENY_BLOCKING flag on
+         * expect a reply per command and so can not execute subscribe.
+         *
+         * Notice that we have a special treatment for multi because of
+         * backword compatibility
+         */
+        addReplyError(c, "PSUBSCRIBE is not allowed for DENY BLOCKING client");
+        return;
+    }
+
     for (j = 1; j < c->argc; j++)
         pubsubSubscribePattern(c,c->argv[j]);
     c->flags |= CLIENT_PUBSUB;
diff --git a/src/redismodule.h b/src/redismodule.h
index 0f7134549..59f568c6f 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -120,11 +120,14 @@
 #define REDISMODULE_CTX_FLAGS_MULTI_DIRTY (1<<19)
 /* Redis is currently running inside background child process. */
 #define REDISMODULE_CTX_FLAGS_IS_CHILD (1<<20)
+/* The current client does not allow blocking, either called from
+ * within multi, lua, or from another module using RM_Call */
+#define REDISMODULE_CTX_FLAGS_DENY_BLOCKING (1<<21)
 
 /* Next context flag, must be updated when adding new flags above!
 This flag should not be used directly by the module.
  * Use RedisModule_GetContextFlagsAll instead. */
-#define _REDISMODULE_CTX_FLAGS_NEXT (1<<21)
+#define _REDISMODULE_CTX_FLAGS_NEXT (1<<22)
 
 /* Keyspace changes notification classes. Every class is associated with a
  * character for configuration purposes.
diff --git a/src/replication.c b/src/replication.c
index d1d26a970..745d317ca 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1372,7 +1372,20 @@ void replicationCreateMasterClient(connection *conn, int dbid) {
     server.master = createClient(conn);
     if (conn)
         connSetReadHandler(server.master->conn, readQueryFromClient);
+
+    /**
+     * Important note:
+     * The CLIENT_DENY_BLOCKING flag is not, and should not, be set here.
+     * For commands like BLPOP, it makes no sense to block the master
+     * connection, and such blocking attempt will probably cause deadlock and
+     * break the replication. We consider such a thing as a bug because
+     * commands as BLPOP should never be sent on the replication link.
+     * A possible use-case for blocking the replication link is if a module wants
+     * to pass the execution to a background thread and unblock after the
+     * execution is done. This is the reason why we allow blocking the replication
+     * connection. */
     server.master->flags |= CLIENT_MASTER;
+
     server.master->authenticated = 1;
     server.master->reploff = server.master_initial_offset;
     server.master->read_reploff = server.master->reploff;
diff --git a/src/scripting.c b/src/scripting.c
index c8ff51b91..732cbfc62 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -724,6 +724,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) {
             call_flags |= CMD_CALL_PROPAGATE_REPL;
     }
     call(c,call_flags);
+    serverAssert((c->flags & CLIENT_BLOCKED) == 0);
 
     /* Convert the result of the Redis command into a suitable Lua type.
      * The first thing we need is to create a single string from the client
@@ -1255,6 +1256,9 @@ void scriptingInit(int setup) {
     if (server.lua_client == NULL) {
         server.lua_client = createClient(NULL);
         server.lua_client->flags |= CLIENT_LUA;
+
+        /* We do not want to allow blocking commands inside Lua */
+        server.lua_client->flags |= CLIENT_DENY_BLOCKING;
     }
 
     /* Lua beginners often don't use "local", this is likely to introduce
@@ -2717,4 +2721,3 @@ void luaLdbLineHook(lua_State *lua, lua_Debug *ar) {
         server.lua_time_start = mstime();
     }
 }
-
diff --git a/src/server.c b/src/server.c
index 1ce6b68bd..ac5b30e90 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4887,6 +4887,14 @@ void infoCommand(client *c) {
 }
 
 void monitorCommand(client *c) {
+    if (c->flags & CLIENT_DENY_BLOCKING) {
+        /**
+         * A client that has CLIENT_DENY_BLOCKING flag on
+         * expects a reply per command and so can't execute MONITOR. */
+        addReplyError(c, "MONITOR is not allowed for DENY BLOCKING client");
+        return;
+    }
+
     /* ignore MONITOR if already slave or in monitor mode */
     if (c->flags & CLIENT_SLAVE) return;
 
diff --git a/src/server.h b/src/server.h
index 66ee066a1..34c4e8360 100644
--- a/src/server.h
+++ b/src/server.h
@@ -267,6 +267,9 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
 #define CLIENT_PROTOCOL_ERROR (1ULL<<39) /* Protocol error chatting with it. */
 #define CLIENT_CLOSE_AFTER_COMMAND (1ULL<<40) /* Close after executing commands
                                                * and writing entire reply. */
+#define CLIENT_DENY_BLOCKING (1ULL<<41) /* Indicate that the client should not be blocked.
+                                           currently, turned on inside MULTI, Lua, RM_Call,
+                                           and AOF client */
 
 /* Client block type (btype field in client structure)
  * if CLIENT_BLOCKED flag is set. */
diff --git a/src/t_list.c b/src/t_list.c
index cdfc0ff51..ce1472d93 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -905,9 +905,9 @@ void blockingPopGenericCommand(client *c, int where) {
         }
     }
 
-    /* If we are inside a MULTI/EXEC and the list is empty the only thing
+    /* If we are not allowed to block the client, the only thing
      * we can do is treating it as a timeout (even with timeout 0). */
-    if (c->flags & CLIENT_MULTI) {
+    if (c->flags & CLIENT_DENY_BLOCKING) {
         addReplyNullArray(c);
         return;
     }
@@ -930,8 +930,8 @@ void blmoveGenericCommand(client *c, int wherefrom, int whereto, mstime_t timeou
     if (checkType(c,key,OBJ_LIST)) return;
 
     if (key == NULL) {
-        if (c->flags & CLIENT_MULTI) {
-            /* Blocking against an empty list in a multi state
+        if (c->flags & CLIENT_DENY_BLOCKING) {
+            /* Blocking against an empty list when blocking is not allowed
              * returns immediately. */
             addReplyNull(c);
         } else {
diff --git a/src/t_stream.c b/src/t_stream.c
index 02e4c3242..b25e54a02 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1521,7 +1521,10 @@ void xreadCommand(client *c) {
         char *o = c->argv[i]->ptr;
         if (!strcasecmp(o,"BLOCK") && moreargs) {
             if (c->flags & CLIENT_LUA) {
-                /* There is no sense to use BLOCK option within LUA */
+                /*
+                 * Although the CLIENT_DENY_BLOCKING flag should protect from blocking the client
+                 * on Lua/MULTI/RM_Call we want special treatment for Lua to keep backword compatibility.
+                 * There is no sense to use BLOCK option within Lua. */
                 addReplyErrorFormat(c, "%s command is not allowed with BLOCK option from scripts", (char *)c->argv[0]->ptr);
                 return;
             }
@@ -1732,9 +1735,9 @@ void xreadCommand(client *c) {
 
     /* Block if needed. */
     if (timeout != -1) {
-        /* If we are inside a MULTI/EXEC and the list is empty the only thing
+        /* If we are not allowed to block the client, the only thing
          * we can do is treating it as a timeout (even with timeout 0). */
-        if (c->flags & CLIENT_MULTI) {
+        if (c->flags & CLIENT_DENY_BLOCKING) {
             addReplyNullArray(c);
             goto cleanup;
         }
diff --git a/src/t_zset.c b/src/t_zset.c
index 6147674ff..f228e76dd 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -3632,9 +3632,9 @@ void blockingGenericZpopCommand(client *c, int where) {
         }
     }
 
-    /* If we are inside a MULTI/EXEC and the zset is empty the only thing
+    /* If we are not allowed to block the client and the zset is empty the only thing
      * we can do is treating it as a timeout (even with timeout 0). */
-    if (c->flags & CLIENT_MULTI) {
+    if (c->flags & CLIENT_DENY_BLOCKING) {
         addReplyNullArray(c);
         return;
     }
diff --git a/tests/modules/blockedclient.c b/tests/modules/blockedclient.c
index 558e06502..cfa8319d8 100644
--- a/tests/modules/blockedclient.c
+++ b/tests/modules/blockedclient.c
@@ -65,6 +65,12 @@ int acquire_gil(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
         return REDISMODULE_OK;
     }
 
+    if ((allFlags & REDISMODULE_CTX_FLAGS_DENY_BLOCKING) &&
+        (flags & REDISMODULE_CTX_FLAGS_DENY_BLOCKING)) {
+        RedisModule_ReplyWithSimpleString(ctx, "Blocked client is not allowed");
+        return REDISMODULE_OK;
+    }
+
     /* This command handler tries to acquire the GIL twice
      * once in the worker thread using "RedisModule_ThreadSafeContextLock"
      * second in the sub-worker thread
@@ -79,6 +85,28 @@ int acquire_gil(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     return REDISMODULE_OK;
 }
 
+int do_rm_call(RedisModuleCtx *ctx, RedisModuleString **argv, int argc){
+    UNUSED(argv);
+    UNUSED(argc);
+
+    if(argc < 2){
+        return RedisModule_WrongArity(ctx);
+    }
+
+    const char* cmd = RedisModule_StringPtrLen(argv[1], NULL);
+
+    RedisModuleCallReply* rep = RedisModule_Call(ctx, cmd, "v", argv + 2, argc - 2);
+    if(!rep){
+        RedisModule_ReplyWithError(ctx, "NULL reply returned");
+    }else{
+        RedisModule_ReplyWithCallReply(ctx, rep);
+        RedisModule_FreeCallReply(rep);
+    }
+
+    return REDISMODULE_OK;
+}
+
+
 int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
@@ -89,5 +117,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     if (RedisModule_CreateCommand(ctx, "acquire_gil", acquire_gil, "", 0, 0, 0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
 
+    if (RedisModule_CreateCommand(ctx, "do_rm_call", do_rm_call, "", 0, 0, 0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
     return REDISMODULE_OK;
 }
diff --git a/tests/unit/moduleapi/blockedclient.tcl b/tests/unit/moduleapi/blockedclient.tcl
index 5a541d138..33ecfff22 100644
--- a/tests/unit/moduleapi/blockedclient.tcl
+++ b/tests/unit/moduleapi/blockedclient.tcl
@@ -14,4 +14,62 @@ start_server {tags {"modules"}} {
     	r acquire_gil
     	assert_equal {{Blocked client is not supported inside multi}} [r exec]
     }
+
+    test {Locked GIL acquisition from RM_Call} {
+    	assert_equal {Blocked client is not allowed} [r do_rm_call acquire_gil]
+    }
+
+    test {Blocking command are not block the client on RM_Call} {
+    	r lpush l test
+    	assert_equal [r do_rm_call blpop l 0] {l test}
+    	
+    	r lpush l test
+    	assert_equal [r do_rm_call brpop l 0] {l test}
+    	
+    	r lpush l1 test
+    	assert_equal [r do_rm_call brpoplpush l1 l2 0] {test}
+    	assert_equal [r do_rm_call brpop l2 0] {l2 test}
+
+    	r lpush l1 test
+    	assert_equal [r do_rm_call blmove l1 l2 LEFT LEFT 0] {test}
+    	assert_equal [r do_rm_call brpop l2 0] {l2 test}
+
+    	r ZADD zset1 0 a 1 b 2 c
+    	assert_equal [r do_rm_call bzpopmin zset1 0] {zset1 a 0}
+    	assert_equal [r do_rm_call bzpopmax zset1 0] {zset1 c 2}
+
+    	r xgroup create s g $ MKSTREAM
+    	r xadd s * foo bar
+    	assert {[r do_rm_call xread BLOCK 0 STREAMS s 0-0] ne {}}
+    	assert {[r do_rm_call xreadgroup group g c BLOCK 0 STREAMS s >] ne {}}
+
+    	assert {[r do_rm_call blpop empty_list 0] eq {}}
+        assert {[r do_rm_call brpop empty_list 0] eq {}}
+        assert {[r do_rm_call brpoplpush empty_list1 empty_list2 0] eq {}}
+        assert {[r do_rm_call blmove empty_list1 empty_list2 LEFT LEFT 0] eq {}}
+        
+        assert {[r do_rm_call bzpopmin empty_zset 0] eq {}}
+        assert {[r do_rm_call bzpopmax empty_zset 0] eq {}}
+       
+        r xgroup create empty_stream g $ MKSTREAM
+        assert {[r do_rm_call xread BLOCK 0 STREAMS empty_stream $] eq {}}
+        assert {[r do_rm_call xreadgroup group g c BLOCK 0 STREAMS empty_stream >] eq {}}
+
+    }
+
+    test {Monitor disallow inside RM_Call} {
+        set e {}
+        catch {
+            r do_rm_call monitor
+        } e
+        set e
+    } {*MONITOR is not allow*}
+
+    test {subscribe disallow inside RM_Call} {
+        set e {}
+        catch {
+            r do_rm_call subscribe x
+        } e
+        set e
+    } {*subscribe is not allow*}
 }
diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl
index e45a2d8f4..43259b1c0 100644
--- a/tests/unit/multi.tcl
+++ b/tests/unit/multi.tcl
@@ -504,4 +504,21 @@ start_server {tags {"multi"}} {
         $r2 config set maxmemory 0
         $r2 close
     }
+
+    test {Blocking commands ignores the timeout} {
+        r xgroup create s g $ MKSTREAM
+
+        set m [r multi]
+        r blpop empty_list 0
+        r brpop empty_list 0
+        r brpoplpush empty_list1 empty_list2 0
+        r blmove empty_list1 empty_list2 LEFT LEFT 0
+        r bzpopmin empty_zset 0
+        r bzpopmax empty_zset 0
+        r xread BLOCK 0 STREAMS s $
+        r xreadgroup group g c BLOCK 0 STREAMS s >
+        set res [r exec]
+
+        list $m $res
+    } {OK {{} {} {} {} {} {} {} {}}}
 }
diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl
index 6bcba4c3f..0da1eda9f 100644
--- a/tests/unit/scripting.tcl
+++ b/tests/unit/scripting.tcl
@@ -140,12 +140,42 @@ start_server {tags {"scripting"}} {
         } {*execution time*}
     }
 
-    test {EVAL - Scripts can't run certain commands} {
+    test {EVAL - Scripts can't run blpop command} {
         set e {}
         catch {r eval {return redis.pcall('blpop','x',0)} 0} e
         set e
     } {*not allowed*}
 
+    test {EVAL - Scripts can't run brpop command} {
+        set e {}
+        catch {r eval {return redis.pcall('brpop','empty_list',0)} 0} e
+        set e
+    } {*not allowed*}
+
+    test {EVAL - Scripts can't run brpoplpush command} {
+        set e {}
+        catch {r eval {return redis.pcall('brpoplpush','empty_list1', 'empty_list2',0)} 0} e
+        set e
+    } {*not allowed*}
+
+    test {EVAL - Scripts can't run blmove command} {
+        set e {}
+        catch {r eval {return redis.pcall('blmove','empty_list1', 'empty_list2', 'LEFT', 'LEFT', 0)} 0} e
+        set e
+    } {*not allowed*}
+
+    test {EVAL - Scripts can't run bzpopmin command} {
+        set e {}
+        catch {r eval {return redis.pcall('bzpopmin','empty_zset', 0)} 0} e
+        set e
+    } {*not allowed*}
+
+    test {EVAL - Scripts can't run bzpopmax command} {
+        set e {}
+        catch {r eval {return redis.pcall('bzpopmax','empty_zset', 0)} 0} e
+        set e
+    } {*not allowed*}
+
     test {EVAL - Scripts can't run XREAD and XREADGROUP with BLOCK option} {
         r del s
         r xgroup create s g $ MKSTREAM

From f8ae991717f10c837c1a76b2954dae56ecb0e6bc Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Wed, 18 Nov 2020 10:16:21 +0100
Subject: [PATCH 0809/1098] EXISTS should not alter LRU, OBJECT should not
 reveal expired keys on replica (#8016)

The bug was introduced by #5021 which only attempted avoid EXIST on an
already expired key from returning 1 on a replica.

Before that commit, dbExists was used instead of
lookupKeyRead (which had an undesired effect to "touch" the LRU/LFU)

Other than that, this commit fixes OBJECT to also come empty handed on
expired keys in replica.

And DEBUG DIGEST-VALUE to behave like DEBUG OBJECT (get the data from
the key regardless of it's expired state)
---
 src/db.c                       | 25 ++++++++++++-------------
 src/debug.c                    |  6 +++++-
 src/object.c                   |  5 +----
 src/server.h                   |  1 +
 tests/unit/introspection-2.tcl |  3 ++-
 5 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/src/db.c b/src/db.c
index b8abb6877..fbbd63748 100644
--- a/src/db.c
+++ b/src/db.c
@@ -102,11 +102,8 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) {
         /* Key expired. If we are in the context of a master, expireIfNeeded()
          * returns 0 only when the key does not exist at all, so it's safe
          * to return NULL ASAP. */
-        if (server.masterhost == NULL) {
-            server.stat_keyspace_misses++;
-            notifyKeyspaceEvent(NOTIFY_KEY_MISS, "keymiss", key, db->id);
-            return NULL;
-        }
+        if (server.masterhost == NULL)
+            goto keymiss;
 
         /* However if we are in the context of a slave, expireIfNeeded() will
          * not really try to expire the key, it only returns information
@@ -125,19 +122,21 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) {
             server.current_client->cmd &&
             server.current_client->cmd->flags & CMD_READONLY)
         {
-            server.stat_keyspace_misses++;
-            notifyKeyspaceEvent(NOTIFY_KEY_MISS, "keymiss", key, db->id);
-            return NULL;
+            goto keymiss;
         }
     }
     val = lookupKey(db,key,flags);
-    if (val == NULL) {
+    if (val == NULL)
+        goto keymiss;
+    server.stat_keyspace_hits++;
+    return val;
+
+keymiss:
+    if (!(flags & LOOKUP_NONOTIFY)) {
         server.stat_keyspace_misses++;
         notifyKeyspaceEvent(NOTIFY_KEY_MISS, "keymiss", key, db->id);
     }
-    else
-        server.stat_keyspace_hits++;
-    return val;
+    return NULL;
 }
 
 /* Like lookupKeyReadWithFlags(), but does not use any flag, which is the
@@ -592,7 +591,7 @@ void existsCommand(client *c) {
     int j;
 
     for (j = 1; j < c->argc; j++) {
-        if (lookupKeyRead(c->db,c->argv[j])) count++;
+        if (lookupKeyReadWithFlags(c->db,c->argv[j],LOOKUP_NOTOUCH)) count++;
     }
     addReplyLongLong(c,count);
 }
diff --git a/src/debug.c b/src/debug.c
index 36cef5625..87e81eadb 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -644,7 +644,11 @@ NULL
         for (int j = 2; j < c->argc; j++) {
             unsigned char digest[20];
             memset(digest,0,20); /* Start with a clean result */
-            robj *o = lookupKeyReadWithFlags(c->db,c->argv[j],LOOKUP_NOTOUCH);
+
+            /* We don't use lookupKey because a debug command should
+             * work on logically expired keys */
+            dictEntry *de;
+            robj *o = ((de = dictFind(c->db->dict,c->argv[j]->ptr)) == NULL) ? NULL : dictGetVal(de);
             if (o) xorObjectDigest(c->db,c->argv[j],digest,o);
 
             sds d = sdsempty();
diff --git a/src/object.c b/src/object.c
index eef5e6d9e..d35ba0eaf 100644
--- a/src/object.c
+++ b/src/object.c
@@ -1229,10 +1229,7 @@ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
 /* This is a helper function for the OBJECT command. We need to lookup keys
  * without any modification of LRU or other parameters. */
 robj *objectCommandLookup(client *c, robj *key) {
-    dictEntry *de;
-
-    if ((de = dictFind(c->db->dict,key->ptr)) == NULL) return NULL;
-    return (robj*) dictGetVal(de);
+    return lookupKeyReadWithFlags(c->db,key,LOOKUP_NOTOUCH|LOOKUP_NONOTIFY);
 }
 
 robj *objectCommandLookupOrReply(client *c, robj *key, robj *reply) {
diff --git a/src/server.h b/src/server.h
index 34c4e8360..53055897a 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2179,6 +2179,7 @@ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
                        long long lru_clock, int lru_multiplier);
 #define LOOKUP_NONE 0
 #define LOOKUP_NOTOUCH (1<<0)
+#define LOOKUP_NONOTIFY (1<<1)
 void dbAdd(redisDb *db, robj *key, robj *val);
 int dbAddRDBLoad(redisDb *db, sds key, robj *val);
 void dbOverwrite(redisDb *db, robj *key, robj *val);
diff --git a/tests/unit/introspection-2.tcl b/tests/unit/introspection-2.tcl
index 0968c2933..a34a15d67 100644
--- a/tests/unit/introspection-2.tcl
+++ b/tests/unit/introspection-2.tcl
@@ -3,11 +3,12 @@ proc cmdstat {cmd} {
 }
 
 start_server {tags {"introspection"}} {
-    test {TTL and TYPYE do not alter the last access time of a key} {
+    test {TTL, TYPE and EXISTS do not alter the last access time of a key} {
         r set foo bar
         after 3000
         r ttl foo
         r type foo
+        r exists foo
         assert {[r object idletime foo] >= 2}
     }
 

From a221a313ae689f1646e1a98969edf41bff4982e2 Mon Sep 17 00:00:00 2001
From: Rosen Penev <rosenp@gmail.com>
Date: Fri, 20 Nov 2020 08:13:11 -0800
Subject: [PATCH 0810/1098] fix compilation with uClibc-ng (#8054)

backtrace can be compile time disabled.
---
 src/config.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/config.h b/src/config.h
index b9c68f38b..649b51d4a 100644
--- a/src/config.h
+++ b/src/config.h
@@ -65,7 +65,7 @@
 /* Test for backtrace() */
 #if defined(__APPLE__) || (defined(__linux__) && defined(__GLIBC__)) || \
     defined(__FreeBSD__) || ((defined(__OpenBSD__) || defined(__NetBSD__)) && defined(USE_BACKTRACE))\
- || defined(__DragonFly__)
+ || defined(__DragonFly__) || (defined(__UCLIBC__) && defined(__UCLIBC_HAS_BACKTRACE__))
 #define HAVE_BACKTRACE 1
 #endif
 

From 61954951edbda670bfbae8be0147daa64df95f26 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 22 Nov 2020 13:57:56 +0200
Subject: [PATCH 0811/1098] Fix oom-score-adj-values range, abs options, and
 bug when used in config file (#8046)

Fix: When oom-score-adj-values is provided in the config file after
oom-score-adj yes, it'll take an immediate action, before
readOOMScoreAdj was acquired, resulting in an error (out of range score
due to uninitialized value. delay the reaction the real call is made by
main().

Since the values are clamped to -1000..1000, and they're
applied as an offset from the value at startup (which may be -1000), we
need to allow the offsets to reach to +2000 so that a value of +1000 is
achievable in case the value at startup was -1000.

Adding an option for absolute values rather than relative ones.
---
 redis.conf   | 23 ++++++++++++++---------
 src/config.c | 25 +++++++++++++++++++------
 src/server.c |  6 ++++--
 src/server.h |  5 +++++
 4 files changed, 42 insertions(+), 17 deletions(-)

diff --git a/redis.conf b/redis.conf
index dcb5f6241..b1c8e4f0c 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1086,21 +1086,26 @@ lazyfree-lazy-user-del no
 # for all its processes, depending on their role. The default scores will
 # attempt to have background child processes killed before all others, and
 # replicas killed before masters.
-
+#
+# Redis supports three options:
+#
+# no:       Don't make changes to oom-score-adj (default).
+# yes:      Alias to "relative" see below.
+# absolute: Values in oom-score-adj-values are written as is to the kernel.
+# relative: Values are used relative to the initial value of oom_score_adj when
+#           the server starts and are then clamped to a range of -1000 to 1000.
+#           Because typically the initial value is 0, they will often match the
+#           absolute values.
 oom-score-adj no
 
 # When oom-score-adj is used, this directive controls the specific values used
-# for master, replica and background child processes. Values range -1000 to
-# 1000 (higher means more likely to be killed).
+# for master, replica and background child processes. Values range -2000 to
+# 2000 (higher means more likely to be killed).
 #
 # Unprivileged processes (not root, and without CAP_SYS_RESOURCE capabilities)
 # can freely increase their value, but not decrease it below its initial
-# settings.
-#
-# Values are used relative to the initial value of oom_score_adj when the server
-# starts. Because typically the initial value is 0, they will often match the
-# absolute values.
-
+# settings. This means that setting oom-score-adj to "relative" and setting the
+# oom-score-adj-values to positive values will always succeed.
 oom-score-adj-values 0 200 800
 
 
diff --git a/src/config.c b/src/config.c
index 07ed91e5f..b4fd9bee9 100644
--- a/src/config.c
+++ b/src/config.c
@@ -104,6 +104,15 @@ configEnum tls_auth_clients_enum[] = {
     {"optional", TLS_CLIENT_AUTH_OPTIONAL},
     {NULL, 0}
 };
+
+configEnum oom_score_adj_enum[] = {
+    {"no", OOM_SCORE_ADJ_NO},
+    {"yes", OOM_SCORE_RELATIVE},
+    {"relative", OOM_SCORE_RELATIVE},
+    {"absolute", OOM_SCORE_ADJ_ABSOLUTE},
+    {NULL, 0}
+};
+
 /* Output buffer limits presets. */
 clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = {
     {0, 0, 0}, /* normal */
@@ -293,7 +302,7 @@ void queueLoadModule(sds path, sds *argv, int argc) {
  * server.oom_score_adj_values if valid.
  */
 
-static int updateOOMScoreAdjValues(sds *args, char **err) {
+static int updateOOMScoreAdjValues(sds *args, char **err, int apply) {
     int i;
     int values[CONFIG_OOM_COUNT];
 
@@ -301,8 +310,8 @@ static int updateOOMScoreAdjValues(sds *args, char **err) {
         char *eptr;
         long long val = strtoll(args[i], &eptr, 10);
 
-        if (*eptr != '\0' || val < -1000 || val > 1000) {
-            if (err) *err = "Invalid oom-score-adj-values, elements must be between -1000 and 1000.";
+        if (*eptr != '\0' || val < -2000 || val > 2000) {
+            if (err) *err = "Invalid oom-score-adj-values, elements must be between -2000 and 2000.";
             return C_ERR;
         }
 
@@ -326,6 +335,10 @@ static int updateOOMScoreAdjValues(sds *args, char **err) {
         old_values[i] = server.oom_score_adj_values[i];
         server.oom_score_adj_values[i] = values[i];
     }
+    
+    /* When parsing the config file, we want to apply only when all is done. */
+    if (!apply)
+        return C_OK;
 
     /* Update */
     if (setOOMScoreAdj(-1) == C_ERR) {
@@ -559,7 +572,7 @@ void loadServerConfigFromString(char *config) {
             server.client_obuf_limits[class].soft_limit_bytes = soft;
             server.client_obuf_limits[class].soft_limit_seconds = soft_seconds;
         } else if (!strcasecmp(argv[0],"oom-score-adj-values") && argc == 1 + CONFIG_OOM_COUNT) {
-            if (updateOOMScoreAdjValues(&argv[1], &err) == C_ERR) goto loaderr;
+            if (updateOOMScoreAdjValues(&argv[1], &err, 0) == C_ERR) goto loaderr;
         } else if (!strcasecmp(argv[0],"notify-keyspace-events") && argc == 2) {
             int flags = keyspaceEventsStringToFlags(argv[1]);
 
@@ -822,7 +835,7 @@ void configSetCommand(client *c) {
         int success = 1;
 
         sds *v = sdssplitlen(o->ptr, sdslen(o->ptr), " ", 1, &vlen);
-        if (vlen != CONFIG_OOM_COUNT || updateOOMScoreAdjValues(v, &errstr) == C_ERR)
+        if (vlen != CONFIG_OOM_COUNT || updateOOMScoreAdjValues(v, &errstr, 1) == C_ERR)
             success = 0;
 
         sdsfreesplitres(v, vlen);
@@ -2313,7 +2326,6 @@ standardConfig configs[] = {
     createBoolConfig("crash-log-enabled", NULL, MODIFIABLE_CONFIG, server.crashlog_enabled, 1, NULL, updateSighandlerEnabled),
     createBoolConfig("crash-memcheck-enabled", NULL, MODIFIABLE_CONFIG, server.memcheck_enabled, 1, NULL, NULL),
     createBoolConfig("use-exit-on-panic", NULL, MODIFIABLE_CONFIG, server.use_exit_on_panic, 0, NULL, NULL),
-    createBoolConfig("oom-score-adj", NULL, MODIFIABLE_CONFIG, server.oom_score_adj, 0, NULL, updateOOMScoreAdj),
     createBoolConfig("disable-thp", NULL, MODIFIABLE_CONFIG, server.disable_thp, 1, NULL, NULL),
 
     /* String Configs */
@@ -2339,6 +2351,7 @@ standardConfig configs[] = {
     createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, server.verbosity, LL_NOTICE, NULL, NULL),
     createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, server.maxmemory_policy, MAXMEMORY_NO_EVICTION, NULL, NULL),
     createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, AOF_FSYNC_EVERYSEC, NULL, NULL),
+    createEnumConfig("oom-score-adj", NULL, MODIFIABLE_CONFIG, oom_score_adj_enum, server.oom_score_adj, OOM_SCORE_ADJ_NO, NULL, updateOOMScoreAdj),
 
     /* Integer configs */
     createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, 16, INTEGER_CONFIG, NULL, NULL),
diff --git a/src/server.c b/src/server.c
index ac5b30e90..3c1f1cb6b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2667,7 +2667,7 @@ static void readOOMScoreAdj(void) {
  */
 int setOOMScoreAdj(int process_class) {
 
-    if (!server.oom_score_adj) return C_OK;
+    if (server.oom_score_adj == OOM_SCORE_ADJ_NO) return C_OK;
     if (process_class == -1)
         process_class = (server.masterhost ? CONFIG_OOM_REPLICA : CONFIG_OOM_MASTER);
 
@@ -2678,7 +2678,9 @@ int setOOMScoreAdj(int process_class) {
     int val;
     char buf[64];
 
-    val = server.oom_score_adj_base + server.oom_score_adj_values[process_class];
+    val = server.oom_score_adj_values[process_class];
+    if (server.oom_score_adj == OOM_SCORE_RELATIVE)
+        val += server.oom_score_adj_base;
     if (val > 1000) val = 1000;
     if (val < -1000) val = -1000;
 
diff --git a/src/server.h b/src/server.h
index 53055897a..4690151c0 100644
--- a/src/server.h
+++ b/src/server.h
@@ -382,6 +382,11 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
 #define SET_OP_DIFF 1
 #define SET_OP_INTER 2
 
+/* oom-score-adj defines */
+#define OOM_SCORE_ADJ_NO 0
+#define OOM_SCORE_RELATIVE 1
+#define OOM_SCORE_ADJ_ABSOLUTE 2
+
 /* Redis maxmemory strategies. Instead of using just incremental number
  * for this defines, we use a set of flags so that testing for certain
  * properties common to multiple policies is faster. */

From e6fa47380a5274119ed37c7a5ea7455d4b7dbdcc Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 22 Nov 2020 14:00:51 +0200
Subject: [PATCH 0812/1098] Fix bug with module GIL being released prematurely
 (#8061)

This is hopefully usually harmles.
The server.ready_keys will usually be empty so the code after releasing
the GIL will soon be done.
The only case where it'll actually process things is when a module
releases a client (or module) blocked on a key, by triggering this NOT
from within a command (e.g. a timer event).

This bug was introduced in redis 6.0.9, see #7903
---
 src/server.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/server.c b/src/server.c
index 3c1f1cb6b..f30d33555 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2306,15 +2306,17 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     /* Close clients that need to be closed asynchronous */
     freeClientsInAsyncFreeQueue();
 
+    /* Try to process blocked clients every once in while. Example: A module
+     * calls RM_SignalKeyAsReady from within a timer callback (So we don't
+     * visit processCommand() at all). */
+    handleClientsBlockedOnKeys();
+
     /* Before we are going to sleep, let the threads access the dataset by
      * releasing the GIL. Redis main thread will not touch anything at this
      * time. */
     if (moduleCount()) moduleReleaseGIL();
 
-    /* Try to process blocked clients every once in while. Example: A module
-     * calls RM_SignalKeyAsReady from within a timer callback (So we don't
-     * visit processCommand() at all). */
-    handleClientsBlockedOnKeys();
+    /* Do NOT add anything below moduleReleaseGIL !!! */
 }
 
 /* This function is called immediately after the event loop multiplexing
@@ -2323,6 +2325,9 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
 void afterSleep(struct aeEventLoop *eventLoop) {
     UNUSED(eventLoop);
 
+    /* Do NOT add anything above moduleAcquireGIL !!! */
+
+    /* Aquire the modules GIL so that their threads won't touch anything. */
     if (!ProcessingEventsWhileBlocked) {
         if (moduleCount()) moduleAcquireGIL();
     }

From f207e1682fadfe343c55102a1324b9b5c830e074 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuancode@163.com>
Date: Sun, 22 Nov 2020 20:12:45 +0800
Subject: [PATCH 0813/1098] Fix diskless replication failure when has non-rdb
 child process (#8070)

If we enable diskless replication, set repl-diskless-sync-delay to 0,
and master has non-rdb child process such as rewrite aof child, master
will try to start to a new BGSAVE but fails immediately (before fork)
when replicas ask for full synchronization, and master always fails
to start a new BGSAVE and disconnects with replicas until non-rdb
child process exists.

this bug was introduced in #6271 (not yet released in 6.0.x)
---
 src/replication.c | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index 745d317ca..a288478ff 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -826,18 +826,16 @@ void syncCommand(client *c) {
 
     /* CASE 3: There is no BGSAVE is progress. */
     } else {
-        if (server.repl_diskless_sync && (c->slave_capa & SLAVE_CAPA_EOF)) {
+        if (server.repl_diskless_sync && (c->slave_capa & SLAVE_CAPA_EOF) &&
+            server.repl_diskless_sync_delay)
+        {
             /* Diskless replication RDB child is created inside
              * replicationCron() since we want to delay its start a
              * few seconds to wait for more slaves to arrive. */
-            if (server.repl_diskless_sync_delay)
-                serverLog(LL_NOTICE,"Delay next BGSAVE for diskless SYNC");
-            else
-                startBgsaveForReplication(c->slave_capa);
+            serverLog(LL_NOTICE,"Delay next BGSAVE for diskless SYNC");
         } else {
-            /* Target is disk (or the slave is not capable of supporting
-             * diskless replication) and we don't have a BGSAVE in progress,
-             * let's start one. */
+            /* We don't have a BGSAVE in progress, let's start one. Diskless
+             * or disk-based mode is determined by replica's capacity. */
             if (!hasActiveChildProcess()) {
                 startBgsaveForReplication(c->slave_capa);
             } else {

From 08d3e929e584b174bcbf0e2426913d0761303e7f Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 22 Nov 2020 14:40:38 +0200
Subject: [PATCH 0814/1098] Clean up building with USE_SYSTEMD. (#8073)

When USE_SYSTEMD=yes is specified, try to use pkg-config to determine
libsystemd linker flags. If not found, silently fall back to simply
using "-lsystemd".

We now use a LIBSYSTEMD_LIBS variable so users can explicitly override
it and specify their own library.

If USE_SYSTEMD is unspecified the old behavior of auto-enabling it if
both pkg-config and libsystemd are available is retained.
---
 src/Makefile | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/Makefile b/src/Makefile
index a32e4c150..653a6e6b0 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -180,6 +180,8 @@ FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps
 
 # Determine systemd support and/or build preference (defaulting to auto-detection)
 BUILD_WITH_SYSTEMD=no
+LIBSYSTEMD_LIBS=-lsystemd
+
 # If 'USE_SYSTEMD' in the environment is neither "no" nor "yes", try to
 # auto-detect libsystemd's presence and link accordingly.
 ifneq ($(USE_SYSTEMD),no)
@@ -188,17 +190,18 @@ ifneq ($(USE_SYSTEMD),no)
 # (unless a later check tells us otherwise)
 ifeq ($(LIBSYSTEMD_PKGCONFIG),0)
 	BUILD_WITH_SYSTEMD=yes
+	LIBSYSTEMD_LIBS=$(shell $(PKG_CONFIG) --libs libsystemd)
 endif
 endif
+
+# If 'USE_SYSTEMD' is set to "yes" use pkg-config if available or fall back to
+# default -lsystemd.
 ifeq ($(USE_SYSTEMD),yes)
-ifneq ($(LIBSYSTEMD_PKGCONFIG),0)
-$(error USE_SYSTEMD is set to "$(USE_SYSTEMD)", but $(PKG_CONFIG) cannot find libsystemd)
-endif
-# Force building with libsystemd
 	BUILD_WITH_SYSTEMD=yes
 endif
+
 ifeq ($(BUILD_WITH_SYSTEMD),yes)
-	FINAL_LIBS+=$(shell $(PKG_CONFIG) --libs libsystemd)
+	FINAL_LIBS+=$(LIBSYSTEMD_LIBS)
 	FINAL_CFLAGS+= -DHAVE_LIBSYSTEMD
 endif
 

From 0b1d89d7ae61cc9f2546cf4dbbfc5fcd7d29b518 Mon Sep 17 00:00:00 2001
From: xindoo <xindoo@users.noreply.github.com>
Date: Sun, 22 Nov 2020 20:56:45 +0800
Subject: [PATCH 0815/1098] fix comment error about zslDeleteRangeByScore range
 (#8075)

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/t_zset.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/t_zset.c b/src/t_zset.c
index f228e76dd..1f5f00a62 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -377,7 +377,8 @@ zskiplistNode *zslLastInRange(zskiplist *zsl, zrangespec *range) {
 }
 
 /* Delete all the elements with score between min and max from the skiplist.
- * Min and max are inclusive, so a score >= min || score <= max is deleted.
+ * Both min and max can be inclusive or exclusive (see range->minex and
+ * range->maxex). When inclusive a score >= min && score <= max is deleted.
  * Note that this function takes the reference to the hash table view of the
  * sorted set, in order to remove the elements from the hash table too. */
 unsigned long zslDeleteRangeByScore(zskiplist *zsl, zrangespec *range, dict *dict) {

From 0963edbc65e563262dadf514a13782d639d3cab0 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Mon, 23 Nov 2020 14:37:58 +0200
Subject: [PATCH 0816/1098] Adds tests for XADD/XTRIM's MAXLEN arguments
 (#8083)

---
 tests/unit/type/stream.tcl | 45 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl
index b02d6a0fb..4b3672d64 100644
--- a/tests/unit/type/stream.tcl
+++ b/tests/unit/type/stream.tcl
@@ -102,6 +102,30 @@ start_server {
         }
     }
 
+    test {XADD with MAXLEN option and the '=' argument} {
+        r DEL mystream
+        for {set j 0} {$j < 1000} {incr j} {
+            if {rand() < 0.9} {
+                r XADD mystream MAXLEN = 5 * xitem $j
+            } else {
+                r XADD mystream MAXLEN = 5 * yitem $j
+            }
+        }
+        assert {[r XLEN mystream] == 5}
+    }
+
+    test {XADD with MAXLEN option and the '~' argument} {
+        r DEL mystream
+        for {set j 0} {$j < 1000} {incr j} {
+            if {rand() < 0.9} {
+                r XADD mystream MAXLEN ~ 555 * xitem $j
+            } else {
+                r XADD mystream MAXLEN ~ 555 * yitem $j
+            }
+        }
+        assert {[r XLEN mystream] == 600}
+    }
+
     test {XADD with NOMKSTREAM option} {
         r DEL mystream
         assert_equal "" [r XADD mystream NOMKSTREAM * item 1 value a]
@@ -378,6 +402,27 @@ start_server {
         r XADD x * f2 v2
         assert_equal [r XRANGE x - +] {{2577343934890-18446744073709551615 {f v}} {2577343934891-0 {f2 v2}}}
     }
+
+    test {XTRIM with MAXLEN option basic test} {
+        r DEL mystream
+        for {set j 0} {$j < 1000} {incr j} {
+            if {rand() < 0.9} {
+                r XADD mystream * xitem $j
+            } else {
+                r XADD mystream * yitem $j
+            }
+        }
+        r XTRIM mystream MAXLEN 666
+        assert {[r XLEN mystream] == 666}
+        r XTRIM mystream MAXLEN = 555
+        assert {[r XLEN mystream] == 555}
+        r XTRIM mystream MAXLEN ~ 444
+        assert {[r XLEN mystream] == 500}
+        r XTRIM mystream MAXLEN ~ 400
+        assert {[r XLEN mystream] == 400}
+    }
+
+    
 }
 
 start_server {tags {"stream"} overrides {appendonly yes}} {

From f16b52cb7deb1c5793243c11498623df75986e73 Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Mon, 23 Nov 2020 14:14:33 +0000
Subject: [PATCH 0817/1098] redis_set_thread_title support for Haiku. (#8060)

---
 src/config.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/config.h b/src/config.h
index 649b51d4a..2cd44f927 100644
--- a/src/config.h
+++ b/src/config.h
@@ -241,6 +241,9 @@ void setproctitle(const char *fmt, ...);
 #elif defined __NetBSD__
 #include <pthread.h>
 #define redis_set_thread_title(name) pthread_setname_np(pthread_self(), "%s", name)
+#elif defined __HAIKU__
+#include <kernel/OS.h>
+#define redis_set_thread_title(name) rename_thread(find_thread(0), name)
 #else
 #if (defined __APPLE__ && defined(MAC_OS_X_VERSION_10_7))
 int pthread_setname_np(const char *name);

From 7e5a6313f0add995c723351532d994118e3e8a6d Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 24 Nov 2020 17:58:10 +0200
Subject: [PATCH 0818/1098] Fix use-after-free issue in spt_copyenv. (#8088)

Seems to have gone unnoticed for a long time, because at least with
glibc it will only be triggered if setenv() was called before spt_init,
which Redis doesn't.

Fixes #8064.
---
 src/setproctitle.c | 69 +++++++++++++++++++++++++++++++++-------------
 1 file changed, 50 insertions(+), 19 deletions(-)

diff --git a/src/setproctitle.c b/src/setproctitle.c
index 5f91d7bfe..707383673 100644
--- a/src/setproctitle.c
+++ b/src/setproctitle.c
@@ -50,6 +50,10 @@
 #if !HAVE_SETPROCTITLE
 #if (defined __linux || defined __APPLE__)
 
+#ifdef __GLIBC__
+#define HAVE_CLEARENV
+#endif
+
 extern char **environ;
 
 static struct {
@@ -80,11 +84,9 @@ static inline size_t spt_min(size_t a, size_t b) {
  * For discussion on the portability of the various methods, see
  * http://lists.freebsd.org/pipermail/freebsd-stable/2008-June/043136.html
  */
-static int spt_clearenv(void) {
-#if __GLIBC__
-	clearenv();
-
-	return 0;
+int spt_clearenv(void) {
+#ifdef HAVE_CLEARENV
+	return clearenv();
 #else
 	extern char **environ;
 	static char **tmp;
@@ -100,34 +102,62 @@ static int spt_clearenv(void) {
 } /* spt_clearenv() */
 
 
-static int spt_copyenv(char *oldenv[]) {
+static int spt_copyenv(int envc, char *oldenv[]) {
 	extern char **environ;
+	char **envcopy = NULL;
 	char *eq;
 	int i, error;
+	int envsize;
 
 	if (environ != oldenv)
 		return 0;
 
-	if ((error = spt_clearenv()))
-		goto error;
+	/* Copy environ into envcopy before clearing it. Shallow copy is
+	 * enough as clearenv() only clears the environ array.
+	 */
+	envsize = (envc + 1) * sizeof(char *);
+	envcopy = malloc(envsize);
+	if (!envcopy)
+		return ENOMEM;
+	memcpy(envcopy, oldenv, envsize);
 
-	for (i = 0; oldenv[i]; i++) {
-		if (!(eq = strchr(oldenv[i], '=')))
+	/* Note that the state after clearenv() failure is undefined, but we'll
+	 * just assume an error means it was left unchanged.
+	 */
+	if ((error = spt_clearenv())) {
+		environ = oldenv;
+		free(envcopy);
+		return error;
+	}
+
+	/* Set environ from envcopy */
+	for (i = 0; envcopy[i]; i++) {
+		if (!(eq = strchr(envcopy[i], '=')))
 			continue;
 
 		*eq = '\0';
-		error = (0 != setenv(oldenv[i], eq + 1, 1))? errno : 0;
+		error = (0 != setenv(envcopy[i], eq + 1, 1))? errno : 0;
 		*eq = '=';
 
-		if (error)
-			goto error;
+		/* On error, do our best to restore state */
+		if (error) {
+#ifdef HAVE_CLEARENV
+			/* We don't assume it is safe to free environ, so we
+			 * may leak it. As clearenv() was shallow using envcopy
+			 * here is safe.
+			 */
+			environ = envcopy;
+#else
+			free(envcopy);
+			free(environ);  /* Safe to free, we have just alloc'd it */
+			environ = oldenv;
+#endif
+			return error;
+		}
 	}
 
+	free(envcopy);
 	return 0;
-error:
-	environ = oldenv;
-
-	return error;
 } /* spt_copyenv() */
 
 
@@ -152,7 +182,7 @@ static int spt_copyargs(int argc, char *argv[]) {
 void spt_init(int argc, char *argv[]) {
         char **envp = environ;
 	char *base, *end, *nul, *tmp;
-	int i, error;
+	int i, error, envc;
 
 	if (!(base = argv[0]))
 		return;
@@ -173,6 +203,7 @@ void spt_init(int argc, char *argv[]) {
 
 		end = envp[i] + strlen(envp[i]) + 1;
 	}
+	envc = i;
 
 	if (!(SPT.arg0 = strdup(argv[0])))
 		goto syerr;
@@ -195,7 +226,7 @@ void spt_init(int argc, char *argv[]) {
 #endif
 
 
-	if ((error = spt_copyenv(envp)))
+	if ((error = spt_copyenv(envc, envp)))
 		goto error;
 
 	if ((error = spt_copyargs(argc, argv)))

From 25f457c7f69e7f0254cc5d585eadd784dd30d91c Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Wed, 25 Nov 2020 03:40:58 +0800
Subject: [PATCH 0819/1098] Avoid excessive malloc and free in copyCommand robj
 creation (#8067)

Avoid multiple conditional judgments
Avoid allocating robj->ptr when we're gonna replace it right after.
---
 src/t_hash.c | 27 +++++++++------------------
 src/t_list.c |  5 ++---
 src/t_set.c  | 24 +++++++-----------------
 src/t_zset.c | 24 ++++++------------------
 4 files changed, 24 insertions(+), 56 deletions(-)

diff --git a/src/t_hash.c b/src/t_hash.c
index 5c844ce30..f3fde87aa 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -515,29 +515,17 @@ robj *hashTypeDup(robj *o) {
 
     serverAssert(o->type == OBJ_HASH);
 
-    switch (o->encoding) {
-        case OBJ_ENCODING_ZIPLIST:
-            hobj = createHashObject();
-            break;
-        case OBJ_ENCODING_HT:
-            hobj = createHashObject();
-            hashTypeConvert(hobj, OBJ_ENCODING_HT);
-            dict *d = o->ptr;
-            dictExpand(hobj->ptr, dictSize(d));
-            break;
-        default:
-            serverPanic("Wrong encoding.");
-            break;
-    }
-
     if(o->encoding == OBJ_ENCODING_ZIPLIST){
         unsigned char *zl = o->ptr;
         size_t sz = ziplistBlobLen(zl);
         unsigned char *new_zl = zmalloc(sz);
         memcpy(new_zl, zl, sz);
-        zfree(hobj->ptr);
-        hobj->ptr = new_zl;
+        hobj = createObject(OBJ_HASH, new_zl);
+        hobj->encoding = OBJ_ENCODING_ZIPLIST;
     } else if(o->encoding == OBJ_ENCODING_HT){
+        dict *d = dictCreate(&hashDictType, NULL);
+        dictExpand(d, dictSize((const dict*)o->ptr));
+
         hi = hashTypeInitIterator(o);
         while (hashTypeNext(hi) != C_ERR) {
             sds field, value;
@@ -549,9 +537,12 @@ robj *hashTypeDup(robj *o) {
             newvalue = sdsdup(value);
 
             /* Add a field-value pair to a new hash object. */
-            dictAdd(hobj->ptr,newfield,newvalue);
+            dictAdd(d,newfield,newvalue);
         }
         hashTypeReleaseIterator(hi);
+
+        hobj = createObject(OBJ_HASH, d);
+        hobj->encoding = OBJ_ENCODING_HT;
     } else {
         serverPanic("Unknown hash encoding");
     }
diff --git a/src/t_list.c b/src/t_list.c
index ce1472d93..acabe7be2 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -202,14 +202,13 @@ robj *listTypeDup(robj *o) {
 
     switch (o->encoding) {
         case OBJ_ENCODING_QUICKLIST:
-            lobj = createQuicklistObject();
+            lobj = createObject(OBJ_LIST, quicklistDup(o->ptr));
+            lobj->encoding = OBJ_ENCODING_QUICKLIST;
             break;
         default:
             serverPanic("Wrong encoding.");
             break;
     }
-    zfree(lobj->ptr);
-    lobj->ptr = quicklistDup(o->ptr);
     return lobj;
 }
 
diff --git a/src/t_set.c b/src/t_set.c
index cb4c5002f..b2da14cd5 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -275,27 +275,17 @@ robj *setTypeDup(robj *o) {
     serverAssert(o->type == OBJ_SET);
 
     /* Create a new set object that have the same encoding as the original object's encoding */
-    switch (o->encoding) {
-        case OBJ_ENCODING_INTSET:
-            set = createIntsetObject();
-            break;
-        case OBJ_ENCODING_HT:
-            set = createSetObject();
-            dict *d = o->ptr;
-            dictExpand(set->ptr, dictSize(d));
-            break;
-        default:
-            serverPanic("Wrong encoding.");
-            break;
-    }
-    if (set->encoding == OBJ_ENCODING_INTSET) {
+    if (o->encoding == OBJ_ENCODING_INTSET) {
         intset *is = o->ptr;
         size_t size = intsetBlobLen(is);
         intset *newis = zmalloc(size);
         memcpy(newis,is,size);
-        zfree(set->ptr);
-        set->ptr = newis;
-    } else if (set->encoding == OBJ_ENCODING_HT) {
+        set = createObject(OBJ_SET, newis);
+        set->encoding = OBJ_ENCODING_INTSET;
+    } else if (o->encoding == OBJ_ENCODING_HT) {
+        set = createSetObject();
+        dict *d = o->ptr;
+        dictExpand(set->ptr, dictSize(d));
         si = setTypeInitIterator(o);
         while (setTypeNext(si, &elesds, &intobj) != -1) {
             setTypeAdd(set, elesds);
diff --git a/src/t_zset.c b/src/t_zset.c
index 1f5f00a62..a6ce20132 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -1567,30 +1567,18 @@ robj *zsetDup(robj *o) {
     serverAssert(o->type == OBJ_ZSET);
 
     /* Create a new sorted set object that have the same encoding as the original object's encoding */
-    switch (o->encoding) {
-        case OBJ_ENCODING_ZIPLIST:
-            zobj = createZsetZiplistObject();
-            break;
-        case OBJ_ENCODING_SKIPLIST:
-            zobj = createZsetObject();
-            zs = o->ptr;
-            new_zs = zobj->ptr;
-            dictExpand(new_zs->dict,dictSize(zs->dict));
-            break;
-        default:
-            serverPanic("Wrong encoding.");
-            break;
-    }
-    if (zobj->encoding == OBJ_ENCODING_ZIPLIST) {
+    if (o->encoding == OBJ_ENCODING_ZIPLIST) {
         unsigned char *zl = o->ptr;
         size_t sz = ziplistBlobLen(zl);
         unsigned char *new_zl = zmalloc(sz);
         memcpy(new_zl, zl, sz);
-        zfree(zobj->ptr);
-        zobj->ptr = new_zl;
-    } else if (zobj->encoding == OBJ_ENCODING_SKIPLIST) {
+        zobj = createObject(OBJ_ZSET, new_zl);
+        zobj->encoding = OBJ_ENCODING_ZIPLIST;
+    } else if (o->encoding == OBJ_ENCODING_SKIPLIST) {
+        zobj = createZsetObject();
         zs = o->ptr;
         new_zs = zobj->ptr;
+        dictExpand(new_zs->dict,dictSize(zs->dict));
         zskiplist *zsl = zs->zsl;
         zskiplistNode *ln;
         sds ele;

From 0719388cfb1a79160204314beb1de1f9c29a3684 Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Wed, 25 Nov 2020 20:15:32 +0000
Subject: [PATCH 0820/1098] raspberry build fix. (#8095)

__ILP32__ is 32 bits ABI and does not imply x86, this patch resolves this.
---
 src/debug.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/debug.c b/src/debug.c
index 87e81eadb..d32a8ddce 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -971,7 +971,7 @@ static void *getMcontextEip(ucontext_t *uc) {
     #endif
 #elif defined(__linux__)
     /* Linux */
-    #if defined(__i386__) || defined(__ILP32__)
+    #if defined(__i386__) || ((defined(__X86_64__) || defined(__x86_64__)) && defined(__ILP32__))
     return (void*) uc->uc_mcontext.gregs[14]; /* Linux 32 */
     #elif defined(__X86_64__) || defined(__x86_64__)
     return (void*) uc->uc_mcontext.gregs[16]; /* Linux 64 */
@@ -1139,7 +1139,7 @@ void logRegisters(ucontext_t *uc) {
 /* Linux */
 #elif defined(__linux__)
     /* Linux x86 */
-    #if defined(__i386__) || defined(__ILP32__)
+    #if defined(__i386__) || ((defined(__X86_64__) || defined(__x86_64__)) && defined(__ILP32__))
     serverLog(LL_WARNING,
     "\n"
     "EAX:%08lx EBX:%08lx ECX:%08lx EDX:%08lx\n"

From 79a7c17176f109b81af171ce564b0354febbfd3f Mon Sep 17 00:00:00 2001
From: Dipankar Achinta <di.punk.car19@gmail.com>
Date: Thu, 26 Nov 2020 03:06:21 +0530
Subject: [PATCH 0821/1098] Fix typo in ae.c file (#7895)

---
 src/ae.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ae.c b/src/ae.c
index e6f8df877..1c3a4e091 100644
--- a/src/ae.c
+++ b/src/ae.c
@@ -352,7 +352,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
     /* Nothing to do? return ASAP */
     if (!(flags & AE_TIME_EVENTS) && !(flags & AE_FILE_EVENTS)) return 0;
 
-    /* Note that we want call select() even if there are no
+    /* Note that we want to call select() even if there are no
      * file events to process as long as we want to process time
      * events, in order to sleep until the next time event is ready
      * to fire. */

From cf88779527c769906f66eb39e9d58e9ee63fcba0 Mon Sep 17 00:00:00 2001
From: kukey <wei.kukey@gmail.com>
Date: Thu, 26 Nov 2020 05:37:54 +0800
Subject: [PATCH 0822/1098] Merge two aeDeleteFileEvent refs into one (#7521)

Merge two aeDeleteFileEvent refs into one
---
 src/connection.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/connection.c b/src/connection.c
index a4610a42e..3e2021fb5 100644
--- a/src/connection.c
+++ b/src/connection.c
@@ -147,8 +147,7 @@ void *connGetPrivateData(connection *conn) {
 /* Close the connection and free resources. */
 static void connSocketClose(connection *conn) {
     if (conn->fd != -1) {
-        aeDeleteFileEvent(server.el,conn->fd,AE_READABLE);
-        aeDeleteFileEvent(server.el,conn->fd,AE_WRITABLE);
+        aeDeleteFileEvent(server.el,conn->fd, AE_READABLE | AE_WRITABLE);
         close(conn->fd);
         conn->fd = -1;
     }

From bbc2c44541fbb50c5d1fc001afe2d2fca3a42eda Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 25 Nov 2020 23:39:01 +0200
Subject: [PATCH 0823/1098] INFO client_recent_max_input_buffer includes argv
 array (#8065)

this metric already includes the argv bytes, like what clientsCronTrackClientsMemUsage does, but it's missing the array itself.

p.s. For the purpose of tracking expensive clients we don't need to include the size of the client struct and the static reply buffer in it.
---
 src/server.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index f30d33555..7054b225e 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1615,7 +1615,8 @@ size_t ClientsPeakMemInput[CLIENTS_PEAK_MEM_USAGE_SLOTS];
 size_t ClientsPeakMemOutput[CLIENTS_PEAK_MEM_USAGE_SLOTS];
 
 int clientsCronTrackExpansiveClients(client *c) {
-    size_t in_usage = sdsZmallocSize(c->querybuf) + c->argv_len_sum;
+    size_t in_usage = sdsZmallocSize(c->querybuf) + c->argv_len_sum +
+	              (c->argv ? zmalloc_size(c->argv) : 0);
     size_t out_usage = getClientOutputBufferMemoryUsage(c);
     int i = server.unixtime % CLIENTS_PEAK_MEM_USAGE_SLOTS;
     int zeroidx = (i+1) % CLIENTS_PEAK_MEM_USAGE_SLOTS;

From cb5eadb33b2ff38e8c1bf31ccf4c864339a96bf4 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 26 Nov 2020 10:58:01 +0200
Subject: [PATCH 0824/1098] bitops limited to proto_max_bulk_len rather than
 512MB (#8096)

we recently did that for SETRANGE and APPEND
---
 src/bitops.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/bitops.c b/src/bitops.c
index 0f45aa00d..5550b60d1 100644
--- a/src/bitops.c
+++ b/src/bitops.c
@@ -36,7 +36,7 @@
 
 /* Count number of bits set in the binary array pointed by 's' and long
  * 'count' bytes. The implementation of this function is required to
- * work with an input string length up to 512 MB. */
+ * work with an input string length up to 512 MB or more (server.proto_max_bulk_len) */
 size_t redisPopcount(void *s, long count) {
     size_t bits = 0;
     unsigned char *p = s;
@@ -405,7 +405,7 @@ void printBits(unsigned char *p, unsigned long count) {
 
 /* This helper function used by GETBIT / SETBIT parses the bit offset argument
  * making sure an error is returned if it is negative or if it overflows
- * Redis 512 MB limit for the string value.
+ * Redis 512 MB limit for the string value or more (server.proto_max_bulk_len).
  *
  * If the 'hash' argument is true, and 'bits is positive, then the command
  * will also parse bit offsets prefixed by "#". In such a case the offset
@@ -428,8 +428,8 @@ int getBitOffsetFromArgument(client *c, robj *o, size_t *offset, int hash, int b
     /* Adjust the offset by 'bits' for #<offset> form. */
     if (usehash) loffset *= bits;
 
-    /* Limit offset to 512MB in bytes */
-    if ((loffset < 0) || ((unsigned long long)loffset >> 3) >= (512*1024*1024))
+    /* Limit offset to server.proto_max_bulk_len (512MB in bytes by default) */
+    if ((loffset < 0) || (loffset >> 3) >= server.proto_max_bulk_len)
     {
         addReplyError(c,err);
         return C_ERR;

From ada2ac9ae2c04422f693c8d989768b97c29b8996 Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Sun, 29 Nov 2020 11:08:47 +0100
Subject: [PATCH 0825/1098] XPENDING with IDLE (#7972)

Used to filter stream pending entries by their idle-time,
useful for XCLAIMing entries that have not been processed
for some time
---
 src/t_stream.c                     | 46 ++++++++++++++++++++++--------
 tests/unit/type/stream-cgroups.tcl | 17 +++++++++++
 2 files changed, 51 insertions(+), 12 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index b25e54a02..e57916200 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -2136,26 +2136,27 @@ cleanup:
     if (ids != static_ids) zfree(ids);
 }
 
-/* XPENDING <key> <group> [<start> <stop> <count> [<consumer>]]
+/* XPENDING <key> <group> [[IDLE <idle>] <start> <stop> <count> [<consumer>]]
  *
  * If start and stop are omitted, the command just outputs information about
  * the amount of pending messages for the key/group pair, together with
  * the minimum and maximum ID of pending messages.
  *
  * If start and stop are provided instead, the pending messages are returned
- * with informations about the current owner, number of deliveries and last
+ * with information about the current owner, number of deliveries and last
  * delivery time and so forth. */
 void xpendingCommand(client *c) {
     int justinfo = c->argc == 3; /* Without the range just outputs general
                                     informations about the PEL. */
     robj *key = c->argv[1];
     robj *groupname = c->argv[2];
-    robj *consumername = (c->argc == 7) ? c->argv[6] : NULL;
+    robj *consumername = NULL;
     streamID startid, endid;
     long long count;
+    long long minidle = 0;
 
-    /* Start and stop, and the consumer, can be omitted. */
-    if (c->argc != 3 && c->argc != 6 && c->argc != 7) {
+    /* Start and stop, and the consumer, can be omitted. Also the IDLE modifier. */
+    if (c->argc != 3 && (c->argc < 6 || c->argc > 9)) {
         addReply(c,shared.syntaxerr);
         return;
     }
@@ -2163,13 +2164,31 @@ void xpendingCommand(client *c) {
     /* Parse start/end/count arguments ASAP if needed, in order to report
      * syntax errors before any other error. */
     if (c->argc >= 6) {
-        if (getLongLongFromObjectOrReply(c,c->argv[5],&count,NULL) == C_ERR)
+        int startidx = 3; /* Without IDLE */
+
+        if (!strcasecmp(c->argv[3]->ptr, "IDLE")) {
+            if (getLongLongFromObjectOrReply(c, c->argv[4], &minidle, NULL) == C_ERR)
+                return;
+            if (c->argc < 8) {
+                /* If IDLE was provided we must have at least 'start end count' */
+                addReply(c,shared.syntaxerr);
+                return;
+            }
+            /* Search for rest of arguments after 'IDLE <idle>' */
+            startidx += 2;
+        }
+        if (getLongLongFromObjectOrReply(c,c->argv[startidx+2],&count,NULL) == C_ERR)
             return;
         if (count < 0) count = 0;
-        if (streamParseIDOrReply(c,c->argv[3],&startid,0) == C_ERR)
+        if (streamParseIDOrReply(c,c->argv[startidx],&startid,0) == C_ERR)
             return;
-        if (streamParseIDOrReply(c,c->argv[4],&endid,UINT64_MAX) == C_ERR)
+        if (streamParseIDOrReply(c,c->argv[startidx+1],&endid,UINT64_MAX) == C_ERR)
             return;
+
+        if (startidx+3 < c->argc) {
+            /* 'consumer' was provided */
+            consumername = c->argv[startidx+3];
+        }
     }
 
     /* Lookup the key and the group inside the stream. */
@@ -2228,9 +2247,7 @@ void xpendingCommand(client *c) {
             setDeferredArrayLen(c,arraylen_ptr,arraylen);
             raxStop(&ri);
         }
-    }
-    /* XPENDING <key> <group> <start> <stop> <count> [<consumer>] variant. */
-    else {
+    } else { /* <start>, <stop> and <count> provided, return actual pending entries (not just info) */
         streamConsumer *consumer = NULL;
         if (consumername) {
             consumer = streamLookupConsumer(group,
@@ -2262,6 +2279,11 @@ void xpendingCommand(client *c) {
         while(count && raxNext(&ri) && memcmp(ri.key,endkey,ri.key_len) <= 0) {
             streamNACK *nack = ri.data;
 
+            if (minidle) {
+                mstime_t this_idle = now - nack->delivery_time;
+                if (this_idle < minidle) continue;
+            }
+
             arraylen++;
             count--;
             addReplyArrayLen(c,4);
@@ -2495,7 +2517,7 @@ void xclaimCommand(client *c) {
              *
              * Note that the nack could be created by FORCE, in this
              * case there was no pre-existing entry and minidle should
-             * be ignored, but in that case nick->consumer is NULL. */
+             * be ignored, but in that case nack->consumer is NULL. */
             if (nack->consumer && minidle) {
                 mstime_t this_idle = now - nack->delivery_time;
                 if (this_idle < minidle) continue;
diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl
index 74ddc98ba..41e2ab979 100644
--- a/tests/unit/type/stream-cgroups.tcl
+++ b/tests/unit/type/stream-cgroups.tcl
@@ -70,6 +70,23 @@ start_server {
         assert {[llength $pending] == 2}
     }
 
+    test {XPENDING only group} {
+        set pending [r XPENDING mystream mygroup]
+        assert {[llength $pending] == 4}
+    }
+
+    test {XPENDING with IDLE} {
+        after 20
+        set pending [r XPENDING mystream mygroup IDLE 99999999 - + 10 client-1]
+        assert {[llength $pending] == 0}
+        set pending [r XPENDING mystream mygroup IDLE 1 - + 10 client-1]
+        assert {[llength $pending] == 2}
+        set pending [r XPENDING mystream mygroup IDLE 99999999 - + 10]
+        assert {[llength $pending] == 0}
+        set pending [r XPENDING mystream mygroup IDLE 1 - + 10]
+        assert {[llength $pending] == 4}
+    }
+
     test {XACK is able to remove items from the client/group PEL} {
         set pending [r XPENDING mystream mygroup - + 10 client-1]
         set id1 [lindex $pending 0 0]

From d322e7baba3065eb3ff03ca90e4a9a4484aaa3a7 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Mon, 30 Nov 2020 15:00:16 +0200
Subject: [PATCH 0826/1098] Adds support for nested args in cli's help (#8074)

See https://github.com/redis/redis-doc/pull/1443

Also allows nameless commands.
---
 utils/generate-command-help.rb | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/utils/generate-command-help.rb b/utils/generate-command-help.rb
index e57acf4b9..929761b24 100755
--- a/utils/generate-command-help.rb
+++ b/utils/generate-command-help.rb
@@ -25,9 +25,15 @@ GROUPS_BY_NAME = Hash[*
 ].freeze
 
 def argument arg
-  name = arg["name"].is_a?(Array) ? arg["name"].join(" ") : arg["name"]
-  name = arg["enum"].join "|" if "enum" == arg["type"]
-  name = arg["command"] + " " + name if arg["command"]
+  if "block" == arg["type"]
+    name = arg["block"].map do |entry|
+      argument entry
+    end.join " "
+  else
+    name = arg["name"].is_a?(Array) ? arg["name"].join(" ") : arg["name"]
+    name = arg["enum"].join "|" if "enum" == arg["type"]
+    name = arg["command"] + (name ? " " + name : "") if arg["command"]
+  end
   if arg["multiple"]
     name = "#{name} [#{name} ...]"
   end

From 04056b767fbcdc1df6fc536467234c04d88bee9d Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Tue, 1 Dec 2020 05:03:53 +0800
Subject: [PATCH 0827/1098] BITOP speedup when or/and output is 0/255, stop
 processing further keys (#8110)

when performing the and operation, if the output is 0, we can jump out of the loop.
when performing an or operation, if the output is 0xff, we can jump out of the loop.
---
 src/bitops.c | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/bitops.c b/src/bitops.c
index 5550b60d1..5e996679b 100644
--- a/src/bitops.c
+++ b/src/bitops.c
@@ -735,12 +735,23 @@ void bitopCommand(client *c) {
             output = (len[0] <= j) ? 0 : src[0][j];
             if (op == BITOP_NOT) output = ~output;
             for (i = 1; i < numkeys; i++) {
+                int skip = 0;
                 byte = (len[i] <= j) ? 0 : src[i][j];
                 switch(op) {
-                case BITOP_AND: output &= byte; break;
-                case BITOP_OR:  output |= byte; break;
+                case BITOP_AND:
+                    output &= byte;
+                    skip = (output == 0);
+                    break;
+                case BITOP_OR:
+                    output |= byte;
+                    skip = (output == 0xff);
+                    break;
                 case BITOP_XOR: output ^= byte; break;
                 }
+
+                if (skip) {
+                    break;
+                }
             }
             res[j] = output;
         }

From c85bf2352d2a3dd04872124d33066403b114a7e7 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuancode@163.com>
Date: Tue, 1 Dec 2020 05:15:14 +0800
Subject: [PATCH 0828/1098] Reset average ttl when empty databases (#8106)

On FLUSHDB or full sync, reset old average TTL stat.
This Stat is incrementally collected by the master over time when it searches for expired keys.
---
 src/db.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/db.c b/src/db.c
index fbbd63748..e51f79eaa 100644
--- a/src/db.c
+++ b/src/db.c
@@ -416,6 +416,9 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(
             dictEmpty(dbarray[j].dict,callback);
             dictEmpty(dbarray[j].expires,callback);
         }
+        /* Because we will start a new database, reset average ttl. */
+        dbarray[j].avg_ttl = 0;
+        dbarray[j].expires_cursor = 0;
     }
 
     /* Post-flush actions */

From c1b1e8c329567e90ec0edcb8c176e9f58a7d7435 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Tue, 1 Dec 2020 14:21:39 +0200
Subject: [PATCH 0829/1098] Adds pub/sub channel patterns to ACL (#7993)

Fixes #7923.

This PR appropriates the special `&` symbol (because `@` and `*` are taken),
followed by a literal value or pattern for describing the Pub/Sub patterns that
an ACL user can interact with. It is similar to the existing key patterns
mechanism in function (additive) and implementation (copy-pasta). It also adds
the allchannels and resetchannels ACL keywords, naturally.

The default user is given allchannels permissions, whereas new users get
whatever is defined by the acl-pubsub-default configuration directive. For
backward compatibility in 6.2, the default of this directive is allchannels but
this is likely to be changed to resetchannels in the next major version for
stronger default security settings.

Unless allchannels is set for the user, channel access permissions are checked
as follows :
* Calls to both PUBLISH and SUBSCRIBE will fail unless a pattern matching the
  argumentative channel name(s) exists for the user.
* Calls to PSUBSCRIBE will fail unless the pattern(s) provided as an argument
  literally exist(s) in the user's list.

Such failures are logged to the ACL log.

Runtime changes to channel permissions for a user with existing subscribing
clients cause said clients to disconnect unless the new permissions permit the
connections to continue. Note, however, that PSUBSCRIBErs' patterns are matched
literally, so given the change bar:* -> b*, pattern subscribers to bar:* will be
disconnected.

Notes/questions:
* UNSUBSCRIBE, PUNSUBSCRIBE and PUBSUB remain unprotected due to lack of reasons
  for touching them.
---
 redis.conf                             |  26 +++
 src/acl.c                              | 227 +++++++++++++++++++++++--
 src/config.c                           |   7 +
 src/multi.c                            |  30 +++-
 src/pubsub.c                           |  29 +++-
 src/redis-cli.c                        |   9 +-
 src/scripting.c                        |  24 ++-
 src/server.c                           |   2 +-
 src/server.h                           |  20 ++-
 tests/unit/acl.tcl                     | 104 +++++++++++
 tests/unit/moduleapi/blockedclient.tcl |   4 +-
 11 files changed, 442 insertions(+), 40 deletions(-)

diff --git a/redis.conf b/redis.conf
index b1c8e4f0c..b11459b26 100644
--- a/redis.conf
+++ b/redis.conf
@@ -747,6 +747,11 @@ replica-priority 100
 #               It is possible to specify multiple patterns.
 #  allkeys      Alias for ~*
 #  resetkeys    Flush the list of allowed keys patterns.
+#  &<pattern>   Add a glob-style pattern of Pub/Sub channels that can be
+#               accessed by the user. It is possible to specify multiple channel
+#               patterns.
+#  allchannels  Alias for &*
+#  resetchannels            Flush the list of allowed channel patterns.
 #  ><password>  Add this password to the list of valid password for the user.
 #               For example >mypass will add "mypass" to the list.
 #               This directive clears the "nopass" flag (see later).
@@ -820,6 +825,27 @@ acllog-max-len 128
 #
 # requirepass foobared
 
+# New users are initialized with restrictive permissions by default, via the
+# equivalent of this ACL rule 'off resetkeys -@all'. Starting with Redis 6.2, it
+# is possible to manage access to Pub/Sub channels with ACL rules as well. The
+# default Pub/Sub channels permission if new users is controlled by the 
+# acl-pubsub-default configuration directive, which accepts one of these values:
+#
+# allchannels: grants access to all Pub/Sub channels
+# resetchannels: revokes access to all Pub/Sub channels
+#
+# To ensure backward compatibility while upgrading Redis 6.0, acl-pubsub-default
+# defaults to the 'allchannels' permission.
+#
+# Future compatibility note: it is very likely that in a future version of Redis
+# the directive's default of 'allchannels' will be changed to 'resetchannels' in
+# order to provide better out-of-the-box Pub/Sub security. Therefore, it is
+# recommended that you explicitly define Pub/Sub permissions for all users
+# rather then rely on implicit default values. Once you've set explicit
+# Pub/Sub for all exisitn users, you should uncomment the following line.
+#
+# acl-pubsub-default resetchannels
+
 # Command renaming (DEPRECATED).
 #
 # ------------------------------------------------------------------------
diff --git a/src/acl.c b/src/acl.c
index e8f9d3724..e9acc11a9 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -92,6 +92,7 @@ struct ACLUserFlag {
     {"on", USER_FLAG_ENABLED},
     {"off", USER_FLAG_DISABLED},
     {"allkeys", USER_FLAG_ALLKEYS},
+    {"allchannels", USER_FLAG_ALLCHANNELS},
     {"allcommands", USER_FLAG_ALLCOMMANDS},
     {"nopass", USER_FLAG_NOPASS},
     {NULL,0} /* Terminator. */
@@ -241,16 +242,20 @@ user *ACLCreateUser(const char *name, size_t namelen) {
     if (raxFind(Users,(unsigned char*)name,namelen) != raxNotFound) return NULL;
     user *u = zmalloc(sizeof(*u));
     u->name = sdsnewlen(name,namelen);
-    u->flags = USER_FLAG_DISABLED;
+    u->flags = USER_FLAG_DISABLED | server.acl_pubusub_default;
     u->allowed_subcommands = NULL;
     u->passwords = listCreate();
     u->patterns = listCreate();
+    u->channels = listCreate();
     listSetMatchMethod(u->passwords,ACLListMatchSds);
     listSetFreeMethod(u->passwords,ACLListFreeSds);
     listSetDupMethod(u->passwords,ACLListDupSds);
     listSetMatchMethod(u->patterns,ACLListMatchSds);
     listSetFreeMethod(u->patterns,ACLListFreeSds);
     listSetDupMethod(u->patterns,ACLListDupSds);
+    listSetMatchMethod(u->channels,ACLListMatchSds);
+    listSetFreeMethod(u->channels,ACLListFreeSds);
+    listSetDupMethod(u->channels,ACLListDupSds);
     memset(u->allowed_commands,0,sizeof(u->allowed_commands));
     raxInsert(Users,(unsigned char*)name,namelen,u,NULL);
     return u;
@@ -279,6 +284,7 @@ void ACLFreeUser(user *u) {
     sdsfree(u->name);
     listRelease(u->passwords);
     listRelease(u->patterns);
+    listRelease(u->channels);
     ACLResetSubcommands(u);
     zfree(u);
 }
@@ -319,8 +325,10 @@ void ACLFreeUserAndKillClients(user *u) {
 void ACLCopyUser(user *dst, user *src) {
     listRelease(dst->passwords);
     listRelease(dst->patterns);
+    listRelease(dst->channels);
     dst->passwords = listDup(src->passwords);
     dst->patterns = listDup(src->patterns);
+    dst->channels = listDup(src->channels);
     memcpy(dst->allowed_commands,src->allowed_commands,
            sizeof(dst->allowed_commands));
     dst->flags = src->flags;
@@ -602,9 +610,10 @@ sds ACLDescribeUser(user *u) {
 
     /* Flags. */
     for (int j = 0; ACLUserFlags[j].flag; j++) {
-        /* Skip the allcommands and allkeys flags because they'll be emitted
-         * later as ~* and +@all. */
+        /* Skip the allcommands, allkeys and allchannels flags because they'll
+         * be emitted later as +@all, ~* and &*. */
         if (ACLUserFlags[j].flag == USER_FLAG_ALLKEYS ||
+            ACLUserFlags[j].flag == USER_FLAG_ALLCHANNELS ||
             ACLUserFlags[j].flag == USER_FLAG_ALLCOMMANDS) continue;
         if (u->flags & ACLUserFlags[j].flag) {
             res = sdscat(res,ACLUserFlags[j].name);
@@ -636,6 +645,19 @@ sds ACLDescribeUser(user *u) {
         }
     }
 
+    /* Pub/sub channel patterns. */
+    if (u->flags & USER_FLAG_ALLCHANNELS) {
+        res = sdscatlen(res,"&* ",3);
+    } else {
+        listRewind(u->channels,&li);
+        while((ln = listNext(&li))) {
+            sds thispat = listNodeValue(ln);
+            res = sdscatlen(res,"&",1);
+            res = sdscatsds(res,thispat);
+            res = sdscatlen(res," ",1);
+        }
+    }
+
     /* Command rules. */
     sds rules = ACLDescribeUserCommandRules(u);
     res = sdscatsds(res,rules);
@@ -681,7 +703,6 @@ void ACLResetSubcommands(user *u) {
     u->allowed_subcommands = NULL;
 }
 
-
 /* Add a subcommand to the list of subcommands for the user 'u' and
  * the command id specified. */
 void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) {
@@ -742,6 +763,12 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) {
  *              It is possible to specify multiple patterns.
  * allkeys      Alias for ~*
  * resetkeys    Flush the list of allowed keys patterns.
+ * &<pattern>   Add a pattern of channels that can be mentioned as part of
+ *              Pub/Sub commands. For instance &* allows all the channels. The
+ *              pattern is a glob-style pattern like the one of PSUBSCRIBE.
+ *              It is possible to specify multiple patterns.
+ * allchannels              Alias for &*
+ * resetchannels            Flush the list of allowed keys patterns.
  * ><password>  Add this password to the list of valid password for the user.
  *              For example >mypass will add "mypass" to the list.
  *              This directive clears the "nopass" flag (see later).
@@ -780,7 +807,7 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) {
  *
  * When an error is returned, errno is set to the following values:
  *
- * EINVAL: The specified opcode is not understood or the key pattern is
+ * EINVAL: The specified opcode is not understood or the key/channel pattern is
  *         invalid (contains non allowed characters).
  * ENOENT: The command name or command category provided with + or - is not
  *         known.
@@ -788,6 +815,8 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) {
  *         fully added.
  * EEXIST: You are adding a key pattern after "*" was already added. This is
  *         almost surely an error on the user side.
+ * EISDIR: You are adding a channel pattern after "*" was already added. This is
+ *         almost surely an error on the user side.
  * ENODEV: The password you are trying to remove from the user does not exist.
  * EBADMSG: The hash you are trying to add is not a valid hash.
  */
@@ -808,6 +837,14 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
     } else if (!strcasecmp(op,"resetkeys")) {
         u->flags &= ~USER_FLAG_ALLKEYS;
         listEmpty(u->patterns);
+    } else if (!strcasecmp(op,"allchannels") ||
+               !strcasecmp(op,"&*"))
+    {
+        u->flags |= USER_FLAG_ALLCHANNELS;
+        listEmpty(u->channels);
+    } else if (!strcasecmp(op,"resetchannels")) {
+        u->flags &= ~USER_FLAG_ALLCHANNELS;
+        listEmpty(u->channels);
     } else if (!strcasecmp(op,"allcommands") ||
                !strcasecmp(op,"+@all"))
     {
@@ -875,12 +912,29 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
         }
         sds newpat = sdsnewlen(op+1,oplen-1);
         listNode *ln = listSearchKey(u->patterns,newpat);
-        /* Avoid re-adding the same pattern multiple times. */
+        /* Avoid re-adding the same key pattern multiple times. */
         if (ln == NULL)
             listAddNodeTail(u->patterns,newpat);
         else
             sdsfree(newpat);
         u->flags &= ~USER_FLAG_ALLKEYS;
+    } else if (op[0] == '&') {
+        if (u->flags & USER_FLAG_ALLCHANNELS) {
+            errno = EISDIR;
+            return C_ERR;
+        }
+        if (ACLStringHasSpaces(op+1,oplen-1)) {
+            errno = EINVAL;
+            return C_ERR;
+        }
+        sds newpat = sdsnewlen(op+1,oplen-1);
+        listNode *ln = listSearchKey(u->channels,newpat);
+        /* Avoid re-adding the same channel pattern multiple times. */
+        if (ln == NULL)
+            listAddNodeTail(u->channels,newpat);
+        else
+            sdsfree(newpat);
+        u->flags &= ~USER_FLAG_ALLCHANNELS;
     } else if (op[0] == '+' && op[1] != '@') {
         if (strchr(op,'|') == NULL) {
             if (ACLLookupCommand(op+1) == NULL) {
@@ -948,6 +1002,7 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
     } else if (!strcasecmp(op,"reset")) {
         serverAssert(ACLSetUser(u,"resetpass",-1) == C_OK);
         serverAssert(ACLSetUser(u,"resetkeys",-1) == C_OK);
+        serverAssert(ACLSetUser(u,"resetchannels",-1) == C_OK);
         serverAssert(ACLSetUser(u,"off",-1) == C_OK);
         serverAssert(ACLSetUser(u,"-@all",-1) == C_OK);
     } else {
@@ -974,6 +1029,11 @@ char *ACLSetUserStringError(void) {
                  "'allkeys' flag) is not valid and does not have any "
                  "effect. Try 'resetkeys' to start with an empty "
                  "list of patterns";
+    else if (errno == EISDIR)
+        errmsg = "Adding a pattern after the * pattern (or the "
+                 "'allchannels' flag) is not valid and does not have any "
+                 "effect. Try 'resetchannels' to start with an empty "
+                 "list of channels";
     else if (errno == ENODEV)
         errmsg = "The password you are trying to remove from the user does "
                  "not exist";
@@ -989,6 +1049,7 @@ void ACLInitDefaultUser(void) {
     DefaultUser = ACLCreateUser("default",7);
     ACLSetUser(DefaultUser,"+@all",-1);
     ACLSetUser(DefaultUser,"~*",-1);
+    ACLSetUser(DefaultUser,"&*",-1);
     ACLSetUser(DefaultUser,"on",-1);
     ACLSetUser(DefaultUser,"nopass",-1);
 }
@@ -1193,6 +1254,119 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) {
     return ACL_OK;
 }
 
+/* Check if the provided channel is whitelisted by the given allowed channels
+ * list. Glob-style pattern matching is employed, unless the literal flag is
+ * set. Returns ACL_OK if access is granted or ACL_DENIED_CHANNEL otherwise. */
+int ACLCheckPubsubChannelPerm(sds channel, list *allowed, int literal) {
+    listIter li;
+    listNode *ln;
+    size_t clen = sdslen(channel);
+    int match = 0;
+
+    listRewind(allowed,&li);
+    while((ln = listNext(&li))) {
+        sds pattern = listNodeValue(ln);
+        size_t plen = sdslen(pattern);
+
+        if ((literal && !sdscmp(pattern,channel)) || 
+            (!literal && stringmatchlen(pattern,plen,channel,clen,0)))
+        {
+            match = 1;
+            break;
+        }
+    }
+    if (!match) {
+        return ACL_DENIED_CHANNEL;
+    }
+    return ACL_OK;
+}
+
+/* Check if the user's existing pub/sub clients violate the ACL pub/sub
+ * permissions specified via the upcoming argument, and kill them if so. */
+void ACLKillPubsubClientsIfNeeded(user *u, list *upcoming) {
+    listIter li, lpi;
+    listNode *ln, *lpn;
+    robj *o;
+    int kill = 0;
+    
+    /* Nothing to kill when the upcoming are a literal super set of the original
+     * permissions. */
+    listRewind(u->channels,&li);
+    while (!kill && ((ln = listNext(&li)) != NULL)) {
+        sds pattern = listNodeValue(ln);
+        kill = (ACLCheckPubsubChannelPerm(pattern,upcoming,1) ==
+                ACL_DENIED_CHANNEL);
+    }
+    if (!kill) return;
+
+    /* Scan all connected clients to find the user's pub/subs. */
+    listRewind(server.clients,&li);
+    while ((ln = listNext(&li)) != NULL) {
+        client *c = listNodeValue(ln);
+        kill = 0;
+
+        if (c->user == u && getClientType(c) == CLIENT_TYPE_PUBSUB) {
+            /* Check for pattern violations. */
+            listRewind(c->pubsub_patterns,&lpi);
+            while (!kill && ((lpn = listNext(&lpi)) != NULL)) {
+                o = lpn->value;
+                kill = (ACLCheckPubsubChannelPerm(o->ptr,upcoming,1) == 
+                        ACL_DENIED_CHANNEL);
+            }
+            /* Check for channel violations. */
+            if (!kill) {
+                dictIterator *di = dictGetIterator(c->pubsub_channels);
+                dictEntry *de;                
+                while (!kill && ((de = dictNext(di)) != NULL)) {
+                    o = dictGetKey(de);
+                    kill = (ACLCheckPubsubChannelPerm(o->ptr,upcoming,0) ==
+                            ACL_DENIED_CHANNEL);
+                }
+                dictReleaseIterator(di);
+            }
+
+            /* Kill it. */
+            if (kill) {
+                freeClient(c);
+            }
+        }
+    }
+}
+
+/* Check if the pub/sub channels of the command, that's ready to be executed in
+ * the client 'c', can be executed by this client according to the ACLs channels
+ * associated to the client user c->user.
+ * 
+ * idx and count are the index and count of channel arguments from the
+ * command. The literal argument controls whether the user's ACL channels are 
+ * evaluated as literal values or matched as glob-like patterns.
+ *
+ * If the user can execute the command ACL_OK is returned, otherwise 
+ * ACL_DENIED_CHANNEL. */
+int ACLCheckPubsubPerm(client *c, int idx, int count, int literal, int *idxptr) {
+    user *u = c->user;
+
+    /* If there is no associated user, the connection can run anything. */
+    if (u == NULL) return ACL_OK;
+
+    /* Check if the user can access the channels mentioned in the command's
+     * arguments. */
+    if (!(c->user->flags & USER_FLAG_ALLCHANNELS)) {
+        for (int j = idx; j < idx+count; j++) {
+            if (ACLCheckPubsubChannelPerm(c->argv[j]->ptr,u->channels,literal)
+                != ACL_OK) {
+                if (idxptr) *idxptr = j;
+                return ACL_DENIED_CHANNEL;
+            }
+        }
+    }
+
+    /* If we survived all the above checks, the user can execute the
+     * command. */
+    return ACL_OK;
+
+}
+
 /* =============================================================================
  * ACL loading / saving functions
  * ==========================================================================*/
@@ -1608,13 +1782,13 @@ void ACLFreeLogEntry(void *leptr) {
  * the log entry instead of creating many entries for very similar ACL
  * rules issues.
  *
- * The keypos argument is only used when the reason is ACL_DENIED_KEY, since
- * it allows the function to log the key name that caused the problem.
- * Similarly the username is only passed when we failed to authenticate the
- * user with AUTH or HELLO, for the ACL_DENIED_AUTH reason. Otherwise
- * it will just be NULL.
+ * The argpos argument is used when the reason is ACL_DENIED_KEY or 
+ * ACL_DENIED_CHANNEL, since it allows the function to log the key or channel
+ * name that caused the problem. Similarly the username is only passed when we
+ * failed to authenticate the user with AUTH or HELLO, for the ACL_DENIED_AUTH
+ * reason. Otherwise it will just be NULL.
  */
-void addACLLogEntry(client *c, int reason, int keypos, sds username) {
+void addACLLogEntry(client *c, int reason, int argpos, sds username) {
     /* Create a new entry. */
     struct ACLLogEntry *le = zmalloc(sizeof(*le));
     le->count = 1;
@@ -1624,8 +1798,9 @@ void addACLLogEntry(client *c, int reason, int keypos, sds username) {
 
     switch(reason) {
     case ACL_DENIED_CMD: le->object = sdsnew(c->cmd->name); break;
-    case ACL_DENIED_KEY: le->object = sdsnew(c->argv[keypos]->ptr); break;
-    case ACL_DENIED_AUTH: le->object = sdsnew(c->argv[0]->ptr); break;
+    case ACL_DENIED_KEY: le->object = sdsdup(c->argv[argpos]->ptr); break;
+    case ACL_DENIED_CHANNEL: le->object = sdsdup(c->argv[argpos]->ptr); break;
+    case ACL_DENIED_AUTH: le->object = sdsdup(c->argv[0]->ptr); break;
     default: le->object = sdsempty();
     }
 
@@ -1733,6 +1908,11 @@ void aclCommand(client *c) {
             }
         }
 
+        /* Existing pub/sub clients authenticated with the user may need to be
+         * disconnected if (some of) their channel permissions were revoked. */
+        if (u && !(tempu->flags & USER_FLAG_ALLCHANNELS))
+            ACLKillPubsubClientsIfNeeded(u,tempu->channels);
+
         /* Overwrite the user with the temporary user we modified above. */
         if (!u) u = ACLCreateUser(username,sdslen(username));
         serverAssert(u != NULL);
@@ -1768,7 +1948,7 @@ void aclCommand(client *c) {
             return;
         }
 
-        addReplyMapLen(c,4);
+        addReplyMapLen(c,5);
 
         /* Flags */
         addReplyBulkCString(c,"flags");
@@ -1813,6 +1993,22 @@ void aclCommand(client *c) {
                 addReplyBulkCBuffer(c,thispat,sdslen(thispat));
             }
         }
+
+        /* Pub/sub patterns */
+        addReplyBulkCString(c,"channels");
+        if (u->flags & USER_FLAG_ALLCHANNELS) {
+            addReplyArrayLen(c,1);
+            addReplyBulkCBuffer(c,"*",1);
+        } else {
+            addReplyArrayLen(c,listLength(u->channels));
+            listIter li;
+            listNode *ln;
+            listRewind(u->channels,&li);
+            while((ln = listNext(&li))) {
+                sds thispat = listNodeValue(ln);
+                addReplyBulkCBuffer(c,thispat,sdslen(thispat));
+            }
+        }
     } else if ((!strcasecmp(sub,"list") || !strcasecmp(sub,"users")) &&
                c->argc == 2)
     {
@@ -1950,6 +2146,7 @@ void aclCommand(client *c) {
             switch(le->reason) {
             case ACL_DENIED_CMD: reasonstr="command"; break;
             case ACL_DENIED_KEY: reasonstr="key"; break;
+            case ACL_DENIED_CHANNEL: reasonstr="channel"; break;
             case ACL_DENIED_AUTH: reasonstr="auth"; break;
             default: reasonstr="unknown";
             }
diff --git a/src/config.c b/src/config.c
index b4fd9bee9..8dc241ed7 100644
--- a/src/config.c
+++ b/src/config.c
@@ -113,6 +113,12 @@ configEnum oom_score_adj_enum[] = {
     {NULL, 0}
 };
 
+configEnum acl_pubsub_default_enum[] = {
+    {"allchannels", USER_FLAG_ALLCHANNELS},
+    {"resetchannels", 0},
+    {NULL, 0}
+};
+
 /* Output buffer limits presets. */
 clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = {
     {0, 0, 0}, /* normal */
@@ -2352,6 +2358,7 @@ standardConfig configs[] = {
     createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, server.maxmemory_policy, MAXMEMORY_NO_EVICTION, NULL, NULL),
     createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, AOF_FSYNC_EVERYSEC, NULL, NULL),
     createEnumConfig("oom-score-adj", NULL, MODIFIABLE_CONFIG, oom_score_adj_enum, server.oom_score_adj, OOM_SCORE_ADJ_NO, NULL, updateOOMScoreAdj),
+    createEnumConfig("acl-pubsub-default", NULL, MODIFIABLE_CONFIG, acl_pubsub_default_enum, server.acl_pubusub_default, USER_FLAG_ALLCHANNELS, NULL, NULL),
 
     /* Integer configs */
     createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, 16, INTEGER_CONFIG, NULL, NULL),
diff --git a/src/multi.c b/src/multi.c
index 7790b0223..c519828c4 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -198,18 +198,34 @@ void execCommand(client *c) {
             must_propagate = 1;
         }
 
-        int acl_keypos;
-        int acl_retval = ACLCheckCommandPerm(c,&acl_keypos);
+        /* ACL permissions are also checked at the time of execution in case
+         * they were changed after the commands were ququed. */
+        int acl_errpos;
+        int acl_retval = ACLCheckCommandPerm(c,&acl_errpos);
+        if (acl_retval == ACL_OK && c->cmd->proc == publishCommand)
+            acl_retval = ACLCheckPubsubPerm(c,1,1,0,&acl_errpos);
         if (acl_retval != ACL_OK) {
-            addACLLogEntry(c,acl_retval,acl_keypos,NULL);
+            char *reason;
+            switch (acl_retval) {
+            case ACL_DENIED_CMD:
+                reason = "no permission to execute the command or subcommand";
+                break;
+            case ACL_DENIED_KEY:
+                reason = "no permission to touch the specified keys";
+                break;
+            case ACL_DENIED_CHANNEL:
+                reason = "no permission to publish to the specified channel";
+                break;
+            default:
+                reason = "no permission";
+                break;
+            }
+            addACLLogEntry(c,acl_retval,acl_errpos,NULL);
             addReplyErrorFormat(c,
                 "-NOPERM ACLs rules changed between the moment the "
                 "transaction was accumulated and the EXEC call. "
                 "This command is no longer allowed for the "
-                "following reason: %s",
-                (acl_retval == ACL_DENIED_CMD) ?
-                "no permission to execute the command or subcommand" :
-                "no permission to touch the specified keys");
+                "following reason: %s", reason);
         } else {
             call(c,server.loading ? CMD_CALL_NONE : CMD_CALL_FULL);
             serverAssert((c->flags & CLIENT_BLOCKED) == 0);
diff --git a/src/pubsub.c b/src/pubsub.c
index b7f7d9673..9e41464bf 100644
--- a/src/pubsub.c
+++ b/src/pubsub.c
@@ -353,13 +353,29 @@ int pubsubPublishMessage(robj *channel, robj *message) {
     return receivers;
 }
 
+/* This wraps handling ACL channel permissions for the given client. */
+int pubsubCheckACLPermissionsOrReply(client *c, int idx, int count, int literal) {
+    /* Check if the user can run the command according to the current
+     * ACLs. */
+    int acl_chanpos;
+    int acl_retval = ACLCheckPubsubPerm(c,idx,count,literal,&acl_chanpos);
+    if (acl_retval == ACL_DENIED_CHANNEL) {
+        addACLLogEntry(c,acl_retval,acl_chanpos,NULL);
+        addReplyError(c,
+            "-NOPERM this user has no permissions to access "
+            "one of the channels used as arguments");
+    }
+    return acl_retval;
+}
+
 /*-----------------------------------------------------------------------------
  * Pubsub commands implementation
  *----------------------------------------------------------------------------*/
 
+/* SUBSCRIBE channel [channel ...] */
 void subscribeCommand(client *c) {
     int j;
-
+    if (pubsubCheckACLPermissionsOrReply(c,1,c->argc-1,0) != ACL_OK) return;
     if ((c->flags & CLIENT_DENY_BLOCKING) && !(c->flags & CLIENT_MULTI)) {
         /**
          * A client that has CLIENT_DENY_BLOCKING flag on
@@ -368,7 +384,7 @@ void subscribeCommand(client *c) {
          * Notice that we have a special treatment for multi because of
          * backword compatibility
          */
-        addReplyError(c, "subscribe is not allow on DENY BLOCKING client");
+        addReplyError(c, "SUBSCRIBE isn't allowed for a DENY BLOCKING client");
         return;
     }
 
@@ -377,6 +393,7 @@ void subscribeCommand(client *c) {
     c->flags |= CLIENT_PUBSUB;
 }
 
+/* UNSUBSCRIBE [channel [channel ...]] */
 void unsubscribeCommand(client *c) {
     if (c->argc == 1) {
         pubsubUnsubscribeAllChannels(c,1);
@@ -389,9 +406,10 @@ void unsubscribeCommand(client *c) {
     if (clientSubscriptionsCount(c) == 0) c->flags &= ~CLIENT_PUBSUB;
 }
 
+/* PSUBSCRIBE pattern [pattern ...] */
 void psubscribeCommand(client *c) {
     int j;
-
+    if (pubsubCheckACLPermissionsOrReply(c,1,c->argc-1,1) != ACL_OK) return;
     if ((c->flags & CLIENT_DENY_BLOCKING) && !(c->flags & CLIENT_MULTI)) {
         /**
          * A client that has CLIENT_DENY_BLOCKING flag on
@@ -400,7 +418,7 @@ void psubscribeCommand(client *c) {
          * Notice that we have a special treatment for multi because of
          * backword compatibility
          */
-        addReplyError(c, "PSUBSCRIBE is not allowed for DENY BLOCKING client");
+        addReplyError(c, "PSUBSCRIBE isn't allowed for a DENY BLOCKING client");
         return;
     }
 
@@ -409,6 +427,7 @@ void psubscribeCommand(client *c) {
     c->flags |= CLIENT_PUBSUB;
 }
 
+/* PUNSUBSCRIBE [pattern [pattern ...]] */
 void punsubscribeCommand(client *c) {
     if (c->argc == 1) {
         pubsubUnsubscribeAllPatterns(c,1);
@@ -421,7 +440,9 @@ void punsubscribeCommand(client *c) {
     if (clientSubscriptionsCount(c) == 0) c->flags &= ~CLIENT_PUBSUB;
 }
 
+/* PUBLISH <channel> <message> */
 void publishCommand(client *c) {
+    if (pubsubCheckACLPermissionsOrReply(c,1,1,0) != ACL_OK) return;
     int receivers = pubsubPublishMessage(c->argv[1],c->argv[2]);
     if (server.cluster_enabled)
         clusterPropagatePublish(c->argv[1],c->argv[2]);
diff --git a/src/redis-cli.c b/src/redis-cli.c
index c09a6b798..cd10d22e4 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -1363,9 +1363,16 @@ static int cliSendCommand(int argc, char **argv, long repeat) {
             /* Unset our default PUSH handler so this works in RESP2/RESP3 */
             redisSetPushCallback(context, NULL);
 
-            while (1) {
+            while (config.pubsub_mode) {
                 if (cliReadReply(output_raw) != REDIS_OK) exit(1);
+                if (config.last_cmd_type == REDIS_REPLY_ERROR) {
+                    if (config.push_output) {
+                        redisSetPushCallback(context, cliPushHandler);
+                    }
+                    config.pubsub_mode = 0;
+                }
             }
+            continue;
         }
 
         if (config.slave_mode) {
diff --git a/src/scripting.c b/src/scripting.c
index 732cbfc62..3ed3e2f64 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -606,17 +606,31 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) {
     }
 
     /* Check the ACLs. */
-    int acl_keypos;
-    int acl_retval = ACLCheckCommandPerm(c,&acl_keypos);
+    int acl_errpos;
+    int acl_retval = ACLCheckCommandPerm(c,&acl_errpos);
+    if (acl_retval == ACL_OK && c->cmd->proc == publishCommand)
+        acl_retval = ACLCheckPubsubPerm(c,1,1,0,&acl_errpos);
     if (acl_retval != ACL_OK) {
-        addACLLogEntry(c,acl_retval,acl_keypos,NULL);
-        if (acl_retval == ACL_DENIED_CMD)
+        addACLLogEntry(c,acl_retval,acl_errpos,NULL);
+        switch (acl_retval) {
+        case ACL_DENIED_CMD:
             luaPushError(lua, "The user executing the script can't run this "
                               "command or subcommand");
-        else
+            break;
+        case ACL_DENIED_KEY:
             luaPushError(lua, "The user executing the script can't access "
                               "at least one of the keys mentioned in the "
                               "command arguments");
+            break;
+        case ACL_DENIED_AUTH:
+            luaPushError(lua, "The user executing the script can't publish "
+                              "to the channel mentioned in the command");
+            break;
+        default:
+            luaPushError(lua, "The user executing the script is lacking the "
+                              "permissions for the command");
+            break;
+        }
         goto cleanup;
     }
 
diff --git a/src/server.c b/src/server.c
index 7054b225e..16e7d3fba 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4899,7 +4899,7 @@ void monitorCommand(client *c) {
         /**
          * A client that has CLIENT_DENY_BLOCKING flag on
          * expects a reply per command and so can't execute MONITOR. */
-        addReplyError(c, "MONITOR is not allowed for DENY BLOCKING client");
+        addReplyError(c, "MONITOR isn't allowed for DENY BLOCKING client");
         return;
     }
 
diff --git a/src/server.h b/src/server.h
index 4690151c0..a46954096 100644
--- a/src/server.h
+++ b/src/server.h
@@ -773,6 +773,8 @@ typedef struct readyList {
                                            no AUTH is needed, and every
                                            connection is immediately
                                            authenticated. */
+#define USER_FLAG_ALLCHANNELS (1<<5)    /* The user can mention any Pub/Sub
+                                           channel. */
 typedef struct {
     sds name;       /* The username as an SDS string. */
     uint64_t flags; /* See USER_FLAG_* */
@@ -796,6 +798,10 @@ typedef struct {
     list *patterns;  /* A list of allowed key patterns. If this field is NULL
                         the user cannot mention any key in a command, unless
                         the flag ALLKEYS is set in the user. */
+    list *channels;  /* A list of allowed Pub/Sub channel patterns. If this
+                        field is NULL the user cannot mention any channel in a
+                        `PUBLISH` or [P][UNSUSBSCRIBE] command, unless the flag
+                        ALLCHANNELS is set in the user. */
 } user;
 
 /* With multiplexing we need to take per-client state.
@@ -1488,11 +1494,12 @@ struct redisServer {
     long long latency_monitor_threshold;
     dict *latency_events;
     /* ACLs */
-    char *acl_filename;     /* ACL Users file. NULL if not configured. */
+    char *acl_filename;           /* ACL Users file. NULL if not configured. */
     unsigned long acllog_max_len; /* Maximum length of the ACL LOG list. */
-    sds requirepass;        /* Remember the cleartext password set with the
-                               old "requirepass" directive for backward
-                               compatibility with Redis <= 5. */
+    sds requirepass;              /* Remember the cleartext password set with
+                                     the old "requirepass" directive for
+                                     backward compatibility with Redis <= 5. */
+    int acl_pubusub_default;      /* Default ACL pub/sub channels flag */
     /* Assert & bug reporting */
     int watchdog_period;  /* Software watchdog period in ms. 0 = off */
     /* System hardware info */
@@ -1961,17 +1968,19 @@ void sendChildCOWInfo(int ptype, char *pname);
 extern rax *Users;
 extern user *DefaultUser;
 void ACLInit(void);
-/* Return values for ACLCheckCommandPerm(). */
+/* Return values for ACLCheckCommandPerm() and ACLCheckPubsubPerm(). */
 #define ACL_OK 0
 #define ACL_DENIED_CMD 1
 #define ACL_DENIED_KEY 2
 #define ACL_DENIED_AUTH 3 /* Only used for ACL LOG entries. */
+#define ACL_DENIED_CHANNEL 4 /* Only used for pub/sub commands */
 int ACLCheckUserCredentials(robj *username, robj *password);
 int ACLAuthenticateUser(client *c, robj *username, robj *password);
 unsigned long ACLGetCommandID(const char *cmdname);
 void ACLClearCommandID(void);
 user *ACLGetUserByName(const char *name, size_t namelen);
 int ACLCheckCommandPerm(client *c, int *keyidxptr);
+int ACLCheckPubsubPerm(client *c, int idx, int count, int literal, int *idxptr);
 int ACLSetUser(user *u, const char *op, ssize_t oplen);
 sds ACLDefaultUserFirstPassword(void);
 uint64_t ACLGetCommandCategoryFlagByName(const char *name);
@@ -2096,6 +2105,7 @@ struct redisMemOverhead *getMemoryOverheadData(void);
 void freeMemoryOverheadData(struct redisMemOverhead *mh);
 void checkChildrenDone(void);
 int setOOMScoreAdj(int process_class);
+void rejectCommandFormat(client *c, const char *fmt, ...);
 
 #define RESTART_SERVER_NONE 0
 #define RESTART_SERVER_GRACEFULLY (1<<0)     /* Do proper shutdown. */
diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl
index d35a012f4..856bf6196 100644
--- a/tests/unit/acl.tcl
+++ b/tests/unit/acl.tcl
@@ -81,6 +81,100 @@ start_server {tags {"acl"}} {
         set e
     } {*NOPERM*key*}
 
+    test {By default users are able to publish to any channel} {
+        r ACL setuser psuser on >pspass +acl +client +@pubsub
+        r AUTH psuser pspass
+        r PUBLISH foo bar
+    } {0}
+
+    test {By default users are able to subscribe to any channel} {
+        set rd [redis_deferring_client]
+        $rd AUTH psuser pspass
+        $rd read
+        $rd SUBSCRIBE foo
+        assert_match {subscribe foo 1} [$rd read]
+        $rd close
+    } {0}
+
+    test {By default users are able to subscribe to any pattern} {
+        set rd [redis_deferring_client]
+        $rd AUTH psuser pspass
+        $rd read
+        $rd PSUBSCRIBE bar*
+        assert_match {psubscribe bar\* 1} [$rd read]
+        $rd close
+    } {0}
+
+    test {It's possible to allow publishing to a subset of channels} {
+        r ACL setuser psuser resetchannels &foo:1 &bar:*
+        assert_equal {0} [r PUBLISH foo:1 somemessage]
+        assert_equal {0} [r PUBLISH bar:2 anothermessage]
+        catch {r PUBLISH zap:3 nosuchmessage} e
+        set e
+    } {*NOPERM*channel*}
+
+    test {It's possible to allow subscribing to a subset of channels} {
+        set rd [redis_deferring_client]
+        $rd AUTH psuser pspass
+        $rd read
+        $rd SUBSCRIBE foo:1
+        assert_match {subscribe foo:1 1} [$rd read]
+        $rd SUBSCRIBE bar:2
+        assert_match {subscribe bar:2 2} [$rd read]
+        $rd SUBSCRIBE zap:3
+        catch {$rd read} e
+        set e
+    } {*NOPERM*channel*}
+
+    test {It's possible to allow subscribing to a subset of channel patterns} {
+        set rd [redis_deferring_client]
+        $rd AUTH psuser pspass
+        $rd read
+        $rd PSUBSCRIBE foo:1
+        assert_match {psubscribe foo:1 1} [$rd read]
+        $rd PSUBSCRIBE bar:*
+        assert_match {psubscribe bar:\* 2} [$rd read]
+        $rd PSUBSCRIBE bar:baz
+        catch {$rd read} e
+        set e
+    } {*NOPERM*channel*}
+    
+    test {Subscribers are killed when revoked of channel permission} {
+        set rd [redis_deferring_client]
+        r ACL setuser psuser resetchannels &foo:1
+        $rd AUTH psuser pspass
+        $rd CLIENT SETNAME deathrow
+        $rd SUBSCRIBE foo:1
+        r ACL setuser psuser resetchannels
+        assert_no_match {*deathrow*} [r CLIENT LIST]
+        $rd close
+    } {0}
+
+    test {Subscribers are killed when revoked of pattern permission} {
+        set rd [redis_deferring_client]
+        r ACL setuser psuser resetchannels &bar:*
+        $rd AUTH psuser pspass
+        $rd CLIENT SETNAME deathrow
+        $rd PSUBSCRIBE bar:*
+        r ACL setuser psuser resetchannels
+        assert_no_match {*deathrow*} [r CLIENT LIST]
+        $rd close
+    } {0}
+
+    test {Subscribers are pardoned if literal permissions are retained and/or gaining allchannels} {
+        set rd [redis_deferring_client]
+        r ACL setuser psuser resetchannels &foo:1 &bar:*
+        $rd AUTH psuser pspass
+        $rd CLIENT SETNAME pardoned
+        $rd SUBSCRIBE foo:1
+        $rd PSUBSCRIBE bar:*
+        r ACL setuser psuser resetchannels &foo:1 &bar:* &baz:qaz &zoo:*
+        assert_match {*pardoned*} [r CLIENT LIST]
+        r ACL setuser psuser allchannels
+        assert_match {*pardoned*} [r CLIENT LIST]
+        $rd close
+    } {0}
+
     test {Users can be configured to authenticate with any password} {
         r ACL setuser newuser nopass
         r AUTH newuser zipzapblabla
@@ -166,6 +260,7 @@ start_server {tags {"acl"}} {
         r ACL LOG RESET
         r ACL setuser antirez >foo on +set ~object:1234
         r ACL setuser antirez +eval +multi +exec
+        r ACL setuser antirez resetchannels +publish
         r AUTH antirez foo
         catch {r GET foo}
         r AUTH default ""
@@ -195,6 +290,15 @@ start_server {tags {"acl"}} {
         assert {[dict get $entry object] eq {somekeynotallowed}}
     }
 
+    test {ACL LOG is able to log channel access violations and channel name} {
+        r AUTH antirez foo
+        catch {r PUBLISH somechannelnotallowed nullmsg}
+        r AUTH default ""
+        set entry [lindex [r ACL LOG] 0]
+        assert {[dict get $entry reason] eq {channel}}
+        assert {[dict get $entry object] eq {somechannelnotallowed}}
+    }
+
     test {ACL LOG RESET is able to flush the entries in the log} {
         r ACL LOG RESET
         assert {[llength [r ACL LOG]] == 0}
diff --git a/tests/unit/moduleapi/blockedclient.tcl b/tests/unit/moduleapi/blockedclient.tcl
index 33ecfff22..60039659e 100644
--- a/tests/unit/moduleapi/blockedclient.tcl
+++ b/tests/unit/moduleapi/blockedclient.tcl
@@ -63,7 +63,7 @@ start_server {tags {"modules"}} {
             r do_rm_call monitor
         } e
         set e
-    } {*MONITOR is not allow*}
+    } {*ERR*DENY BLOCKING*}
 
     test {subscribe disallow inside RM_Call} {
         set e {}
@@ -71,5 +71,5 @@ start_server {tags {"modules"}} {
             r do_rm_call subscribe x
         } e
         set e
-    } {*subscribe is not allow*}
+    } {*ERR*DENY BLOCKING*}
 }

From 3ba2281f96098691d9d672ce8b484207379b09ee Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Wed, 2 Dec 2020 03:41:26 +0800
Subject: [PATCH 0830/1098] Improve dbid range check for SELECT, MOVE, COPY
 (#8085)

SELECT used to read the index into a `long` variable, and then pass it to a function
that takes an `int`, possibly causing an overflow before the range check.

Now all these commands use better and cleaner range check, and that also results in
a slight change of the error response in case of an invalid database index.

SELECT:
in the past it would have returned either `-ERR invalid DB index` (if not a number),
or `-ERR DB index is out of range` (if not between 1..16 or alike).
now it'll return either `-ERR value is out of range` (if not a number), or
`-ERR value is out of range, value must between -2147483648 and 2147483647`
(if not in the range for an int), or `-ERR DB index is out of range`
(if not between 0..16 or alike)


MOVE:
in the past it would only fail with `-ERR index out of range` no matter the reason.
now return the same errors as the new ones for SELECT mentioned above.
(i.e. unlike for SELECT even for a value like 17 we changed the error message)

COPY:
doesn't really matter how it behaved in the past (new command), new behavior is
like the above two.
---
 src/db.c                | 33 ++++++++++++++++-----------------
 src/object.c            | 10 ++++++++++
 src/server.h            |  1 +
 tests/unit/keyspace.tcl |  4 ++--
 4 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/src/db.c b/src/db.c
index e51f79eaa..32baf0974 100644
--- a/src/db.c
+++ b/src/db.c
@@ -600,10 +600,9 @@ void existsCommand(client *c) {
 }
 
 void selectCommand(client *c) {
-    long id;
+    int id;
 
-    if (getLongFromObjectOrReply(c, c->argv[1], &id,
-        "invalid DB index") != C_OK)
+    if (getIntFromObjectOrReply(c, c->argv[1], &id, NULL) != C_OK)
         return;
 
     if (server.cluster_enabled && id != 0) {
@@ -1022,8 +1021,8 @@ void renamenxCommand(client *c) {
 void moveCommand(client *c) {
     robj *o;
     redisDb *src, *dst;
-    int srcid;
-    long long dbid, expire;
+    int srcid, dbid;
+    long long expire;
 
     if (server.cluster_enabled) {
         addReplyError(c,"MOVE is not allowed in cluster mode");
@@ -1034,11 +1033,11 @@ void moveCommand(client *c) {
     src = c->db;
     srcid = c->db->id;
 
-    if (getLongLongFromObject(c->argv[2],&dbid) == C_ERR ||
-        dbid < INT_MIN || dbid > INT_MAX ||
-        selectDb(c,dbid) == C_ERR)
-    {
-        addReply(c,shared.outofrangeerr);
+    if (getIntFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK)
+        return;
+
+    if (selectDb(c,dbid) == C_ERR) {
+        addReplyError(c,"DB index is out of range");
         return;
     }
     dst = c->db;
@@ -1084,8 +1083,8 @@ void moveCommand(client *c) {
 void copyCommand(client *c) {
     robj *o;
     redisDb *src, *dst;
-    int srcid;
-    long long dbid, expire;
+    int srcid, dbid;
+    long long expire;
     int j, replace = 0, delete = 0;
 
     /* Obtain source and target DB pointers 
@@ -1100,11 +1099,11 @@ void copyCommand(client *c) {
         if (!strcasecmp(c->argv[j]->ptr,"replace")) {
             replace = 1;
         } else if (!strcasecmp(c->argv[j]->ptr, "db") && additional >= 1) {
-            if (getLongLongFromObject(c->argv[j+1], &dbid) == C_ERR ||
-                dbid < INT_MIN || dbid > INT_MAX ||
-                selectDb(c, dbid) == C_ERR)
-            {
-                addReplyError(c,"invalid DB index");
+            if (getIntFromObjectOrReply(c, c->argv[j+1], &dbid, NULL) != C_OK)
+                return;
+
+            if (selectDb(c, dbid) == C_ERR) {
+                addReplyError(c,"DB index is out of range");
                 return;
             }
             dst = c->db;
diff --git a/src/object.c b/src/object.c
index d35ba0eaf..71eceb6d6 100644
--- a/src/object.c
+++ b/src/object.c
@@ -747,6 +747,16 @@ int getPositiveLongFromObjectOrReply(client *c, robj *o, long *target, const cha
     return getRangeLongFromObjectOrReply(c, o, 0, LONG_MAX, target, msg);
 }
 
+int getIntFromObjectOrReply(client *c, robj *o, int *target, const char *msg) {
+    long value;
+
+    if (getRangeLongFromObjectOrReply(c, o, INT_MIN, INT_MAX, &value, msg) != C_OK)
+        return C_ERR;
+
+    *target = value;
+    return C_OK;
+}
+
 char *strEncoding(int encoding) {
     switch(encoding) {
     case OBJ_ENCODING_RAW: return "raw";
diff --git a/src/server.h b/src/server.h
index a46954096..0bcbeabfb 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1872,6 +1872,7 @@ int getDoubleFromObject(const robj *o, double *target);
 int getLongLongFromObject(robj *o, long long *target);
 int getLongDoubleFromObject(robj *o, long double *target);
 int getLongDoubleFromObjectOrReply(client *c, robj *o, long double *target, const char *msg);
+int getIntFromObjectOrReply(client *c, robj *o, int *target, const char *msg);
 char *strEncoding(int encoding);
 int compareStringObjects(robj *a, robj *b);
 int collateStringObjects(robj *a, robj *b);
diff --git a/tests/unit/keyspace.tcl b/tests/unit/keyspace.tcl
index a74651db9..c69741814 100644
--- a/tests/unit/keyspace.tcl
+++ b/tests/unit/keyspace.tcl
@@ -217,7 +217,7 @@ start_server {tags {"keyspace"}} {
         r set mykey foobar
         catch {r copy mykey mynewkey DB notanumber} e
         set e
-    } {*ERR*invalid DB index}
+    } {ERR value is not an integer or out of range}
 
     test {COPY can copy key expire metadata as well} {
         r set mykey foobar ex 100
@@ -398,7 +398,7 @@ start_server {tags {"keyspace"}} {
         r set mykey hello
         catch {r move mykey notanumber} e
         set e
-    } {*ERR*index out of range}
+    } {ERR value is not an integer or out of range}
 
     test {MOVE can move key expire metadata as well} {
         r select 10

From 69b7113bb52360f330d184b5b270625d35902ad6 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Tue, 1 Dec 2020 11:46:45 -0800
Subject: [PATCH 0831/1098] Getset fix (#8118)

* Fixed SET GET executing on wrong type

Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
---
 src/t_string.c             | 2 +-
 tests/unit/type/string.tcl | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/t_string.c b/src/t_string.c
index e8b37663d..3ecc473bd 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -90,7 +90,7 @@ void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire,
     }
 
     if (flags & OBJ_SET_GET) {
-        getGenericCommand(c);
+        if (getGenericCommand(c) == C_ERR) return;
     }
 
     genericSetKey(c,c->db,key,val,flags & OBJ_SET_KEEPTTL,1);
diff --git a/tests/unit/type/string.tcl b/tests/unit/type/string.tcl
index 5058acec4..16e961623 100644
--- a/tests/unit/type/string.tcl
+++ b/tests/unit/type/string.tcl
@@ -415,6 +415,14 @@ start_server {tags {"string"}} {
       list $err1 $err2
     } {*syntax err* *syntax err*}
 
+    test {Extended SET GET with incorrect type should result in wrong type error} {
+      r del foo
+      r rpush foo waffle
+      catch {r set foo bar GET} err1
+      assert_equal "waffle" [r rpop foo]
+      set err1
+    } {*WRONGTYPE*}
+
     test {Extended SET EX option} {
         r del foo
         r set foo bar ex 10

From 7885faf18b776eaed4a7ddcf8fb8bce699820eea Mon Sep 17 00:00:00 2001
From: luhuachao <14054766+chimney-lee@users.noreply.github.com>
Date: Wed, 2 Dec 2020 19:17:25 +0800
Subject: [PATCH 0832/1098] Modify help msg PING_BULK to PING_MBULK in
 benchmark (#8109)

As described in redis-benchamrk help message 'The test names are the same as the ones produced as output.', In redis-benchmark output, we can only see PING_BULK, but the cmd `redis-benchmark -t ping_bulk` is not supported. We  have to run it with ping_mbulk which is not user friendly.
---
 src/redis-benchmark.c                 | 2 +-
 tests/integration/redis-benchmark.tcl | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 0ec9cb1eb..d8c62daa3 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -1829,7 +1829,7 @@ int main(int argc, const char **argv) {
 
         if (test_is_selected("ping_mbulk") || test_is_selected("ping")) {
             len = redisFormatCommand(&cmd,"PING");
-            benchmark("PING_BULK",cmd,len);
+            benchmark("PING_MBULK",cmd,len);
             free(cmd);
         }
 
diff --git a/tests/integration/redis-benchmark.tcl b/tests/integration/redis-benchmark.tcl
index 4a4be2ebc..5a4f09952 100644
--- a/tests/integration/redis-benchmark.tcl
+++ b/tests/integration/redis-benchmark.tcl
@@ -33,7 +33,7 @@ start_server {tags {"benchmark"}} {
                 puts [colorstr red "redis-benchmark non zero code. first line: $first_line"]
                 fail "redis-benchmark non zero code. first line: $first_line"
             }
-            # ping total calls are 2*issued commands per test due to PING_INLINE and PING_BULK
+            # ping total calls are 2*issued commands per test due to PING_INLINE and PING_MBULK
             assert_match  {*calls=200,*} [cmdstat ping]
             assert_match  {*calls=100,*} [cmdstat set]
             assert_match  {*calls=100,*} [cmdstat get]

From 84e0489562a0f61b95755dc863bfca79c23b6ef0 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 2 Dec 2020 13:47:25 +0200
Subject: [PATCH 0833/1098] Remove end-of-life CentOS 6 workflows. (#8121)

---
 .github/workflows/ci.yml    | 10 ----------
 .github/workflows/daily.yml | 22 ----------------------
 2 files changed, 32 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index be2be1789..2582c53a4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -58,13 +58,3 @@ jobs:
       run: |
         yum -y install gcc make
         make REDIS_CFLAGS='-Werror'
-
-  build-centos6-jemalloc:
-    runs-on: ubuntu-latest
-    container: centos:6
-    steps:
-    - uses: actions/checkout@v1
-    - name: make
-      run: |
-        yum -y install gcc make
-        make
diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index d78252026..028c44e0c 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -137,28 +137,6 @@ jobs:
     - name: cluster tests
       run: ./runtest-cluster
 
-  test-centos6-jemalloc:
-    runs-on: ubuntu-latest
-    if: github.repository == 'redis/redis'
-    container: centos:6
-    timeout-minutes: 14400
-    steps:
-    - uses: actions/checkout@v1
-    - name: make
-      run: |
-        yum -y install gcc make
-        make
-    - name: test
-      run: |
-        yum -y install which tcl util-linux-ng
-        ./runtest --accurate --verbose
-    - name: module api test
-      run: ./runtest-moduleapi --verbose
-    - name: sentinel tests
-      run: ./runtest-sentinel
-    - name: cluster tests
-      run: ./runtest-cluster
-
   test-centos7-tls:
     runs-on: ubuntu-latest
     if: github.repository == 'redis/redis'

From b55a827ea2e19bd6cd48f216e8e6caa34299f9b9 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuancode@163.com>
Date: Wed, 2 Dec 2020 19:56:11 +0800
Subject: [PATCH 0834/1098] Backup keys to slots map and restore when fail to
 sync if diskless-load type is swapdb in cluster mode (#8108)

When replica diskless-load type is swapdb in cluster mode, we didn't backup
keys to slots map, so we will lose keys to slots map if fail to sync.
Now we backup keys to slots map at first, and restore it properly when fail.

This commit includes a refactory/cleanup of the backups mechanism (moving it to db.c and re-structuring it a bit).

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/bio.c                                     |   2 +-
 src/db.c                                      | 218 +++++++++++++-----
 src/lazyfree.c                                |  20 +-
 src/replication.c                             |  69 ++----
 src/server.h                                  |  18 +-
 .../cluster/tests/17-diskless-load-swapdb.tcl |  77 +++++++
 6 files changed, 279 insertions(+), 125 deletions(-)
 create mode 100644 tests/cluster/tests/17-diskless-load-swapdb.tcl

diff --git a/src/bio.c b/src/bio.c
index a512def0a..a11bcb18b 100644
--- a/src/bio.c
+++ b/src/bio.c
@@ -203,7 +203,7 @@ void *bioProcessBackgroundJobs(void *arg) {
             /* What we free changes depending on what arguments are set:
              * arg1 -> free the object at pointer.
              * arg2 & arg3 -> free two dictionaries (a Redis DB).
-             * only arg3 -> free the skiplist. */
+             * only arg3 -> free the radix tree. */
             if (job->arg1)
                 lazyfreeFreeObjectFromBioThread(job->arg1);
             else if (job->arg2 && job->arg3)
diff --git a/src/db.c b/src/db.c
index 32baf0974..b52c6f3f2 100644
--- a/src/db.c
+++ b/src/db.c
@@ -34,6 +34,13 @@
 #include <signal.h>
 #include <ctype.h>
 
+/* Database backup. */
+struct dbBackup {
+    redisDb *dbarray;
+    rax *slots_to_keys;
+    uint64_t slots_keys_count[CLUSTER_SLOTS];
+};
+
 /*-----------------------------------------------------------------------------
  * C-level DB API
  *----------------------------------------------------------------------------*/
@@ -359,48 +366,18 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) {
     return o;
 }
 
-/* Remove all keys from all the databases in a Redis server.
- * If callback is given the function is called from time to time to
- * signal that work is in progress.
+/* Remove all keys from the database(s) structure. The dbarray argument
+ * may not be the server main DBs (could be a backup).
  *
- * The dbnum can be -1 if all the DBs should be flushed, or the specified
- * DB number if we want to flush only a single Redis database number.
- *
- * Flags are be EMPTYDB_NO_FLAGS if no special flags are specified or
- * 1. EMPTYDB_ASYNC if we want the memory to be freed in a different thread.
- * 2. EMPTYDB_BACKUP if we want to empty the backup dictionaries created by
- *    disklessLoadMakeBackups. In that case we only free memory and avoid
- *    firing module events.
- * and the function to return ASAP.
- *
- * On success the function returns the number of keys removed from the
- * database(s). Otherwise -1 is returned in the specific case the
- * DB number is out of range, and errno is set to EINVAL. */
-long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(void*)) {
-    int async = (flags & EMPTYDB_ASYNC);
-    int backup = (flags & EMPTYDB_BACKUP); /* Just free the memory, nothing else */
-    RedisModuleFlushInfoV1 fi = {REDISMODULE_FLUSHINFO_VERSION,!async,dbnum};
+ * The dbnum can be -1 if all the DBs should be emptied, or the specified
+ * DB index if we want to empty only a single database.
+ * The function returns the number of keys removed from the database(s). */
+long long emptyDbStructure(redisDb *dbarray, int dbnum, int async,
+                           void(callback)(void*))
+{
     long long removed = 0;
-
-    if (dbnum < -1 || dbnum >= server.dbnum) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    /* Pre-flush actions */
-    if (!backup) {
-        /* Fire the flushdb modules event. */
-        moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB,
-                              REDISMODULE_SUBEVENT_FLUSHDB_START,
-                              &fi);
-
-        /* Make sure the WATCHed keys are affected by the FLUSH* commands.
-         * Note that we need to call the function while the keys are still
-         * there. */
-        signalFlushedDb(dbnum);
-    }
-
     int startdb, enddb;
+
     if (dbnum == -1) {
         startdb = 0;
         enddb = server.dbnum-1;
@@ -416,34 +393,139 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(
             dictEmpty(dbarray[j].dict,callback);
             dictEmpty(dbarray[j].expires,callback);
         }
-        /* Because we will start a new database, reset average ttl. */
+        /* Because all keys of database are removed, reset average ttl. */
         dbarray[j].avg_ttl = 0;
         dbarray[j].expires_cursor = 0;
     }
 
-    /* Post-flush actions */
-    if (!backup) {
-        if (server.cluster_enabled) {
-            if (async) {
-                slotToKeyFlushAsync();
-            } else {
-                slotToKeyFlush();
-            }
-        }
-        if (dbnum == -1) flushSlaveKeysWithExpireList();
-
-        /* Also fire the end event. Note that this event will fire almost
-         * immediately after the start event if the flush is asynchronous. */
-        moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB,
-                              REDISMODULE_SUBEVENT_FLUSHDB_END,
-                              &fi);
-    }
-
     return removed;
 }
 
+/* Remove all keys from all the databases in a Redis server.
+ * If callback is given the function is called from time to time to
+ * signal that work is in progress.
+ *
+ * The dbnum can be -1 if all the DBs should be flushed, or the specified
+ * DB number if we want to flush only a single Redis database number.
+ *
+ * Flags are be EMPTYDB_NO_FLAGS if no special flags are specified or
+ * EMPTYDB_ASYNC if we want the memory to be freed in a different thread
+ * and the function to return ASAP.
+ *
+ * On success the function returns the number of keys removed from the
+ * database(s). Otherwise -1 is returned in the specific case the
+ * DB number is out of range, and errno is set to EINVAL. */
 long long emptyDb(int dbnum, int flags, void(callback)(void*)) {
-    return emptyDbGeneric(server.db, dbnum, flags, callback);
+    int async = (flags & EMPTYDB_ASYNC);
+    RedisModuleFlushInfoV1 fi = {REDISMODULE_FLUSHINFO_VERSION,!async,dbnum};
+    long long removed = 0;
+
+    if (dbnum < -1 || dbnum >= server.dbnum) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    /* Fire the flushdb modules event. */
+    moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB,
+                          REDISMODULE_SUBEVENT_FLUSHDB_START,
+                          &fi);
+
+    /* Make sure the WATCHed keys are affected by the FLUSH* commands.
+     * Note that we need to call the function while the keys are still
+     * there. */
+    signalFlushedDb(dbnum);
+
+    /* Empty redis database structure. */
+    removed = emptyDbStructure(server.db, dbnum, async, callback);
+
+    /* Flush slots to keys map if enable cluster, we can flush entire
+     * slots to keys map whatever dbnum because only support one DB
+     * in cluster mode. */
+    if (server.cluster_enabled) slotToKeyFlush(async);
+
+    if (dbnum == -1) flushSlaveKeysWithExpireList();
+
+    /* Also fire the end event. Note that this event will fire almost
+     * immediately after the start event if the flush is asynchronous. */
+    moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB,
+                          REDISMODULE_SUBEVENT_FLUSHDB_END,
+                          &fi);
+
+    return removed;
+}
+
+/* Store a backup of the database for later use, and put an empty one
+ * instead of it. */
+dbBackup *backupDb(void) {
+    dbBackup *backup = zmalloc(sizeof(dbBackup));
+
+    /* Backup main DBs. */
+    backup->dbarray = zmalloc(sizeof(redisDb)*server.dbnum);
+    for (int i=0; i<server.dbnum; i++) {
+        backup->dbarray[i] = server.db[i];
+        server.db[i].dict = dictCreate(&dbDictType,NULL);
+        server.db[i].expires = dictCreate(&keyptrDictType,NULL);
+    }
+
+    /* Backup cluster slots to keys map if enable cluster. */
+    if (server.cluster_enabled) {
+        backup->slots_to_keys = server.cluster->slots_to_keys;
+        memcpy(backup->slots_keys_count, server.cluster->slots_keys_count,
+            sizeof(server.cluster->slots_keys_count));
+        server.cluster->slots_to_keys = raxNew();
+        memset(server.cluster->slots_keys_count, 0,
+            sizeof(server.cluster->slots_keys_count));
+    }
+
+    return backup;
+}
+
+/* Discard a previously created backup, this can be slow (similar to FLUSHALL)
+ * Arguments are similar to the ones of emptyDb, see EMPTYDB_ flags. */
+void discardDbBackup(dbBackup *buckup, int flags, void(callback)(void*)) {
+    int async = (flags & EMPTYDB_ASYNC);
+
+    /* Release main DBs backup . */
+    emptyDbStructure(buckup->dbarray, -1, async, callback);
+    for (int i=0; i<server.dbnum; i++) {
+        dictRelease(buckup->dbarray[i].dict);
+        dictRelease(buckup->dbarray[i].expires);
+    }
+
+    /* Release slots to keys map backup if enable cluster. */
+    if (server.cluster_enabled) freeSlotsToKeysMap(buckup->slots_to_keys, async);
+
+    /* Release buckup. */
+    zfree(buckup->dbarray);
+    zfree(buckup);
+}
+
+/* Restore the previously created backup (discarding what currently resides
+ * in the db).
+ * This function should be called after the current contents of the database
+ * was emptied with a previous call to emptyDb (possibly using the async mode). */
+void restoreDbBackup(dbBackup *buckup) {
+    /* Restore main DBs. */
+    for (int i=0; i<server.dbnum; i++) {
+        serverAssert(dictSize(server.db[i].dict) == 0);
+        serverAssert(dictSize(server.db[i].expires) == 0);
+        dictRelease(server.db[i].dict);
+        dictRelease(server.db[i].expires);
+        server.db[i] = buckup->dbarray[i];
+    }
+
+    /* Restore slots to keys map backup if enable cluster. */
+    if (server.cluster_enabled) {
+        serverAssert(server.cluster->slots_to_keys->numele == 0);
+        raxFree(server.cluster->slots_to_keys);
+        server.cluster->slots_to_keys = buckup->slots_to_keys;
+        memcpy(server.cluster->slots_keys_count, buckup->slots_keys_count,
+                sizeof(server.cluster->slots_keys_count));
+    }
+
+    /* Release buckup. */
+    zfree(buckup->dbarray);
+    zfree(buckup);
 }
 
 int selectDb(client *c, int id) {
@@ -1796,11 +1878,25 @@ void slotToKeyDel(sds key) {
     slotToKeyUpdateKey(key,0);
 }
 
-void slotToKeyFlush(void) {
-    raxFree(server.cluster->slots_to_keys);
+/* Release the radix tree mapping Redis Cluster keys to slots. If 'async'
+ * is true, we release it asynchronously. */
+void freeSlotsToKeysMap(rax *rt, int async) {
+    if (async) {
+        freeSlotsToKeysMapAsync(rt);
+    } else {
+        raxFree(rt);
+    }
+}
+
+/* Empty the slots-keys map of Redis CLuster by creating a new empty one and
+ * freeing the old one. */
+void slotToKeyFlush(int async) {
+    rax *old = server.cluster->slots_to_keys;
+
     server.cluster->slots_to_keys = raxNew();
     memset(server.cluster->slots_keys_count,0,
            sizeof(server.cluster->slots_keys_count));
+    freeSlotsToKeysMap(old, async);
 }
 
 /* Populate the specified array of objects with keys in the specified slot.
diff --git a/src/lazyfree.c b/src/lazyfree.c
index 5a78d5a55..641ab4e64 100644
--- a/src/lazyfree.c
+++ b/src/lazyfree.c
@@ -158,16 +158,10 @@ void emptyDbAsync(redisDb *db) {
     bioCreateBackgroundJob(BIO_LAZY_FREE,NULL,oldht1,oldht2);
 }
 
-/* Empty the slots-keys map of Redis CLuster by creating a new empty one
- * and scheduling the old for lazy freeing. */
-void slotToKeyFlushAsync(void) {
-    rax *old = server.cluster->slots_to_keys;
-
-    server.cluster->slots_to_keys = raxNew();
-    memset(server.cluster->slots_keys_count,0,
-           sizeof(server.cluster->slots_keys_count));
-    atomicIncr(lazyfree_objects,old->numele);
-    bioCreateBackgroundJob(BIO_LAZY_FREE,NULL,NULL,old);
+/* Release the radix tree mapping Redis Cluster keys to slots asynchronously. */
+void freeSlotsToKeysMapAsync(rax *rt) {
+    atomicIncr(lazyfree_objects,rt->numele);
+    bioCreateBackgroundJob(BIO_LAZY_FREE,NULL,NULL,rt);
 }
 
 /* Release objects from the lazyfree thread. It's just decrRefCount()
@@ -180,9 +174,7 @@ void lazyfreeFreeObjectFromBioThread(robj *o) {
 
 /* Release a database from the lazyfree thread. The 'db' pointer is the
  * database which was substituted with a fresh one in the main thread
- * when the database was logically deleted. 'sl' is a skiplist used by
- * Redis Cluster in order to take the hash slots -> keys mapping. This
- * may be NULL if Redis Cluster is disabled. */
+ * when the database was logically deleted. */
 void lazyfreeFreeDatabaseFromBioThread(dict *ht1, dict *ht2) {
     size_t numkeys = dictSize(ht1);
     dictRelease(ht1);
@@ -191,7 +183,7 @@ void lazyfreeFreeDatabaseFromBioThread(dict *ht1, dict *ht2) {
     atomicIncr(lazyfreed_objects,numkeys);
 }
 
-/* Release the skiplist mapping Redis Cluster keys to slots in the
+/* Release the radix tree mapping Redis Cluster keys to slots in the
  * lazyfree thread. */
 void lazyfreeFreeSlotsMapFromBioThread(rax *rt) {
     size_t len = rt->numele;
diff --git a/src/replication.c b/src/replication.c
index a288478ff..64aa41390 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1432,16 +1432,10 @@ static int useDisklessLoad() {
 }
 
 /* Helper function for readSyncBulkPayload() to make backups of the current
- * DBs before socket-loading the new ones. The backups may be restored later
- * or freed by disklessLoadRestoreBackups(). */
-redisDb *disklessLoadMakeBackups(void) {
-    redisDb *backups = zmalloc(sizeof(redisDb)*server.dbnum);
-    for (int i=0; i<server.dbnum; i++) {
-        backups[i] = server.db[i];
-        server.db[i].dict = dictCreate(&dbDictType,NULL);
-        server.db[i].expires = dictCreate(&keyptrDictType,NULL);
-    }
-    return backups;
+ * databases before socket-loading the new ones. The backups may be restored
+ * by disklessLoadRestoreBackup or freed by disklessLoadDiscardBackup later. */
+dbBackup *disklessLoadMakeBackup(void) {
+    return backupDb();
 }
 
 /* Helper function for readSyncBulkPayload(): when replica-side diskless
@@ -1449,30 +1443,15 @@ redisDb *disklessLoadMakeBackups(void) {
  * before loading the new ones from the socket.
  *
  * If the socket loading went wrong, we want to restore the old backups
- * into the server databases. This function does just that in the case
- * the 'restore' argument (the number of DBs to replace) is non-zero.
- *
- * When instead the loading succeeded we want just to free our old backups,
- * in that case the function will do just that when 'restore' is 0. */
-void disklessLoadRestoreBackups(redisDb *backup, int restore, int empty_db_flags)
-{
-    if (restore) {
-        /* Restore. */
-        emptyDbGeneric(server.db,-1,empty_db_flags,replicationEmptyDbCallback);
-        for (int i=0; i<server.dbnum; i++) {
-            dictRelease(server.db[i].dict);
-            dictRelease(server.db[i].expires);
-            server.db[i] = backup[i];
-        }
-    } else {
-        /* Delete (Pass EMPTYDB_BACKUP in order to avoid firing module events) . */
-        emptyDbGeneric(backup,-1,empty_db_flags|EMPTYDB_BACKUP,replicationEmptyDbCallback);
-        for (int i=0; i<server.dbnum; i++) {
-            dictRelease(backup[i].dict);
-            dictRelease(backup[i].expires);
-        }
-    }
-    zfree(backup);
+ * into the server databases. */
+void disklessLoadRestoreBackup(dbBackup *buckup) {
+    restoreDbBackup(buckup);
+}
+
+/* Helper function for readSyncBulkPayload() to discard our old backups
+ * when the loading succeeded. */
+void disklessLoadDiscardBackup(dbBackup *buckup, int flag) {
+    discardDbBackup(buckup, flag, replicationEmptyDbCallback);
 }
 
 /* Asynchronously read the SYNC payload we receive from a master */
@@ -1481,7 +1460,7 @@ void readSyncBulkPayload(connection *conn) {
     char buf[PROTO_IOBUF_LEN];
     ssize_t nread, readlen, nwritten;
     int use_diskless_load = useDisklessLoad();
-    redisDb *diskless_load_backup = NULL;
+    dbBackup *diskless_load_backup = NULL;
     int empty_db_flags = server.repl_slave_lazy_flush ? EMPTYDB_ASYNC :
                                                         EMPTYDB_NO_FLAGS;
     off_t left;
@@ -1662,11 +1641,11 @@ void readSyncBulkPayload(connection *conn) {
         server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB)
     {
         /* Create a backup of server.db[] and initialize to empty
-         * dictionaries */
-        diskless_load_backup = disklessLoadMakeBackups();
+         * dictionaries. */
+        diskless_load_backup = disklessLoadMakeBackup();
     }
     /* We call to emptyDb even in case of REPL_DISKLESS_LOAD_SWAPDB
-     * (Where disklessLoadMakeBackups left server.db empty) because we
+     * (Where disklessLoadMakeBackup left server.db empty) because we
      * want to execute all the auxiliary logic of emptyDb (Namely,
      * fire module events) */
     emptyDb(-1,empty_db_flags,replicationEmptyDbCallback);
@@ -1696,14 +1675,14 @@ void readSyncBulkPayload(connection *conn) {
                 "from socket");
             cancelReplicationHandshake(1);
             rioFreeConn(&rdb, NULL);
+
+            /* Remove the half-loaded data in case we started with
+             * an empty replica. */
+            emptyDb(-1,empty_db_flags,replicationEmptyDbCallback);
+
             if (server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB) {
                 /* Restore the backed up databases. */
-                disklessLoadRestoreBackups(diskless_load_backup,1,
-                                           empty_db_flags);
-            } else {
-                /* Remove the half-loaded data in case we started with
-                 * an empty replica. */
-                emptyDb(-1,empty_db_flags,replicationEmptyDbCallback);
+                disklessLoadRestoreBackup(diskless_load_backup);
             }
 
             /* Note that there's no point in restarting the AOF on SYNC
@@ -1718,7 +1697,7 @@ void readSyncBulkPayload(connection *conn) {
             /* Delete the backup databases we created before starting to load
              * the new RDB. Now the RDB was loaded with success so the old
              * data is useless. */
-            disklessLoadRestoreBackups(diskless_load_backup,0,empty_db_flags);
+            disklessLoadDiscardBackup(diskless_load_backup, empty_db_flags);
         }
 
         /* Verify the end mark is correct. */
diff --git a/src/server.h b/src/server.h
index 0bcbeabfb..a858d5833 100644
--- a/src/server.h
+++ b/src/server.h
@@ -684,6 +684,11 @@ typedef struct redisDb {
     list *defrag_later;         /* List of key names to attempt to defrag one by one, gradually. */
 } redisDb;
 
+/* Declare database backup that include redis main DBs and slots to keys map.
+ * Definition is in db.c. We can't define it here since we define CLUSTER_SLOTS
+ * in cluster.h. */
+typedef struct dbBackup dbBackup;
+
 /* Client MULTI/EXEC state */
 typedef struct multiCmd {
     robj **argv;
@@ -2208,11 +2213,14 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o);
 
 #define EMPTYDB_NO_FLAGS 0      /* No flags. */
 #define EMPTYDB_ASYNC (1<<0)    /* Reclaim memory in another thread. */
-#define EMPTYDB_BACKUP (1<<2)   /* DB array is a backup for REPL_DISKLESS_LOAD_SWAPDB. */
 long long emptyDb(int dbnum, int flags, void(callback)(void*));
-long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(void*));
+long long emptyDbStructure(redisDb *dbarray, int dbnum, int async, void(callback)(void*));
 void flushAllDataAndResetRDB(int flags);
 long long dbTotalServerKeyCount();
+dbBackup *backupDb(void);
+void restoreDbBackup(dbBackup *buckup);
+void discardDbBackup(dbBackup *buckup, int flags, void(callback)(void*));
+
 
 int selectDb(client *c, int id);
 void signalModifiedKey(client *c, redisDb *db, robj *key);
@@ -2225,13 +2233,15 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor);
 int parseScanCursorOrReply(client *c, robj *o, unsigned long *cursor);
 void slotToKeyAdd(sds key);
 void slotToKeyDel(sds key);
-void slotToKeyFlush(void);
 int dbAsyncDelete(redisDb *db, robj *key);
 void emptyDbAsync(redisDb *db);
-void slotToKeyFlushAsync(void);
+void slotToKeyFlush(int async);
 size_t lazyfreeGetPendingObjectsCount(void);
 size_t lazyfreeGetFreedObjectsCount(void);
 void freeObjAsync(robj *key, robj *obj);
+void freeSlotsToKeysMapAsync(rax *rt);
+void freeSlotsToKeysMap(rax *rt, int async);
+
 
 /* API to get key arguments from commands */
 int *getKeysPrepareResult(getKeysResult *result, int numkeys);
diff --git a/tests/cluster/tests/17-diskless-load-swapdb.tcl b/tests/cluster/tests/17-diskless-load-swapdb.tcl
new file mode 100644
index 000000000..a035be7be
--- /dev/null
+++ b/tests/cluster/tests/17-diskless-load-swapdb.tcl
@@ -0,0 +1,77 @@
+# Check replica can restore database buckup correctly if fail to diskless load.
+
+source "../tests/includes/init-tests.tcl"
+
+test "Create a primary with a replica" {
+    create_cluster 1 1
+}
+
+test "Cluster should start ok" {
+    assert_cluster_state ok
+}
+
+test "Cluster is writable" {
+    cluster_write_test 0
+}
+
+test "Right to restore backups when fail to diskless load " {
+    set master [Rn 0]
+    set replica [Rn 1]
+    set master_id 0
+    set replica_id 1
+
+    $replica READONLY
+    $replica config set repl-diskless-load swapdb
+    $replica config rewrite
+    $master config set repl-backlog-size 1024
+    $master config set repl-diskless-sync yes
+    $master config set repl-diskless-sync-delay 0
+    $master config set rdb-key-save-delay 10000
+    $master config set rdbcompression no
+    $master config set appendonly no
+    $master config set save ""
+
+    # Write a key that belongs to slot 0
+    set slot0_key "06S"
+    $master set $slot0_key 1
+    after 100
+    assert_equal {1} [$replica get $slot0_key]
+    assert_equal $slot0_key [$replica CLUSTER GETKEYSINSLOT 0 1]
+
+    # Kill the replica
+    kill_instance redis $replica_id
+
+    # Delete the key from master
+    $master del $slot0_key
+
+    # Replica must full sync with master when start because replication
+    # backlog size is very small, and dumping rdb will cost several seconds.
+    set num 10000
+    set value [string repeat A 1024]
+    set rd [redis_deferring_client redis $master_id]
+    for {set j 0} {$j < $num} {incr j} {
+        $rd set $j $value
+    }
+    for {set j 0} {$j < $num} {incr j} {
+        $rd read
+    }
+
+    # Start the replica again
+    restart_instance redis $replica_id
+    $replica READONLY
+
+    # Start full sync
+    wait_for_condition 500 10 {
+        [string match "*sync*" [$replica role]]
+    } else {
+        fail "Fail to full sync"
+    }
+    after 100
+
+    # Kill master, abort full sync
+    kill_instance redis $master_id
+
+    # Replica keys and keys to slots map still both are right
+    assert_equal {1} [$replica get $slot0_key]
+    assert_equal $slot0_key [$replica CLUSTER GETKEYSINSLOT 0 1]
+}
\ No newline at end of file

From 7459652e3e4dbc7d8330ddc860a2585effaead0e Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Wed, 2 Dec 2020 17:24:27 +0200
Subject: [PATCH 0835/1098] Fix ACL Pub/Sub test timings (#8122)

---
 tests/unit/acl.tcl | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl
index 856bf6196..f5e3259fe 100644
--- a/tests/unit/acl.tcl
+++ b/tests/unit/acl.tcl
@@ -143,8 +143,11 @@ start_server {tags {"acl"}} {
         set rd [redis_deferring_client]
         r ACL setuser psuser resetchannels &foo:1
         $rd AUTH psuser pspass
+        $rd read
         $rd CLIENT SETNAME deathrow
+        $rd read
         $rd SUBSCRIBE foo:1
+        $rd read
         r ACL setuser psuser resetchannels
         assert_no_match {*deathrow*} [r CLIENT LIST]
         $rd close
@@ -154,8 +157,11 @@ start_server {tags {"acl"}} {
         set rd [redis_deferring_client]
         r ACL setuser psuser resetchannels &bar:*
         $rd AUTH psuser pspass
+        $rd read
         $rd CLIENT SETNAME deathrow
+        $rd read
         $rd PSUBSCRIBE bar:*
+        $rd read
         r ACL setuser psuser resetchannels
         assert_no_match {*deathrow*} [r CLIENT LIST]
         $rd close
@@ -165,9 +171,13 @@ start_server {tags {"acl"}} {
         set rd [redis_deferring_client]
         r ACL setuser psuser resetchannels &foo:1 &bar:*
         $rd AUTH psuser pspass
+        $rd read
         $rd CLIENT SETNAME pardoned
+        $rd read
         $rd SUBSCRIBE foo:1
+        $rd read
         $rd PSUBSCRIBE bar:*
+        $rd read
         r ACL setuser psuser resetchannels &foo:1 &bar:* &baz:qaz &zoo:*
         assert_match {*pardoned*} [r CLIENT LIST]
         r ACL setuser psuser allchannels

From 4cd1fb1f40b30e08f37425e626420c580424bbf0 Mon Sep 17 00:00:00 2001
From: Felipe Machado <462154+felipou@users.noreply.github.com>
Date: Thu, 3 Dec 2020 05:12:07 -0300
Subject: [PATCH 0836/1098] Iterate backwards on zdiff/zinter/zunion to
 optimize for zslInsert (#8105)

In the iterator for these functions, we'll traverse the sorted sets
in a reversed way so that largest elements come first. We prefer
this order because it's optimized for insertion in a skiplist, which
is the destination of the elements being iterated in there functions.
---
 src/t_zset.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/t_zset.c b/src/t_zset.c
index a6ce20132..382d3b9ce 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -1965,17 +1965,20 @@ void zuiInitIterator(zsetopsrc *op) {
             serverPanic("Unknown set encoding");
         }
     } else if (op->type == OBJ_ZSET) {
+        /* Sorted sets are traversed in reverse order to optimize for
+         * the insertion of the elements in a new list as in
+         * ZDIFF/ZINTER/ZUNION */
         iterzset *it = &op->iter.zset;
         if (op->encoding == OBJ_ENCODING_ZIPLIST) {
             it->zl.zl = op->subject->ptr;
-            it->zl.eptr = ziplistIndex(it->zl.zl,0);
+            it->zl.eptr = ziplistIndex(it->zl.zl,-2);
             if (it->zl.eptr != NULL) {
                 it->zl.sptr = ziplistNext(it->zl.zl,it->zl.eptr);
                 serverAssert(it->zl.sptr != NULL);
             }
         } else if (op->encoding == OBJ_ENCODING_SKIPLIST) {
             it->sl.zs = op->subject->ptr;
-            it->sl.node = it->sl.zs->zsl->header->level[0].forward;
+            it->sl.node = it->sl.zs->zsl->tail;
         } else {
             serverPanic("Unknown sorted set encoding");
         }
@@ -2082,16 +2085,16 @@ int zuiNext(zsetopsrc *op, zsetopval *val) {
             serverAssert(ziplistGet(it->zl.eptr,&val->estr,&val->elen,&val->ell));
             val->score = zzlGetScore(it->zl.sptr);
 
-            /* Move to next element. */
-            zzlNext(it->zl.zl,&it->zl.eptr,&it->zl.sptr);
+            /* Move to next element (going backwards, see zuiInitIterator). */
+            zzlPrev(it->zl.zl,&it->zl.eptr,&it->zl.sptr);
         } else if (op->encoding == OBJ_ENCODING_SKIPLIST) {
             if (it->sl.node == NULL)
                 return 0;
             val->ele = it->sl.node->ele;
             val->score = it->sl.node->score;
 
-            /* Move to next element. */
-            it->sl.node = it->sl.node->level[0].forward;
+            /* Move to next element. (going backwards, see zuiInitIterator) */
+            it->sl.node = it->sl.node->backward;
         } else {
             serverPanic("Unknown sorted set encoding");
         }

From 441c49002461f14d1aeed5571112d57793df6bb0 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Thu, 3 Dec 2020 14:36:48 +0200
Subject: [PATCH 0837/1098] Adds exclusive ranges to X[REV]RANGE (#8072)

Adds the ability to use exclusive (open) start and end query intervals in XRANGE and XREVRANGE queries.

Fixes #6562
---
 src/stream.h               |  3 +-
 src/t_stream.c             | 78 +++++++++++++++++++++++++++++++++++---
 tests/unit/type/stream.tcl | 26 +++++++++++++
 3 files changed, 100 insertions(+), 7 deletions(-)

diff --git a/src/stream.h b/src/stream.h
index 7e1a24417..1a1e271a8 100644
--- a/src/stream.h
+++ b/src/stream.h
@@ -116,7 +116,8 @@ streamNACK *streamCreateNACK(streamConsumer *consumer);
 void streamDecodeID(void *buf, streamID *id);
 int streamCompareID(streamID *a, streamID *b);
 void streamFreeNACK(streamNACK *na);
-void streamIncrID(streamID *id);
+int streamIncrID(streamID *id);
+int streamDecrID(streamID *id);
 void streamPropagateConsumerCreation(client *c, robj *key, robj *groupname, sds consumername);
 robj *streamDup(robj *o);
 
diff --git a/src/t_stream.c b/src/t_stream.c
index e57916200..bb38b0fb8 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -76,12 +76,16 @@ unsigned long streamLength(const robj *subject) {
     return s->length;
 }
 
-/* Set 'id' to be its successor streamID */
-void streamIncrID(streamID *id) {
+/* Set 'id' to be its successor stream ID.
+ * If 'id' is the maximal possible id, it is wrapped around to 0-0 and a
+ * C_ERR is returned. */
+int streamIncrID(streamID *id) {
+    int ret = C_OK;
     if (id->seq == UINT64_MAX) {
         if (id->ms == UINT64_MAX) {
             /* Special case where 'id' is the last possible streamID... */
             id->ms = id->seq = 0;
+            ret = C_ERR;
         } else {
             id->ms++;
             id->seq = 0;
@@ -89,6 +93,27 @@ void streamIncrID(streamID *id) {
     } else {
         id->seq++;
     }
+    return ret;
+}
+
+/* Set 'id' to be its predecessor stream ID.
+ * If 'id' is the minimal possible id, it remains 0-0 and a C_ERR is
+ * returned. */
+int streamDecrID(streamID *id) {
+    int ret = C_OK;
+    if (id->seq == 0) {
+        if (id->ms == 0) {
+            /* Special case where 'id' is the first possible streamID... */
+            id->ms = id->seq = UINT64_MAX;
+            ret = C_ERR;
+        } else {
+            id->ms--;
+            id->seq = UINT64_MAX;
+        }
+    } else {
+        id->seq--;
+    }
+    return ret;
 }
 
 /* Generate the next stream item ID given the previous one. If the current
@@ -1309,6 +1334,29 @@ int streamParseStrictIDOrReply(client *c, robj *o, streamID *id, uint64_t missin
     return streamGenericParseIDOrReply(c,o,id,missing_seq,1);
 }
 
+/* Helper for parsing a stream ID that is a range query interval. When the
+ * exclude argument is NULL, streamParseIDOrReply() is called and the interval
+ * is treated as close (inclusive). Otherwise, the exclude argument is set if 
+ * the interval is open (the "(" prefix) and streamParseStrictIDOrReply() is
+ * called in that case.
+ */
+int streamParseIntervalIDOrReply(client *c, robj *o, streamID *id, int *exclude, uint64_t missing_seq) {
+    char *p = o->ptr;
+    size_t len = sdslen(p);
+    int invalid = 0;
+    
+    if (exclude != NULL) *exclude = (len > 1 && p[0] == '(');
+    if (exclude != NULL && *exclude) {
+        robj *t = createStringObject(p+1,len-1);
+        invalid = (streamParseStrictIDOrReply(c,t,id,missing_seq) == C_ERR);
+        decrRefCount(t);
+    } else 
+        invalid = (streamParseIDOrReply(c,o,id,missing_seq) == C_ERR);
+    if (invalid)
+        return C_ERR;
+    return C_OK;
+}
+
 /* We propagate MAXLEN ~ <count> as MAXLEN = <resulting-len-of-stream>
  * otherwise trimming is no longer determinsitic on replicas / AOF. */
 void streamRewriteApproxMaxlen(client *c, stream *s, int maxlen_arg_idx) {
@@ -1433,7 +1481,13 @@ void xaddCommand(client *c) {
     signalKeyAsReady(c->db, c->argv[1], OBJ_STREAM);
 }
 
-/* XRANGE/XREVRANGE actual implementation. */
+/* XRANGE/XREVRANGE actual implementation.
+ * The 'start' and 'end' IDs are parsed as follows:
+ *   Incomplete 'start' has its sequence set to 0, and 'end' to UINT64_MAX.
+ *   "-" and "+"" mean the minimal and maximal ID values, respectively.
+ *   The "(" prefix means an open (exclusive) range, so XRANGE stream (1-0 (2-0
+ *   will match anything from 1-1 and 1-UINT64_MAX.
+ */
 void xrangeGenericCommand(client *c, int rev) {
     robj *o;
     stream *s;
@@ -1441,9 +1495,21 @@ void xrangeGenericCommand(client *c, int rev) {
     long long count = -1;
     robj *startarg = rev ? c->argv[3] : c->argv[2];
     robj *endarg = rev ? c->argv[2] : c->argv[3];
-
-    if (streamParseIDOrReply(c,startarg,&startid,0) == C_ERR) return;
-    if (streamParseIDOrReply(c,endarg,&endid,UINT64_MAX) == C_ERR) return;
+    int startex = 0, endex = 0;
+    
+    /* Parse start and end IDs. */
+    if (streamParseIntervalIDOrReply(c,startarg,&startid,&startex,0) != C_OK)
+        return;
+    if (startex && streamIncrID(&startid) != C_OK) {
+        addReplyError(c,"invalid start ID for the interval");
+        return;
+    }
+    if (streamParseIntervalIDOrReply(c,endarg,&endid,&endex,UINT64_MAX) != C_OK)
+        return;
+    if (endex && streamDecrID(&endid) != C_OK) {
+        addReplyError(c,"invalid end ID for the interval");
+        return;
+    }
 
     /* Parse the COUNT option if any. */
     if (c->argc > 4) {
diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl
index 4b3672d64..63cc697c2 100644
--- a/tests/unit/type/stream.tcl
+++ b/tests/unit/type/stream.tcl
@@ -192,6 +192,32 @@ start_server {
         assert {[r xrange mystream - +] == [lreverse [r xrevrange mystream + -]]}
     }
 
+    test {XRANGE exclusive ranges} {
+        set ids {0-1 0-18446744073709551615 1-0 42-0 42-42
+                 18446744073709551615-18446744073709551614
+                 18446744073709551615-18446744073709551615}
+        set total [llength $ids]
+        r multi
+        r DEL vipstream
+        foreach id $ids {
+            r XADD vipstream $id foo bar
+        }
+        r exec
+        assert {[llength [r xrange vipstream - +]] == $total}
+        assert {[llength [r xrange vipstream ([lindex $ids 0] +]] == $total-1}
+        assert {[llength [r xrange vipstream - ([lindex $ids $total-1]]] == $total-1}
+        assert {[llength [r xrange vipstream (0-1 (1-0]] == 1}
+        assert {[llength [r xrange vipstream (1-0 (42-42]] == 1}
+        catch {r xrange vipstream (- +} e
+        assert_match {ERR*} $e
+        catch {r xrange vipstream - (+} e
+        assert_match {ERR*} $e
+        catch {r xrange vipstream (18446744073709551615-18446744073709551615 +} e
+        assert_match {ERR*} $e
+        catch {r xrange vipstream - (0-0} e
+        assert_match {ERR*} $e
+    }
+
     test {XREAD with non empty stream} {
         set res [r XREAD COUNT 1 STREAMS mystream 0-0]
         assert {[lrange [lindex $res 0 1 0 1] 0 1] eq {item 0}}

From 2f41a3856845265ffc6cc3a35524883a8690cff7 Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Thu, 3 Dec 2020 19:36:48 +0100
Subject: [PATCH 0838/1098] Modules: Fix an integer sign bug in
 moduleTimerHandler (#8131)

bug was introduced in 1a91a2700b
---
 src/module.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/module.c b/src/module.c
index 15ab927cb..647272b59 100644
--- a/src/module.c
+++ b/src/module.c
@@ -5469,8 +5469,11 @@ int moduleTimerHandler(struct aeEventLoop *eventLoop, long long id, void *client
         } else {
             /* We call ustime() again instead of using the cached 'now' so that
              * 'next_period' isn't affected by the time it took to execute
-             * previous calls to 'callback. */
-            next_period = (expiretime-ustime())/1000; /* Scale to milliseconds. */
+             * previous calls to 'callback.
+             * We need to cast 'expiretime' so that the compiler will not treat
+             * the difference as unsigned (Causing next_period to be huge) in
+             * case expiretime < ustime() */
+            next_period = ((long long)expiretime-ustime())/1000; /* Scale to milliseconds. */
             break;
         }
     }
@@ -5519,7 +5522,8 @@ RedisModuleTimerID RM_CreateTimer(RedisModuleCtx *ctx, mstime_t period, RedisMod
 
     /* We need to install the main event loop timer if it's not already
      * installed, or we may need to refresh its period if we just installed
-     * a timer that will expire sooner than any other else. */
+     * a timer that will expire sooner than any other else (i.e. the timer
+     * we just installed is the first timer in the Timers rax). */
     if (aeTimer != -1) {
         raxIterator ri;
         raxStart(&ri,Timers);

From 75f9dec64455ed500277922baf7f371a7eb294a4 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuancode@163.com>
Date: Sun, 6 Dec 2020 17:53:04 +0800
Subject: [PATCH 0839/1098] Limit the main db and expires dictionaries to
 expand (#7954)

As we know, redis may reject user's requests or evict some keys if
used memory is over maxmemory. Dictionaries expanding may make
things worse, some big dictionaries, such as main db and expires dict,
may eat huge memory at once for allocating a new big hash table and be
far more than maxmemory after expanding.
There are related issues: #4213 #4583

More details, when expand dict in redis, we will allocate a new big
ht[1] that generally is double of ht[0], The size of ht[1] will be
very big if ht[0] already is big. For db dict, if we have more than
64 million keys, we need to cost 1GB for ht[1] when dict expands.

If the sum of used memory and new hash table of dict needed exceeds
maxmemory, we shouldn't allow the dict to expand. Because, if we
enable keys eviction, we still couldn't add much more keys after
eviction and rehashing, what's worse, redis will keep less keys when
redis only remains a little memory for storing new hash table instead
of users' data. Moreover users can't write data in redis if disable
keys eviction.

What this commit changed ?

Add a new member function expandAllowed for dict type, it provide a way
for caller to allow expand or not. We expose two parameters for this
function: more memory needed for expanding and dict current load factor,
users can implement a function to make a decision by them.
For main db dict and expires dict type, these dictionaries may be very
big and cost huge memory for expanding, so we implement a judgement
function: we can stop dict to expand provisionally if used memory will
be over maxmemory after dict expands, but to guarantee the performance
of redis, we still allow dict to expand if dict load factor exceeds the
safe load factor.
Add test cases to verify we don't allow main db to expand when left
memory is not enough, so that avoid keys eviction.

Other changes:

For new hash table size when expand. Before this commit, the size is
that double used of dict and later _dictNextPower. Actually we aim to
control a dict load factor between 0.5 and 1.0. Now we replace *2 with
+1, since the first check is that used >= size, the outcome of before
will usually be the same as _dictNextPower(used+1). The only case where
it'll differ is when dict_can_resize is false during fork, so that later
the _dictNextPower(used*2) will cause the dict to jump to *4 (i.e.
_dictNextPower(1025*2) will return 4096).
Fix rehash test cases due to changing algorithm of new hash table size
when expand.
---
 src/config.c                |  6 ++--
 src/db.c                    |  2 +-
 src/dict.c                  | 16 ++++++++--
 src/dict.h                  |  1 +
 src/evict.c                 | 14 +++++++++
 src/expire.c                |  3 +-
 src/latency.c               |  3 +-
 src/lazyfree.c              |  2 +-
 src/module.c                |  3 +-
 src/redis-benchmark.c       |  3 +-
 src/redis-cli.c             |  9 ++++--
 src/sentinel.c              |  9 ++++--
 src/server.c                | 60 +++++++++++++++++++++++++++----------
 src/server.h                |  4 ++-
 src/t_zset.c                |  3 +-
 tests/unit/maxmemory.tcl    | 19 ++++++++++++
 tests/unit/other.tcl        |  4 +--
 utils/hashtable/rehashing.c |  3 +-
 18 files changed, 127 insertions(+), 37 deletions(-)

diff --git a/src/config.c b/src/config.c
index 8dc241ed7..84b218cd7 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1085,7 +1085,8 @@ dictType optionToLineDictType = {
     NULL,                       /* val dup */
     dictSdsKeyCaseCompare,      /* key compare */
     dictSdsDestructor,          /* key destructor */
-    dictListDestructor          /* val destructor */
+    dictListDestructor,         /* val destructor */
+    NULL                        /* allow to expand */
 };
 
 dictType optionSetDictType = {
@@ -1094,7 +1095,8 @@ dictType optionSetDictType = {
     NULL,                       /* val dup */
     dictSdsKeyCaseCompare,      /* key compare */
     dictSdsDestructor,          /* key destructor */
-    NULL                        /* val destructor */
+    NULL,                       /* val destructor */
+    NULL                        /* allow to expand */
 };
 
 /* The config rewrite state. */
diff --git a/src/db.c b/src/db.c
index b52c6f3f2..45865b3e8 100644
--- a/src/db.c
+++ b/src/db.c
@@ -464,7 +464,7 @@ dbBackup *backupDb(void) {
     for (int i=0; i<server.dbnum; i++) {
         backup->dbarray[i] = server.db[i];
         server.db[i].dict = dictCreate(&dbDictType,NULL);
-        server.db[i].expires = dictCreate(&keyptrDictType,NULL);
+        server.db[i].expires = dictCreate(&dbExpiresDictType,NULL);
     }
 
     /* Backup cluster slots to keys map if enable cluster. */
diff --git a/src/dict.c b/src/dict.c
index 57287442a..cca5aa921 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -951,6 +951,16 @@ unsigned long dictScan(dict *d,
 
 /* ------------------------- private functions ------------------------------ */
 
+/* Because we may need to allocate huge memory chunk at once when dict
+ * expands, we will check this allocation is allowed or not if the dict
+ * type has expandAllowed member function. */
+static int dictTypeExpandAllowed(dict *d) {
+    if (d->type->expandAllowed == NULL) return 1;
+    return d->type->expandAllowed(
+                    _dictNextPower(d->ht[0].used + 1) * sizeof(dictEntry*),
+                    (double)d->ht[0].used / d->ht[0].size);
+}
+
 /* Expand the hash table if needed */
 static int _dictExpandIfNeeded(dict *d)
 {
@@ -966,9 +976,10 @@ static int _dictExpandIfNeeded(dict *d)
      * the number of buckets. */
     if (d->ht[0].used >= d->ht[0].size &&
         (dict_can_resize ||
-         d->ht[0].used/d->ht[0].size > dict_force_resize_ratio))
+         d->ht[0].used/d->ht[0].size > dict_force_resize_ratio) &&
+        dictTypeExpandAllowed(d))
     {
-        return dictExpand(d, d->ht[0].used*2);
+        return dictExpand(d, d->ht[0].used + 1);
     }
     return DICT_OK;
 }
@@ -1173,6 +1184,7 @@ dictType BenchmarkDictType = {
     NULL,
     compareCallback,
     freeCallback,
+    NULL,
     NULL
 };
 
diff --git a/src/dict.h b/src/dict.h
index dec60f637..fca924abe 100644
--- a/src/dict.h
+++ b/src/dict.h
@@ -62,6 +62,7 @@ typedef struct dictType {
     int (*keyCompare)(void *privdata, const void *key1, const void *key2);
     void (*keyDestructor)(void *privdata, void *key);
     void (*valDestructor)(void *privdata, void *obj);
+    int (*expandAllowed)(size_t moreMem, double usedRatio);
 } dictType;
 
 /* This is our hash table structure. Every dictionary has two of this as we
diff --git a/src/evict.c b/src/evict.c
index d14603aef..3de328fc0 100644
--- a/src/evict.c
+++ b/src/evict.c
@@ -412,6 +412,20 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev
     return C_ERR;
 }
 
+/* Return 1 if used memory is more than maxmemory after allocating more memory,
+ * return 0 if not. Redis may reject user's requests or evict some keys if used
+ * memory exceeds maxmemory, especially, when we allocate huge memory at once. */
+int overMaxmemoryAfterAlloc(size_t moremem) {
+    if (!server.maxmemory) return  0; /* No limit. */
+
+    /* Check quickly. */
+    size_t mem_used = zmalloc_used_memory();
+    if (mem_used + moremem <= server.maxmemory) return 0;
+
+    size_t overhead = freeMemoryGetNotCountedMemory();
+    mem_used = (mem_used > overhead) ? mem_used - overhead : 0;
+    return mem_used + moremem > server.maxmemory;
+}
 
 /* The evictionTimeProc is started when "maxmemory" has been breached and
  * could not immediately be resolved.  This will spin the event loop with short
diff --git a/src/expire.c b/src/expire.c
index cac208573..5433f46ca 100644
--- a/src/expire.c
+++ b/src/expire.c
@@ -433,7 +433,8 @@ void rememberSlaveKeyWithExpire(redisDb *db, robj *key) {
             NULL,                       /* val dup */
             dictSdsKeyCompare,          /* key compare */
             dictSdsDestructor,          /* key destructor */
-            NULL                        /* val destructor */
+            NULL,                       /* val destructor */
+            NULL                        /* allow to expand */
         };
         slaveKeysWithExpire = dictCreate(&dt,NULL);
     }
diff --git a/src/latency.c b/src/latency.c
index c661c5d6d..535e48c2b 100644
--- a/src/latency.c
+++ b/src/latency.c
@@ -53,7 +53,8 @@ dictType latencyTimeSeriesDictType = {
     NULL,                       /* val dup */
     dictStringKeyCompare,       /* key compare */
     dictVanillaFree,            /* key destructor */
-    dictVanillaFree             /* val destructor */
+    dictVanillaFree,            /* val destructor */
+    NULL                        /* allow to expand */
 };
 
 /* ------------------------- Utility functions ------------------------------ */
diff --git a/src/lazyfree.c b/src/lazyfree.c
index 641ab4e64..125e6a1b0 100644
--- a/src/lazyfree.c
+++ b/src/lazyfree.c
@@ -153,7 +153,7 @@ void freeObjAsync(robj *key, robj *obj) {
 void emptyDbAsync(redisDb *db) {
     dict *oldht1 = db->dict, *oldht2 = db->expires;
     db->dict = dictCreate(&dbDictType,NULL);
-    db->expires = dictCreate(&keyptrDictType,NULL);
+    db->expires = dictCreate(&dbExpiresDictType,NULL);
     atomicIncr(lazyfree_objects,dictSize(oldht1));
     bioCreateBackgroundJob(BIO_LAZY_FREE,NULL,oldht1,oldht2);
 }
diff --git a/src/module.c b/src/module.c
index 647272b59..515363583 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7552,7 +7552,8 @@ dictType moduleAPIDictType = {
     NULL,                      /* val dup */
     dictCStringKeyCompare,     /* key compare */
     NULL,                      /* key destructor */
-    NULL                       /* val destructor */
+    NULL,                      /* val destructor */
+    NULL                       /* allow to expand */
 };
 
 int moduleRegisterApi(const char *funcname, void *funcptr) {
diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index d8c62daa3..5567c4af5 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -1299,7 +1299,8 @@ static int fetchClusterSlotsConfiguration(client c) {
         NULL,                      /* val dup */
         dictSdsKeyCompare,         /* key compare */
         NULL,                      /* key destructor */
-        NULL                       /* val destructor */
+        NULL,                      /* val destructor */
+        NULL                       /* allow to expand */
     };
     /* printf("[%d] fetchClusterSlotsConfiguration\n", c->thread_id); */
     dict *masters = dictCreate(&dtype, NULL);
diff --git a/src/redis-cli.c b/src/redis-cli.c
index cd10d22e4..e140435d8 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -2294,7 +2294,8 @@ static dictType clusterManagerDictType = {
     NULL,                      /* val dup */
     dictSdsKeyCompare,         /* key compare */
     NULL,                      /* key destructor */
-    dictSdsDestructor          /* val destructor */
+    dictSdsDestructor,         /* val destructor */
+    NULL                       /* allow to expand */
 };
 
 static dictType clusterManagerLinkDictType = {
@@ -2303,7 +2304,8 @@ static dictType clusterManagerLinkDictType = {
     NULL,                      /* val dup */
     dictSdsKeyCompare,         /* key compare */
     dictSdsDestructor,         /* key destructor */
-    dictListDestructor         /* val destructor */
+    dictListDestructor,        /* val destructor */
+    NULL                       /* allow to expand */
 };
 
 typedef int clusterManagerCommandProc(int argc, char **argv);
@@ -7360,7 +7362,8 @@ static dictType typeinfoDictType = {
     NULL,                      /* val dup */
     dictSdsKeyCompare,         /* key compare */
     NULL,                      /* key destructor (owned by the value)*/
-    type_free                  /* val destructor */
+    type_free,                 /* val destructor */
+    NULL                       /* allow to expand */
 };
 
 static void getKeyTypes(dict *types_dict, redisReply *keys, typeinfo **types) {
diff --git a/src/sentinel.c b/src/sentinel.c
index 87fc20e9f..e05be4384 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -418,7 +418,8 @@ dictType instancesDictType = {
     NULL,                      /* val dup */
     dictSdsKeyCompare,         /* key compare */
     NULL,                      /* key destructor */
-    dictInstancesValDestructor /* val destructor */
+    dictInstancesValDestructor,/* val destructor */
+    NULL                       /* allow to expand */
 };
 
 /* Instance runid (sds) -> votes (long casted to void*)
@@ -431,7 +432,8 @@ dictType leaderVotesDictType = {
     NULL,                      /* val dup */
     dictSdsKeyCompare,         /* key compare */
     NULL,                      /* key destructor */
-    NULL                       /* val destructor */
+    NULL,                      /* val destructor */
+    NULL                       /* allow to expand */
 };
 
 /* Instance renamed commands table. */
@@ -441,7 +443,8 @@ dictType renamedCommandsDictType = {
     NULL,                      /* val dup */
     dictSdsKeyCaseCompare,     /* key compare */
     dictSdsDestructor,         /* key destructor */
-    dictSdsDestructor          /* val destructor */
+    dictSdsDestructor,         /* val destructor */
+    NULL                       /* allow to expand */
 };
 
 /* =========================== Initialization =============================== */
diff --git a/src/server.c b/src/server.c
index 16e7d3fba..bfd256310 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1298,6 +1298,20 @@ uint64_t dictEncObjHash(const void *key) {
     }
 }
 
+/* Return 1 if currently we allow dict to expand. Dict may allocate huge
+ * memory to contain hash buckets when dict expands, that may lead redis
+ * rejects user's requests or evicts some keys, we can stop dict to expand
+ * provisionally if used memory will be over maxmemory after dict expands,
+ * but to guarantee the performance of redis, we still allow dict to expand
+ * if dict load factor exceeds HASHTABLE_MAX_LOAD_FACTOR. */
+int dictExpandAllowed(size_t moreMem, double usedRatio) {
+    if (usedRatio <= HASHTABLE_MAX_LOAD_FACTOR) {
+        return !overMaxmemoryAfterAlloc(moreMem);
+    } else {
+        return 1;
+    }
+}
+
 /* Generic hash table type where keys are Redis Objects, Values
  * dummy pointers. */
 dictType objectKeyPointerValueDictType = {
@@ -1306,7 +1320,8 @@ dictType objectKeyPointerValueDictType = {
     NULL,                      /* val dup */
     dictEncObjKeyCompare,      /* key compare */
     dictObjectDestructor,      /* key destructor */
-    NULL                       /* val destructor */
+    NULL,                      /* val destructor */
+    NULL                       /* allow to expand */
 };
 
 /* Like objectKeyPointerValueDictType(), but values can be destroyed, if
@@ -1317,7 +1332,8 @@ dictType objectKeyHeapPointerValueDictType = {
     NULL,                      /* val dup */
     dictEncObjKeyCompare,      /* key compare */
     dictObjectDestructor,      /* key destructor */
-    dictVanillaFree            /* val destructor */
+    dictVanillaFree,           /* val destructor */
+    NULL                       /* allow to expand */
 };
 
 /* Set dictionary type. Keys are SDS strings, values are not used. */
@@ -1337,7 +1353,8 @@ dictType zsetDictType = {
     NULL,                      /* val dup */
     dictSdsKeyCompare,         /* key compare */
     NULL,                      /* Note: SDS string shared & freed by skiplist */
-    NULL                       /* val destructor */
+    NULL,                      /* val destructor */
+    NULL                       /* allow to expand */
 };
 
 /* Db->dict, keys are sds strings, vals are Redis objects. */
@@ -1347,7 +1364,8 @@ dictType dbDictType = {
     NULL,                       /* val dup */
     dictSdsKeyCompare,          /* key compare */
     dictSdsDestructor,          /* key destructor */
-    dictObjectDestructor   /* val destructor */
+    dictObjectDestructor,       /* val destructor */
+    dictExpandAllowed           /* allow to expand */
 };
 
 /* server.lua_scripts sha (as sds string) -> scripts (as robj) cache. */
@@ -1357,17 +1375,19 @@ dictType shaScriptObjectDictType = {
     NULL,                       /* val dup */
     dictSdsKeyCaseCompare,      /* key compare */
     dictSdsDestructor,          /* key destructor */
-    dictObjectDestructor        /* val destructor */
+    dictObjectDestructor,       /* val destructor */
+    NULL                        /* allow to expand */
 };
 
 /* Db->expires */
-dictType keyptrDictType = {
+dictType dbExpiresDictType = {
     dictSdsHash,                /* hash function */
     NULL,                       /* key dup */
     NULL,                       /* val dup */
     dictSdsKeyCompare,          /* key compare */
     NULL,                       /* key destructor */
-    NULL                        /* val destructor */
+    NULL,                       /* val destructor */
+    dictExpandAllowed           /* allow to expand */
 };
 
 /* Command table. sds string -> command struct pointer. */
@@ -1377,7 +1397,8 @@ dictType commandTableDictType = {
     NULL,                       /* val dup */
     dictSdsKeyCaseCompare,      /* key compare */
     dictSdsDestructor,          /* key destructor */
-    NULL                        /* val destructor */
+    NULL,                       /* val destructor */
+    NULL                        /* allow to expand */
 };
 
 /* Hash type hash table (note that small hashes are represented with ziplists) */
@@ -1387,7 +1408,8 @@ dictType hashDictType = {
     NULL,                       /* val dup */
     dictSdsKeyCompare,          /* key compare */
     dictSdsDestructor,          /* key destructor */
-    dictSdsDestructor           /* val destructor */
+    dictSdsDestructor,          /* val destructor */
+    NULL                        /* allow to expand */
 };
 
 /* Keylist hash table type has unencoded redis objects as keys and
@@ -1399,7 +1421,8 @@ dictType keylistDictType = {
     NULL,                       /* val dup */
     dictObjKeyCompare,          /* key compare */
     dictObjectDestructor,       /* key destructor */
-    dictListDestructor          /* val destructor */
+    dictListDestructor,         /* val destructor */
+    NULL                        /* allow to expand */
 };
 
 /* Cluster nodes hash table, mapping nodes addresses 1.2.3.4:6379 to
@@ -1410,7 +1433,8 @@ dictType clusterNodesDictType = {
     NULL,                       /* val dup */
     dictSdsKeyCompare,          /* key compare */
     dictSdsDestructor,          /* key destructor */
-    NULL                        /* val destructor */
+    NULL,                       /* val destructor */
+    NULL                        /* allow to expand */
 };
 
 /* Cluster re-addition blacklist. This maps node IDs to the time
@@ -1422,7 +1446,8 @@ dictType clusterNodesBlackListDictType = {
     NULL,                       /* val dup */
     dictSdsKeyCaseCompare,      /* key compare */
     dictSdsDestructor,          /* key destructor */
-    NULL                        /* val destructor */
+    NULL,                       /* val destructor */
+    NULL                        /* allow to expand */
 };
 
 /* Modules system dictionary type. Keys are module name,
@@ -1433,7 +1458,8 @@ dictType modulesDictType = {
     NULL,                       /* val dup */
     dictSdsKeyCaseCompare,      /* key compare */
     dictSdsDestructor,          /* key destructor */
-    NULL                        /* val destructor */
+    NULL,                       /* val destructor */
+    NULL                        /* allow to expand */
 };
 
 /* Migrate cache dict type. */
@@ -1443,7 +1469,8 @@ dictType migrateCacheDictType = {
     NULL,                       /* val dup */
     dictSdsKeyCompare,          /* key compare */
     dictSdsDestructor,          /* key destructor */
-    NULL                        /* val destructor */
+    NULL,                       /* val destructor */
+    NULL                        /* allow to expand */
 };
 
 /* Replication cached script dict (server.repl_scriptcache_dict).
@@ -1455,7 +1482,8 @@ dictType replScriptCacheDictType = {
     NULL,                       /* val dup */
     dictSdsKeyCaseCompare,      /* key compare */
     dictSdsDestructor,          /* key destructor */
-    NULL                        /* val destructor */
+    NULL,                       /* val destructor */
+    NULL                        /* allow to expand */
 };
 
 int htNeedsResize(dict *dict) {
@@ -3004,7 +3032,7 @@ void initServer(void) {
     /* Create the Redis databases, and initialize other internal state. */
     for (j = 0; j < server.dbnum; j++) {
         server.db[j].dict = dictCreate(&dbDictType,NULL);
-        server.db[j].expires = dictCreate(&keyptrDictType,NULL);
+        server.db[j].expires = dictCreate(&dbExpiresDictType,NULL);
         server.db[j].expires_cursor = 0;
         server.db[j].blocking_keys = dictCreate(&keylistDictType,NULL);
         server.db[j].ready_keys = dictCreate(&objectKeyPointerValueDictType,NULL);
diff --git a/src/server.h b/src/server.h
index a858d5833..772775bea 100644
--- a/src/server.h
+++ b/src/server.h
@@ -161,6 +161,7 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
 
 /* Hash table parameters */
 #define HASHTABLE_MIN_FILL        10      /* Minimal hash table fill 10% */
+#define HASHTABLE_MAX_LOAD_FACTOR 1.618   /* Maximum hash table load factor. */
 
 /* Command flags. Please check the command table defined in the server.c file
  * for more information about the meaning of every flag. */
@@ -1638,7 +1639,7 @@ extern dictType shaScriptObjectDictType;
 extern double R_Zero, R_PosInf, R_NegInf, R_Nan;
 extern dictType hashDictType;
 extern dictType replScriptCacheDictType;
-extern dictType keyptrDictType;
+extern dictType dbExpiresDictType;
 extern dictType modulesDictType;
 
 /*-----------------------------------------------------------------------------
@@ -2070,6 +2071,7 @@ int zslLexValueLteMax(sds value, zlexrangespec *spec);
 /* Core functions */
 int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *level);
 size_t freeMemoryGetNotCountedMemory();
+int overMaxmemoryAfterAlloc(size_t moremem);
 int processCommand(client *c);
 void setupSignalHandlers(void);
 void removeSignalHandlers(void);
diff --git a/src/t_zset.c b/src/t_zset.c
index 382d3b9ce..0b3cc061e 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -2439,7 +2439,8 @@ dictType setAccumulatorDictType = {
     NULL,                      /* val dup */
     dictSdsKeyCompare,         /* key compare */
     NULL,                      /* key destructor */
-    NULL                       /* val destructor */
+    NULL,                      /* val destructor */
+    NULL                       /* allow to expand */
 };
 
 /* The zunionInterDiffGenericCommand() function is called in order to implement the
diff --git a/tests/unit/maxmemory.tcl b/tests/unit/maxmemory.tcl
index 0f64ddc18..19983de9b 100644
--- a/tests/unit/maxmemory.tcl
+++ b/tests/unit/maxmemory.tcl
@@ -241,3 +241,22 @@ test_slave_buffers {slave buffer are counted correctly} 1000000 10 0 1
 # test again with fewer (and bigger) commands without pipeline, but with eviction
 test_slave_buffers "replica buffer don't induce eviction" 100000 100 1 0
 
+start_server {tags {"maxmemory"}} {
+    test {Don't rehash if used memory exceeds maxmemory after rehash} {
+        r config set maxmemory 0
+        r config set maxmemory-policy allkeys-random
+
+        # Next rehash size is 8192, that will eat 64k memory
+        populate 4096 "" 1
+
+        set used [s used_memory]
+        set limit [expr {$used + 10*1024}]
+        r config set maxmemory $limit
+        r set k1 v1
+        # Next writing command will trigger evicting some keys if last
+        # command trigger DB dict rehash
+        r set k2 v2
+        # There must be 4098 keys because redis doesn't evict keys.
+        r dbsize
+    } {4098}
+}
diff --git a/tests/unit/other.tcl b/tests/unit/other.tcl
index 96e9ee3da..d98dc1bd4 100644
--- a/tests/unit/other.tcl
+++ b/tests/unit/other.tcl
@@ -316,8 +316,8 @@ start_server {tags {"other"}} {
         after 200
 
         # Hash table should rehash since there is no child process,
-        # size is power of two and over 4098, so it is 16384
+        # size is power of two and over 4098, so it is 8192
         r set k3 v3
-        assert_match "*table size: 16384*" [r debug HTSTATS 9]
+        assert_match "*table size: 8192*" [r debug HTSTATS 9]
     }
 }
diff --git a/utils/hashtable/rehashing.c b/utils/hashtable/rehashing.c
index b57a9043a..3c0acb84c 100644
--- a/utils/hashtable/rehashing.c
+++ b/utils/hashtable/rehashing.c
@@ -30,7 +30,8 @@ dictType dictTypeTest = {
     NULL,                          /* val dup */
     dictKeyCompare,                /* key compare */
     NULL,                          /* key destructor */
-    NULL                           /* val destructor */
+    NULL,                          /* val destructor */
+    NULL                           /* allow to expand */
 };
 
 void showBuckets(dictht ht) {

From 1df5bb5687738aae390ee4f6573d975adc3afc96 Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Sun, 6 Dec 2020 12:14:18 +0100
Subject: [PATCH 0840/1098] Make sure we do not propagate nested MULTI/EXEC
 (#8097)

One way this was happening is when a module issued an RM_Call which would inject MULTI.
If the module command that does that was itself issued by something else that already did
added MULTI (e.g. another module, or a Lua script), it would have caused nested MULTI.

In fact the MULTI state in the client or the MULTI_EMITTED flag in the context isn't
the right indication that we need to propagate MULTI or not, because on a nested calls
(possibly a module action called by a keyspace event of another module action), these
flags aren't retained / reflected.

instead there's now a global propagate_in_transaction flag for that.

in addition to that, we now have a global in_eval and in_exec flags, to serve the flags
of RM_GetContextFlags, since their dependence on the current client is wrong for the same
reasons mentioned above.
---
 src/module.c                             |  47 +++++-----
 src/multi.c                              |  20 +++++
 src/scripting.c                          |   3 +
 src/server.c                             |   3 +
 src/server.h                             |   4 +
 tests/modules/keyspace_events.c          | 109 ++++++++++++++++++++++-
 tests/unit/moduleapi/keyspace_events.tcl |  51 ++++++++++-
 7 files changed, 212 insertions(+), 25 deletions(-)

diff --git a/src/module.c b/src/module.c
index 515363583..b5921ed55 100644
--- a/src/module.c
+++ b/src/module.c
@@ -158,14 +158,13 @@ struct RedisModuleCtx {
 typedef struct RedisModuleCtx RedisModuleCtx;
 
 #define REDISMODULE_CTX_INIT {(void*)(unsigned long)&RM_GetApi, NULL, NULL, NULL, NULL, 0, 0, 0, NULL, 0, NULL, NULL, NULL, NULL, {0}}
-#define REDISMODULE_CTX_MULTI_EMITTED (1<<0)
-#define REDISMODULE_CTX_AUTO_MEMORY (1<<1)
-#define REDISMODULE_CTX_KEYS_POS_REQUEST (1<<2)
-#define REDISMODULE_CTX_BLOCKED_REPLY (1<<3)
-#define REDISMODULE_CTX_BLOCKED_TIMEOUT (1<<4)
-#define REDISMODULE_CTX_THREAD_SAFE (1<<5)
-#define REDISMODULE_CTX_BLOCKED_DISCONNECTED (1<<6)
-#define REDISMODULE_CTX_MODULE_COMMAND_CALL (1<<7)
+#define REDISMODULE_CTX_AUTO_MEMORY (1<<0)
+#define REDISMODULE_CTX_KEYS_POS_REQUEST (1<<1)
+#define REDISMODULE_CTX_BLOCKED_REPLY (1<<2)
+#define REDISMODULE_CTX_BLOCKED_TIMEOUT (1<<3)
+#define REDISMODULE_CTX_THREAD_SAFE (1<<4)
+#define REDISMODULE_CTX_BLOCKED_DISCONNECTED (1<<5)
+#define REDISMODULE_CTX_MODULE_COMMAND_CALL (1<<6)
 
 /* This represents a Redis key opened with RM_OpenKey(). */
 struct RedisModuleKey {
@@ -578,12 +577,15 @@ void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) {
 
     /* We don't need to do anything here if the context was never used
      * in order to propagate commands. */
-    if (!(ctx->flags & REDISMODULE_CTX_MULTI_EMITTED)) return;
+    if (!server.propagate_in_transaction) return;
 
-    if (c->flags & CLIENT_LUA) return;
+    /* If this command is executed from with Lua or MULTI/EXEC we do noy
+     * need to propagate EXEC */
+    if (server.in_eval || server.in_exec) return;
 
     /* Handle the replication of the final EXEC, since whatever a command
      * emits is always wrapped around MULTI/EXEC. */
+    beforePropagateMultiOrExec(0);
     alsoPropagate(server.execCommand,c->db->id,&shared.exec,1,
         PROPAGATE_AOF|PROPAGATE_REPL);
 
@@ -1610,9 +1612,9 @@ int RM_ReplyWithLongDouble(RedisModuleCtx *ctx, long double ld) {
 void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) {
     /* Skip this if client explicitly wrap the command with MULTI, or if
      * the module command was called by a script. */
-    if (ctx->client->flags & (CLIENT_MULTI|CLIENT_LUA)) return;
+    if (server.lua_caller || server.in_exec) return;
     /* If we already emitted MULTI return ASAP. */
-    if (ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) return;
+    if (server.propagate_in_transaction) return;
     /* If this is a thread safe context, we do not want to wrap commands
      * executed into MULTI/EXEC, they are executed as single commands
      * from an external client in essence. */
@@ -1625,7 +1627,6 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) {
         redisOpArrayInit(&server.also_propagate);
     }
     execCommandPropagateMulti(ctx->client);
-    ctx->flags |= REDISMODULE_CTX_MULTI_EMITTED;
 }
 
 /* Replicate the specified command and arguments to slaves and AOF, as effect
@@ -1938,20 +1939,16 @@ int RM_GetSelectedDb(RedisModuleCtx *ctx) {
  *                                    background child process.
  */
 int RM_GetContextFlags(RedisModuleCtx *ctx) {
-
     int flags = 0;
+
     /* Client specific flags */
     if (ctx) {
         if (ctx->client) {
-            if (ctx->client->flags & CLIENT_LUA)
-             flags |= REDISMODULE_CTX_FLAGS_LUA;
-            if (ctx->client->flags & CLIENT_MULTI)
-             flags |= REDISMODULE_CTX_FLAGS_MULTI;
             if (ctx->client->flags & CLIENT_DENY_BLOCKING)
-             flags |= REDISMODULE_CTX_FLAGS_DENY_BLOCKING;
+                flags |= REDISMODULE_CTX_FLAGS_DENY_BLOCKING;
             /* Module command received from MASTER, is replicated. */
             if (ctx->client->flags & CLIENT_MASTER)
-             flags |= REDISMODULE_CTX_FLAGS_REPLICATED;
+                flags |= REDISMODULE_CTX_FLAGS_REPLICATED;
         }
 
         /* For DIRTY flags, we need the blocked client if used */
@@ -1961,6 +1958,12 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) {
         }
     }
 
+    if (server.in_eval)
+        flags |= REDISMODULE_CTX_FLAGS_LUA;
+
+    if (server.in_exec)
+        flags |= REDISMODULE_CTX_FLAGS_MULTI;
+
     if (server.cluster_enabled)
         flags |= REDISMODULE_CTX_FLAGS_CLUSTER;
 
@@ -4497,8 +4500,8 @@ void unblockClientFromModule(client *c) {
  */
 RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata) {
     client *c = ctx->client;
-    int islua = c->flags & CLIENT_LUA;
-    int ismulti = c->flags & CLIENT_MULTI;
+    int islua = server.in_eval;
+    int ismulti = server.in_exec;
 
     c->bpop.module_blocked_handle = zmalloc(sizeof(RedisModuleBlockedClient));
     RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle;
diff --git a/src/multi.c b/src/multi.c
index c519828c4..a2f9ecccf 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -113,14 +113,28 @@ void discardCommand(client *c) {
     addReply(c,shared.ok);
 }
 
+void beforePropagateMultiOrExec(int multi) {
+    if (multi) {
+        /* Propagating MULTI */
+        serverAssert(!server.propagate_in_transaction);
+        server.propagate_in_transaction = 1;
+    } else {
+        /* Propagating EXEC */
+        serverAssert(server.propagate_in_transaction == 1);
+        server.propagate_in_transaction = 0;
+    }
+}
+
 /* Send a MULTI command to all the slaves and AOF file. Check the execCommand
  * implementation for more information. */
 void execCommandPropagateMulti(client *c) {
+    beforePropagateMultiOrExec(1);
     propagate(server.multiCommand,c->db->id,&shared.multi,1,
               PROPAGATE_AOF|PROPAGATE_REPL);
 }
 
 void execCommandPropagateExec(client *c) {
+    beforePropagateMultiOrExec(0);
     propagate(server.execCommand,c->db->id,&shared.exec,1,
               PROPAGATE_AOF|PROPAGATE_REPL);
 }
@@ -176,6 +190,9 @@ void execCommand(client *c) {
 
     /* Exec all the queued commands */
     unwatchAllKeys(c); /* Unwatch ASAP otherwise we'll waste CPU cycles */
+
+    server.in_exec = 1;
+
     orig_argv = c->argv;
     orig_argc = c->argc;
     orig_cmd = c->cmd;
@@ -251,6 +268,7 @@ void execCommand(client *c) {
     if (must_propagate) {
         int is_master = server.masterhost == NULL;
         server.dirty++;
+        beforePropagateMultiOrExec(0);
         /* If inside the MULTI/EXEC block this instance was suddenly
          * switched from master to slave (using the SLAVEOF command), the
          * initial MULTI was propagated into the replication backlog, but the
@@ -262,6 +280,8 @@ void execCommand(client *c) {
         }
     }
 
+    server.in_exec = 0;
+
 handle_monitor:
     /* Send EXEC to clients waiting data from MONITOR. We do it here
      * since the natural order of commands execution is actually:
diff --git a/src/scripting.c b/src/scripting.c
index 3ed3e2f64..8dca84478 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -1498,6 +1498,7 @@ void evalGenericCommand(client *c, int evalsha) {
     server.lua_replicate_commands = server.lua_always_replicate_commands;
     server.lua_multi_emitted = 0;
     server.lua_repl = PROPAGATE_AOF|PROPAGATE_REPL;
+    server.in_eval = 1;
 
     /* Get the number of arguments that are keys */
     if (getLongLongFromObjectOrReply(c,c->argv[2],&numkeys,NULL) != C_OK)
@@ -1678,6 +1679,8 @@ void evalGenericCommand(client *c, int evalsha) {
             forceCommandPropagation(c,PROPAGATE_REPL|PROPAGATE_AOF);
         }
     }
+
+    server.in_eval = 0;
 }
 
 void evalCommand(client *c) {
diff --git a/src/server.c b/src/server.c
index bfd256310..c30577da4 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3049,6 +3049,9 @@ void initServer(void) {
     listSetFreeMethod(server.pubsub_patterns,freePubsubPattern);
     listSetMatchMethod(server.pubsub_patterns,listMatchPubsubPattern);
     server.cronloops = 0;
+    server.in_eval = 0;
+    server.in_exec = 0;
+    server.propagate_in_transaction = 0;
     server.rdb_child_pid = -1;
     server.aof_child_pid = -1;
     server.module_child_pid = -1;
diff --git a/src/server.h b/src/server.h
index 772775bea..5bbe7876c 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1116,6 +1116,9 @@ struct redisServer {
     int sentinel_mode;          /* True if this instance is a Sentinel. */
     size_t initial_memory_usage; /* Bytes used after initialization. */
     int always_show_logo;       /* Show logo even for non-stdout logging. */
+    int in_eval;                /* Are we inside EVAL? */
+    int in_exec;                /* Are we inside EXEC? */
+    int propagate_in_transaction;  /* Make sure we don't propagate nested MULTI/EXEC */
     /* Modules */
     dict *moduleapi;            /* Exported core APIs dictionary for modules. */
     dict *sharedapi;            /* Like moduleapi but containing the APIs that
@@ -1834,6 +1837,7 @@ void flagTransaction(client *c);
 void execCommandAbort(client *c, sds error);
 void execCommandPropagateMulti(client *c);
 void execCommandPropagateExec(client *c);
+void beforePropagateMultiOrExec(int multi);
 
 /* Redis object implementation */
 void decrRefCount(robj *o);
diff --git a/tests/modules/keyspace_events.c b/tests/modules/keyspace_events.c
index fac7edd1f..be259d738 100644
--- a/tests/modules/keyspace_events.c
+++ b/tests/modules/keyspace_events.c
@@ -39,7 +39,7 @@
 /** strores all the keys on which we got 'loaded' keyspace notification **/
 RedisModuleDict *loaded_event_log = NULL;
 
-static int KeySpace_Notification(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key){
+static int KeySpace_NotificationLoaded(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key){
     REDISMODULE_NOT_USED(ctx);
     REDISMODULE_NOT_USED(type);
 
@@ -55,6 +55,29 @@ static int KeySpace_Notification(RedisModuleCtx *ctx, int type, const char *even
     return REDISMODULE_OK;
 }
 
+static int KeySpace_NotificationGeneric(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key) {
+    REDISMODULE_NOT_USED(type);
+
+    if (strcmp(event, "del") == 0) {
+        RedisModuleString *copykey = RedisModule_CreateStringPrintf(ctx, "%s_copy", RedisModule_StringPtrLen(key, NULL));
+        RedisModuleCallReply* rep = RedisModule_Call(ctx, "DEL", "s!", copykey);
+        RedisModule_FreeString(ctx, copykey);
+        RedisModule_FreeCallReply(rep);
+
+        int ctx_flags = RedisModule_GetContextFlags(ctx);
+        if (ctx_flags & REDISMODULE_CTX_FLAGS_LUA) {
+            RedisModuleCallReply* rep = RedisModule_Call(ctx, "INCR", "c", "lua");
+            RedisModule_FreeCallReply(rep);
+        }
+        if (ctx_flags & REDISMODULE_CTX_FLAGS_MULTI) {
+            RedisModuleCallReply* rep = RedisModule_Call(ctx, "INCR", "c", "multi");
+            RedisModule_FreeCallReply(rep);
+        }
+    }
+
+    return REDISMODULE_OK;
+}
+
 static int cmdIsKeyLoaded(RedisModuleCtx *ctx, RedisModuleString **argv, int argc){
     if(argc != 2){
         return RedisModule_WrongArity(ctx);
@@ -75,6 +98,68 @@ static int cmdIsKeyLoaded(RedisModuleCtx *ctx, RedisModuleString **argv, int arg
     return REDISMODULE_OK;
 }
 
+static int cmdDelKeyCopy(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 2)
+        return RedisModule_WrongArity(ctx);
+
+    RedisModuleCallReply* rep = RedisModule_Call(ctx, "DEL", "s!", argv[1]);
+    if (!rep) {
+        RedisModule_ReplyWithError(ctx, "NULL reply returned");
+    } else {
+        RedisModule_ReplyWithCallReply(ctx, rep);
+        RedisModule_FreeCallReply(rep);
+    }
+    return REDISMODULE_OK;
+}
+
+/* Call INCR and propagate using RM_Call with `!`. */
+static int cmdIncrCase1(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 2)
+        return RedisModule_WrongArity(ctx);
+
+    RedisModuleCallReply* rep = RedisModule_Call(ctx, "INCR", "s!", argv[1]);
+    if (!rep) {
+        RedisModule_ReplyWithError(ctx, "NULL reply returned");
+    } else {
+        RedisModule_ReplyWithCallReply(ctx, rep);
+        RedisModule_FreeCallReply(rep);
+    }
+    return REDISMODULE_OK;
+}
+
+/* Call INCR and propagate using RM_Replicate. */
+static int cmdIncrCase2(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 2)
+        return RedisModule_WrongArity(ctx);
+
+    RedisModuleCallReply* rep = RedisModule_Call(ctx, "INCR", "s", argv[1]);
+    if (!rep) {
+        RedisModule_ReplyWithError(ctx, "NULL reply returned");
+    } else {
+        RedisModule_ReplyWithCallReply(ctx, rep);
+        RedisModule_FreeCallReply(rep);
+    }
+    RedisModule_Replicate(ctx, "INCR", "s", argv[1]);
+    return REDISMODULE_OK;
+}
+
+/* Call INCR and propagate using RM_ReplicateVerbatim. */
+static int cmdIncrCase3(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 2)
+        return RedisModule_WrongArity(ctx);
+
+    RedisModuleCallReply* rep = RedisModule_Call(ctx, "INCR", "s", argv[1]);
+    if (!rep) {
+        RedisModule_ReplyWithError(ctx, "NULL reply returned");
+    } else {
+        RedisModule_ReplyWithCallReply(ctx, rep);
+        RedisModule_FreeCallReply(rep);
+    }
+    RedisModule_ReplicateVerbatim(ctx);
+    return REDISMODULE_OK;
+}
+
+
 /* This function must be present on each Redis module. It is used in order to
  * register the commands into the Redis server. */
 int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
@@ -94,7 +179,11 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
         return REDISMODULE_ERR;
     }
 
-    if(RedisModule_SubscribeToKeyspaceEvents(ctx, REDISMODULE_NOTIFY_LOADED, KeySpace_Notification) != REDISMODULE_OK){
+    if(RedisModule_SubscribeToKeyspaceEvents(ctx, REDISMODULE_NOTIFY_LOADED, KeySpace_NotificationLoaded) != REDISMODULE_OK){
+        return REDISMODULE_ERR;
+    }
+
+    if(RedisModule_SubscribeToKeyspaceEvents(ctx, REDISMODULE_NOTIFY_GENERIC, KeySpace_NotificationGeneric) != REDISMODULE_OK){
         return REDISMODULE_ERR;
     }
 
@@ -102,6 +191,22 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
         return REDISMODULE_ERR;
     }
 
+    if (RedisModule_CreateCommand(ctx,"keyspace.del_key_copy", cmdDelKeyCopy,"",0,0,0) == REDISMODULE_ERR){
+        return REDISMODULE_ERR;
+    }
+    
+    if (RedisModule_CreateCommand(ctx,"keyspace.incr_case1", cmdIncrCase1,"",0,0,0) == REDISMODULE_ERR){
+        return REDISMODULE_ERR;
+    }
+    
+    if (RedisModule_CreateCommand(ctx,"keyspace.incr_case2", cmdIncrCase2,"",0,0,0) == REDISMODULE_ERR){
+        return REDISMODULE_ERR;
+    }
+    
+    if (RedisModule_CreateCommand(ctx,"keyspace.incr_case3", cmdIncrCase3,"",0,0,0) == REDISMODULE_ERR){
+        return REDISMODULE_ERR;
+    }
+
     return REDISMODULE_OK;
 }
 
diff --git a/tests/unit/moduleapi/keyspace_events.tcl b/tests/unit/moduleapi/keyspace_events.tcl
index 5b3db0c0a..293a62e4e 100644
--- a/tests/unit/moduleapi/keyspace_events.tcl
+++ b/tests/unit/moduleapi/keyspace_events.tcl
@@ -18,5 +18,54 @@ tags "modules" {
             assert_equal {1 t} [r keyspace.is_key_loaded t]
             assert_equal {1 s} [r keyspace.is_key_loaded s]
         }
+
+        test {Nested multi due to RM_Call} {
+            r del multi
+            r del lua
+
+            r set x 1
+            r set x_copy 1
+            r keyspace.del_key_copy x
+            r keyspace.incr_case1 x
+            r keyspace.incr_case2 x
+            r keyspace.incr_case3 x
+            assert_equal {} [r get multi]
+            assert_equal {} [r get lua]
+            r get x
+        } {3}
+        
+        test {Nested multi due to RM_Call, with client MULTI} {
+            r del multi
+            r del lua
+
+            r set x 1
+            r set x_copy 1
+            r multi
+            r keyspace.del_key_copy x
+            r keyspace.incr_case1 x
+            r keyspace.incr_case2 x
+            r keyspace.incr_case3 x
+            r exec
+            assert_equal {1} [r get multi]
+            assert_equal {} [r get lua]
+            r get x
+        } {3}
+        
+        test {Nested multi due to RM_Call, with EVAL} {
+            r del multi
+            r del lua
+
+            r set x 1
+            r set x_copy 1
+            r eval {
+                redis.pcall('keyspace.del_key_copy', KEYS[1])
+                redis.pcall('keyspace.incr_case1', KEYS[1])
+                redis.pcall('keyspace.incr_case2', KEYS[1])
+                redis.pcall('keyspace.incr_case3', KEYS[1])
+            } 1 x
+            assert_equal {} [r get multi]
+            assert_equal {1} [r get lua]
+            r get x
+        } {3}
 	}
-}
\ No newline at end of file
+}

From c4fdf09c0584a3cee32b92f01b7958c72776aedc Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 6 Dec 2020 14:51:22 +0200
Subject: [PATCH 0841/1098] prevent client tracking from causing feedback loop
 in performEvictions (#8100)

When client tracking is enabled signalModifiedKey can increase memory usage,
this can cause the loop in performEvictions to keep running since it was measuring
the memory usage impact of signalModifiedKey.

The section that measures the memory impact of the eviction should be just on dbDelete,
excluding keyspace notification, client tracking, and propagation to AOF and replicas.

This resolves part of the problem described in #8069
p.s. fix took 1 minute, test took about 3 hours to write.
---
 src/evict.c              |  4 +++-
 tests/unit/maxmemory.tcl | 52 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/src/evict.c b/src/evict.c
index 3de328fc0..3642352ad 100644
--- a/src/evict.c
+++ b/src/evict.c
@@ -629,6 +629,8 @@ int performEvictions(void) {
              * we are freeing removing the key, but we can't account for
              * that otherwise we would never exit the loop.
              *
+             * Same for CSC invalidation messages generated by signalModifiedKey.
+             *
              * AOF and Output buffer memory will be freed eventually so
              * we only care about memory used by the key space. */
             delta = (long long) zmalloc_used_memory();
@@ -637,12 +639,12 @@ int performEvictions(void) {
                 dbAsyncDelete(db,keyobj);
             else
                 dbSyncDelete(db,keyobj);
-            signalModifiedKey(NULL,db,keyobj);
             latencyEndMonitor(eviction_latency);
             latencyAddSampleIfNeeded("eviction-del",eviction_latency);
             delta -= (long long) zmalloc_used_memory();
             mem_freed += delta;
             server.stat_evictedkeys++;
+            signalModifiedKey(NULL,db,keyobj);
             notifyKeyspaceEvent(NOTIFY_EVICTED, "evicted",
                 keyobj, db->id);
             decrRefCount(keyobj);
diff --git a/tests/unit/maxmemory.tcl b/tests/unit/maxmemory.tcl
index 19983de9b..d4ad3ff03 100644
--- a/tests/unit/maxmemory.tcl
+++ b/tests/unit/maxmemory.tcl
@@ -260,3 +260,55 @@ start_server {tags {"maxmemory"}} {
         r dbsize
     } {4098}
 }
+
+start_server {tags {"maxmemory"}} {
+    test {client tracking don't cause eviction feedback loop} {
+        r config set maxmemory 0
+        r config set maxmemory-policy allkeys-lru
+        r config set maxmemory-eviction-tenacity 100
+
+        # 10 clients listening on tracking messages
+        set clients {}
+        for {set j 0} {$j < 10} {incr j} {
+            lappend clients [redis_deferring_client]
+        }
+        foreach rd $clients {
+            $rd HELLO 3
+            $rd read ; # Consume the HELLO reply
+            $rd CLIENT TRACKING on
+            $rd read ; # Consume the CLIENT reply
+        }
+
+        # populate 300 keys, with long key name and short value
+        for {set j 0} {$j < 300} {incr j} {
+            set key $j[string repeat x 1000]
+            r set $key x
+
+            # for each key, enable caching for this key
+            foreach rd $clients {
+                $rd get $key
+                $rd read
+            }
+        }
+
+        # set the memory limit which will cause a few keys to be evicted
+        # we need to make sure to evict keynames of a total size of more than
+        # 16kb since the (PROTO_REPLY_CHUNK_BYTES), only after that the
+        # invalidation messages have a chance to trigger further eviction.
+        set used [s used_memory]
+        set limit [expr {$used - 40000}]
+        r config set maxmemory $limit
+
+        # make sure some eviction happened
+        set evicted [s evicted_keys]
+        if {$::verbose} { puts "evicted: $evicted" }
+        assert_range $evicted 10 50
+        foreach rd $clients {
+            $rd read ;# make sure we have some invalidation message waiting
+            $rd close
+        }
+
+        # make sure we didn't drain the database
+        assert_range [r dbsize] 200 300
+    }
+}

From ca1c182567add4092e9cb6ea829e9c5193e8fd55 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 13 Aug 2020 16:41:05 +0300
Subject: [PATCH 0842/1098] Sanitize dump payload: ziplist, listpack, zipmap,
 intset, stream

When loading an encoded payload we will at least do a shallow validation to
check that the size that's encoded in the payload matches the size of the
allocation.
This let's us later use this encoded size to make sure the various offsets
inside encoded payload don't reach outside the allocation, if they do, we'll
assert/panic, but at least we won't segfault or smear memory.

We can also do 'deep' validation which runs on all the records of the encoded
payload and validates that they don't contain invalid offsets. This lets us
detect corruptions early and reject a RESTORE command rather than accepting
it and asserting (crashing) later when accessing that payload via some command.

configuration:
- adding ACL flag skip-sanitize-payload
- adding config sanitize-dump-payload [yes/no/clients]

For now, we don't have a good way to ensure MIGRATE in cluster resharding isn't
being slowed down by these sanitation, so i'm setting the default value to `no`,
but later on it should be set to `clients` by default.

changes:
- changing rdbReportError not to `exit` in RESTORE command
- adding a new stat to be able to later check if cluster MIGRATE isn't being
  slowed down by sanitation.
---
 redis.conf                            |  17 ++
 src/acl.c                             |  10 +
 src/config.c                          |   8 +
 src/defrag.c                          |   1 -
 src/intset.c                          |  27 +++
 src/intset.h                          |   1 +
 src/listpack.c                        | 146 ++++++++++++-
 src/listpack.h                        |   2 +
 src/rdb.c                             |  89 ++++++--
 src/redis-check-rdb.c                 |   1 +
 src/server.c                          |   3 +
 src/server.h                          |  13 ++
 src/stream.h                          |   1 +
 src/t_hash.c                          |   6 +-
 src/t_stream.c                        | 109 +++++++++-
 src/ziplist.c                         | 298 +++++++++++++++++++++-----
 src/ziplist.h                         |   3 +-
 src/zipmap.c                          |  68 ++++++
 src/zipmap.h                          |   1 +
 tests/assets/corrupt_ziplist.rdb      | Bin 0 -> 1415 bytes
 tests/cluster/tests/04-resharding.tcl |   7 +
 tests/integration/corrupt-dump.tcl    | 178 +++++++++++++++
 tests/support/util.tcl                |  16 ++
 tests/test_helper.tcl                 |   1 +
 24 files changed, 916 insertions(+), 90 deletions(-)
 create mode 100644 tests/assets/corrupt_ziplist.rdb
 create mode 100644 tests/integration/corrupt-dump.tcl

diff --git a/redis.conf b/redis.conf
index b11459b26..13766171e 100644
--- a/redis.conf
+++ b/redis.conf
@@ -366,6 +366,21 @@ rdbcompression yes
 # tell the loading code to skip the check.
 rdbchecksum yes
 
+# Enables or disables full sanitation checks for ziplist and listpack etc when
+# loading an RDB or RESTORE payload. This reduces the chances of a assertion or
+# crash later on while processing commands.
+# Options:
+#   no         - Never perform full sanitation
+#   yes        - Always perform full sanitation
+#   clients    - Perform full sanitation only for user connections.
+#                Excludes: RDB files, RESTORE commands received from the master
+#                connection, and client connections which have the
+#                skip-sanitize-payload ACL flag.
+# The default should be 'clients' but since it currently affects cluster
+# resharding via MIGRATE, it is temporarily set to 'no' by default.
+#
+# sanitize-dump-payload no
+
 # The filename where to dump the DB
 dbfilename dump.rdb
 
@@ -725,6 +740,8 @@ replica-priority 100
 #  off          Disable the user: it's no longer possible to authenticate
 #               with this user, however the already authenticated connections
 #               will still work.
+#  skip-sanitize-payload    RESTORE dump-payload sanitation is skipped.
+#  sanitize-payload         RESTORE dump-payload is sanitized (default).
 #  +<command>   Allow the execution of that command
 #  -<command>   Disallow the execution of that command
 #  +@<category> Allow the execution of all the commands in such category
diff --git a/src/acl.c b/src/acl.c
index e9acc11a9..a1a7c4237 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -89,12 +89,15 @@ struct ACLUserFlag {
     const char *name;
     uint64_t flag;
 } ACLUserFlags[] = {
+    /* Note: the order here dictates the emitted order at ACLDescribeUser */
     {"on", USER_FLAG_ENABLED},
     {"off", USER_FLAG_DISABLED},
     {"allkeys", USER_FLAG_ALLKEYS},
     {"allchannels", USER_FLAG_ALLCHANNELS},
     {"allcommands", USER_FLAG_ALLCOMMANDS},
     {"nopass", USER_FLAG_NOPASS},
+    {"skip-sanitize-payload", USER_FLAG_SANITIZE_PAYLOAD_SKIP},
+    {"sanitize-payload", USER_FLAG_SANITIZE_PAYLOAD},
     {NULL,0} /* Terminator. */
 };
 
@@ -829,6 +832,12 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
     } else if (!strcasecmp(op,"off")) {
         u->flags |= USER_FLAG_DISABLED;
         u->flags &= ~USER_FLAG_ENABLED;
+    } else if (!strcasecmp(op,"skip-sanitize-payload")) {
+        u->flags |= USER_FLAG_SANITIZE_PAYLOAD_SKIP;
+        u->flags &= ~USER_FLAG_SANITIZE_PAYLOAD;
+    } else if (!strcasecmp(op,"sanitize-payload")) {
+        u->flags &= ~USER_FLAG_SANITIZE_PAYLOAD_SKIP;
+        u->flags |= USER_FLAG_SANITIZE_PAYLOAD;
     } else if (!strcasecmp(op,"allkeys") ||
                !strcasecmp(op,"~*"))
     {
@@ -1004,6 +1013,7 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
         serverAssert(ACLSetUser(u,"resetkeys",-1) == C_OK);
         serverAssert(ACLSetUser(u,"resetchannels",-1) == C_OK);
         serverAssert(ACLSetUser(u,"off",-1) == C_OK);
+        serverAssert(ACLSetUser(u,"sanitize-payload",-1) == C_OK);
         serverAssert(ACLSetUser(u,"-@all",-1) == C_OK);
     } else {
         errno = EINVAL;
diff --git a/src/config.c b/src/config.c
index 84b218cd7..5fdde44cd 100644
--- a/src/config.c
+++ b/src/config.c
@@ -119,6 +119,13 @@ configEnum acl_pubsub_default_enum[] = {
     {NULL, 0}
 };
 
+configEnum sanitize_dump_payload_enum[] = {
+    {"no", SANITIZE_DUMP_NO},
+    {"yes", SANITIZE_DUMP_YES},
+    {"clients", SANITIZE_DUMP_CLIENTS},
+    {NULL, 0}
+};
+
 /* Output buffer limits presets. */
 clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = {
     {0, 0, 0}, /* normal */
@@ -2361,6 +2368,7 @@ standardConfig configs[] = {
     createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, AOF_FSYNC_EVERYSEC, NULL, NULL),
     createEnumConfig("oom-score-adj", NULL, MODIFIABLE_CONFIG, oom_score_adj_enum, server.oom_score_adj, OOM_SCORE_ADJ_NO, NULL, updateOOMScoreAdj),
     createEnumConfig("acl-pubsub-default", NULL, MODIFIABLE_CONFIG, acl_pubsub_default_enum, server.acl_pubusub_default, USER_FLAG_ALLCHANNELS, NULL, NULL),
+    createEnumConfig("sanitize-dump-payload", NULL, MODIFIABLE_CONFIG, sanitize_dump_payload_enum, server.sanitize_dump_payload, SANITIZE_DUMP_NO, NULL, NULL),
 
     /* Integer configs */
     createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, 16, INTEGER_CONFIG, NULL, NULL),
diff --git a/src/defrag.c b/src/defrag.c
index 04ade30ea..165e55f65 100644
--- a/src/defrag.c
+++ b/src/defrag.c
@@ -5,7 +5,6 @@
  * We do that by scanning the keyspace and for each pointer we have, we can try to
  * ask the allocator if moving it to a new address will help reduce fragmentation.
  *
- * Copyright (c) 2020, Oran Agra
  * Copyright (c) 2020, Redis Labs, Inc
  * All rights reserved.
  *
diff --git a/src/intset.c b/src/intset.c
index 4445a5ca6..0079199cf 100644
--- a/src/intset.c
+++ b/src/intset.c
@@ -281,6 +281,33 @@ size_t intsetBlobLen(intset *is) {
     return sizeof(intset)+intrev32ifbe(is->length)*intrev32ifbe(is->encoding);
 }
 
+int intsetValidateIntegrity(const unsigned char *p, size_t size) {
+    const intset *is = (const intset *)p;
+    /* check that we can actually read the header. */
+    if (size < sizeof(*is))
+        return 0;
+
+    uint32_t encoding = intrev32ifbe(is->encoding);
+
+    size_t record_size;
+    if (encoding == INTSET_ENC_INT64) {
+        record_size = INTSET_ENC_INT64;
+    } else if (encoding == INTSET_ENC_INT32) {
+        record_size = INTSET_ENC_INT32;
+    } else if (encoding == INTSET_ENC_INT16){
+        record_size = INTSET_ENC_INT16;
+    } else {
+        return 0;
+    }
+
+    /* check that the size matchies (all records are inside the buffer). */
+    uint32_t count = intrev32ifbe(is->length);
+    if (sizeof(*is) + count*record_size != size)
+        return 0;
+
+    return 1;
+}
+
 #ifdef REDIS_TEST
 #include <sys/time.h>
 #include <time.h>
diff --git a/src/intset.h b/src/intset.h
index 8119e6636..4fba339d5 100644
--- a/src/intset.h
+++ b/src/intset.h
@@ -46,6 +46,7 @@ int64_t intsetRandom(intset *is);
 uint8_t intsetGet(intset *is, uint32_t pos, int64_t *value);
 uint32_t intsetLen(const intset *is);
 size_t intsetBlobLen(intset *is);
+int intsetValidateIntegrity(const unsigned char *is, size_t size);
 
 #ifdef REDIS_TEST
 int intsetTest(int argc, char *argv[]);
diff --git a/src/listpack.c b/src/listpack.c
index f8c34429e..580954075 100644
--- a/src/listpack.c
+++ b/src/listpack.c
@@ -5,6 +5,7 @@
  *  https://github.com/antirez/listpack
  *
  * Copyright (c) 2017, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2020, Redis Labs, Inc
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -41,6 +42,7 @@
 
 #include "listpack.h"
 #include "listpack_malloc.h"
+#include "redisassert.h"
 
 #define LP_HDR_SIZE 6       /* 32 bit total len + 16 bit number of elements. */
 #define LP_HDR_NUMELE_UNKNOWN UINT16_MAX
@@ -114,6 +116,21 @@
     (p)[5] = ((v)>>8)&0xff; \
 } while(0)
 
+/* Validates that 'p' is not ouside the listpack.
+ * All function that return a pointer to an element in the listpack will assert
+ * that this element is valid, so it can be freely used.
+ * Generally functions such lpNext and lpDelete assume the input pointer is
+ * already validated (since it's the return value of another function). */
+#define ASSERT_INTEGRITY(lp, p) do { \
+    assert((p) >= (lp)+LP_HDR_SIZE && (p) < (lp)+lpGetTotalBytes((lp))); \
+} while (0)
+
+/* Similar to the above, but validates the entire element lenth rather than just
+ * it's pointer. */
+#define ASSERT_INTEGRITY_LEN(lp, p, len) do { \
+    assert((p) >= (lp)+LP_HDR_SIZE && (p)+(len) < (lp)+lpGetTotalBytes((lp))); \
+} while (0)
+
 /* Convert a string into a signed 64 bit integer.
  * The function returns 1 if the string could be parsed into a (non-overflowing)
  * signed 64 bit int, 0 otherwise. The 'value' will be set to the parsed value
@@ -367,9 +384,14 @@ void lpEncodeString(unsigned char *buf, unsigned char *s, uint32_t len) {
     }
 }
 
-/* Return the encoded length of the listpack element pointed by 'p'. If the
- * element encoding is wrong then 0 is returned. */
-uint32_t lpCurrentEncodedSize(unsigned char *p) {
+/* Return the encoded length of the listpack element pointed by 'p'.
+ * This includes the encoding byte, length bytes, and the element data itself.
+ * If the element encoding is wrong then 0 is returned.
+ * Note that this method may access additional bytes (in case of 12 and 32 bit
+ * str), so should only be called when we know 'p' was already validated by
+ * lpCurrentEncodedSizeBytes or ASSERT_INTEGRITY_LEN (possibly since 'p' is
+ * a return value of another function that validated its return. */
+uint32_t lpCurrentEncodedSizeUnsafe(unsigned char *p) {
     if (LP_ENCODING_IS_7BIT_UINT(p[0])) return 1;
     if (LP_ENCODING_IS_6BIT_STR(p[0])) return 1+LP_ENCODING_6BIT_STR_LEN(p);
     if (LP_ENCODING_IS_13BIT_INT(p[0])) return 2;
@@ -383,12 +405,30 @@ uint32_t lpCurrentEncodedSize(unsigned char *p) {
     return 0;
 }
 
+/* Return bytes needed to encode the length of the listpack element pointed by 'p'.
+ * This includes just the encodign byte, and the bytes needed to encode the length
+ * of the element (excluding the element data itself)
+ * If the element encoding is wrong then 0 is returned. */
+uint32_t lpCurrentEncodedSizeBytes(unsigned char *p) {
+    if (LP_ENCODING_IS_7BIT_UINT(p[0])) return 1;
+    if (LP_ENCODING_IS_6BIT_STR(p[0])) return 1;
+    if (LP_ENCODING_IS_13BIT_INT(p[0])) return 1;
+    if (LP_ENCODING_IS_16BIT_INT(p[0])) return 1;
+    if (LP_ENCODING_IS_24BIT_INT(p[0])) return 1;
+    if (LP_ENCODING_IS_32BIT_INT(p[0])) return 1;
+    if (LP_ENCODING_IS_64BIT_INT(p[0])) return 1;
+    if (LP_ENCODING_IS_12BIT_STR(p[0])) return 2;
+    if (LP_ENCODING_IS_32BIT_STR(p[0])) return 5;
+    if (p[0] == LP_EOF) return 1;
+    return 0;
+}
+
 /* Skip the current entry returning the next. It is invalid to call this
  * function if the current element is the EOF element at the end of the
  * listpack, however, while this function is used to implement lpNext(),
  * it does not return NULL when the EOF element is encountered. */
 unsigned char *lpSkip(unsigned char *p) {
-    unsigned long entrylen = lpCurrentEncodedSize(p);
+    unsigned long entrylen = lpCurrentEncodedSizeUnsafe(p);
     entrylen += lpEncodeBacklen(NULL,entrylen);
     p += entrylen;
     return p;
@@ -398,8 +438,9 @@ unsigned char *lpSkip(unsigned char *p) {
  * the pointer to the next element (the one on the right), or NULL if 'p'
  * already pointed to the last element of the listpack. */
 unsigned char *lpNext(unsigned char *lp, unsigned char *p) {
-    ((void) lp); /* lp is not used for now. However lpPrev() uses it. */
+    assert(p);
     p = lpSkip(p);
+    ASSERT_INTEGRITY(lp, p);
     if (p[0] == LP_EOF) return NULL;
     return p;
 }
@@ -408,11 +449,14 @@ unsigned char *lpNext(unsigned char *lp, unsigned char *p) {
  * the pointer to the previous element (the one on the left), or NULL if 'p'
  * already pointed to the first element of the listpack. */
 unsigned char *lpPrev(unsigned char *lp, unsigned char *p) {
+    assert(p);
     if (p-lp == LP_HDR_SIZE) return NULL;
     p--; /* Seek the first backlen byte of the last element. */
     uint64_t prevlen = lpDecodeBacklen(p);
     prevlen += lpEncodeBacklen(NULL,prevlen);
-    return p-prevlen+1; /* Seek the first byte of the previous entry. */
+    p -= prevlen-1; /* Seek the first byte of the previous entry. */
+    ASSERT_INTEGRITY(lp, p);
+    return p;
 }
 
 /* Return a pointer to the first element of the listpack, or NULL if the
@@ -570,8 +614,7 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) {
 
 /* Insert, delete or replace the specified element 'ele' of length 'len' at
  * the specified position 'p', with 'p' being a listpack element pointer
- * obtained with lpFirst(), lpLast(), lpIndex(), lpNext(), lpPrev() or
- * lpSeek().
+ * obtained with lpFirst(), lpLast(), lpNext(), lpPrev() or lpSeek().
  *
  * The element is inserted before, after, or replaces the element pointed
  * by 'p' depending on the 'where' argument, that can be LP_BEFORE, LP_AFTER
@@ -610,6 +653,7 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un
     if (where == LP_AFTER) {
         p = lpSkip(p);
         where = LP_BEFORE;
+        ASSERT_INTEGRITY(lp, p);
     }
 
     /* Store the offset of the element 'p', so that we can obtain its
@@ -639,8 +683,9 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un
     uint64_t old_listpack_bytes = lpGetTotalBytes(lp);
     uint32_t replaced_len  = 0;
     if (where == LP_REPLACE) {
-        replaced_len = lpCurrentEncodedSize(p);
+        replaced_len = lpCurrentEncodedSizeUnsafe(p);
         replaced_len += lpEncodeBacklen(NULL,replaced_len);
+        ASSERT_INTEGRITY_LEN(lp, p, replaced_len);
     }
 
     uint64_t new_listpack_bytes = old_listpack_bytes + enclen + backlen_size
@@ -801,3 +846,86 @@ unsigned char *lpSeek(unsigned char *lp, long index) {
     }
 }
 
+/* Validate the integrity of a single listpack entry and move to the next one.
+ * The input argument 'pp' is a reference to the current record and is advanced on exit.
+ * Returns 1 if valid, 0 if invalid. */
+int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes) {
+#define OUT_OF_RANGE(p) ( \
+        (p) < lp + LP_HDR_SIZE || \
+        (p) > lp + lpbytes - 1)
+    unsigned char *p = *pp;
+    if (!p)
+        return 0;
+
+    if (*p == LP_EOF) {
+        *pp = NULL;
+        return 1;
+    }
+
+    /* check that we can read the encoded size */
+    uint32_t lenbytes = lpCurrentEncodedSizeBytes(p);
+    if (!lenbytes)
+        return 0;
+
+    /* make sure the encoded entry length doesn't rech outside the edge of the listpack */
+    if (OUT_OF_RANGE(p + lenbytes))
+        return 0;
+
+    /* get the entry length and encoded backlen. */
+    unsigned long entrylen = lpCurrentEncodedSizeUnsafe(p);
+    unsigned long encodedBacklen = lpEncodeBacklen(NULL,entrylen);
+    entrylen += encodedBacklen;
+
+    /* make sure the entry doesn't rech outside the edge of the listpack */
+    if (OUT_OF_RANGE(p + entrylen))
+        return 0;
+
+    /* move to the next entry */
+    p += entrylen;
+
+    /* make sure the encoded length at the end patches the one at the beginning. */
+    uint64_t prevlen = lpDecodeBacklen(p-1);
+    if (prevlen + encodedBacklen != entrylen)
+        return 0;
+
+    *pp = p;
+    return 1;
+#undef OUT_OF_RANGE
+}
+
+/* Validate the integrity of the data stracture.
+ * when `deep` is 0, only the integrity of the header is validated.
+ * when `deep` is 1, we scan all the entries one by one. */
+int lpValidateIntegrity(unsigned char *lp, size_t size, int deep){
+    /* Check that we can actually read the header. (and EOF) */
+    if (size < LP_HDR_SIZE + 1)
+        return 0;
+
+    /* Check that the encoded size in the header must match the allocated size. */
+    size_t bytes = lpGetTotalBytes(lp);
+    if (bytes != size)
+        return 0;
+
+    /* The last byte must be the terminator. */
+    if (lp[size-1] != LP_EOF)
+        return 0;
+
+    if (!deep)
+        return 1;
+
+    /* Validate the invividual entries. */
+    uint32_t count = 0;
+    unsigned char *p = lpFirst(lp);
+    while(p) {
+        if (!lpValidateNext(lp, &p, bytes))
+            return 0;
+        count++;
+    }
+
+    /* Check that the count in the header is correct */
+    uint32_t numele = lpGetNumElements(lp);
+    if (numele != LP_HDR_NUMELE_UNKNOWN && numele != count)
+        return 0;
+
+    return 1;
+}
diff --git a/src/listpack.h b/src/listpack.h
index af67b4b41..e8375628b 100644
--- a/src/listpack.h
+++ b/src/listpack.h
@@ -57,5 +57,7 @@ unsigned char *lpNext(unsigned char *lp, unsigned char *p);
 unsigned char *lpPrev(unsigned char *lp, unsigned char *p);
 uint32_t lpBytes(unsigned char *lp);
 unsigned char *lpSeek(unsigned char *lp, long index);
+int lpValidateIntegrity(unsigned char *lp, size_t size, int deep);
+int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes);
 
 #endif
diff --git a/src/rdb.c b/src/rdb.c
index e3f34b3f0..af527f94c 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -68,17 +68,27 @@ void rdbReportError(int corruption_error, int linenum, char *reason, ...) {
     vsnprintf(msg+len,sizeof(msg)-len,reason,ap);
     va_end(ap);
 
-    if (!rdbCheckMode) {
-        if (rdbFileBeingLoaded || corruption_error) {
-            serverLog(LL_WARNING, "%s", msg);
-            char *argv[2] = {"",rdbFileBeingLoaded};
-            redis_check_rdb_main(2,argv,NULL);
-        } else {
-            serverLog(LL_WARNING, "%s. Failure loading rdb format from socket, assuming connection error, resuming operation.", msg);
-            return;
-        }
-    } else {
+    if (!server.loading) {
+        /* If we're in the context of a RESTORE command, just propagate the error. */
+        /* log in VERBOSE, and return (don't exit). */
+        serverLog(LL_VERBOSE, "%s", msg);
+        return;
+    } else if (rdbCheckMode) {
+        /* If we're inside the rdb checker, let it handle the error. */
         rdbCheckError("%s",msg);
+    } else if (rdbFileBeingLoaded) {
+        /* If we're loading an rdb file form disk, run rdb check (and exit) */
+        serverLog(LL_WARNING, "%s", msg);
+        char *argv[2] = {"",rdbFileBeingLoaded};
+        redis_check_rdb_main(2,argv,NULL);
+    } else if (corruption_error) {
+        /* In diskless loading, in case of corrupt file, log and exit. */
+        serverLog(LL_WARNING, "%s. Failure loading rdb format", msg);
+    } else {
+        /* In diskless loading, in case of a short read (not a corrupt
+         * file), log and proceed (don't exit). */
+        serverLog(LL_WARNING, "%s. Failure loading rdb format from socket, assuming connection error, resuming operation.", msg);
+        return;
     }
     serverLog(LL_WARNING, "Terminating server after rdb file reading failure.");
     exit(1);
@@ -1489,6 +1499,17 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
     uint64_t len;
     unsigned int i;
 
+    int deep_integrity_validation = server.sanitize_dump_payload == SANITIZE_DUMP_YES;
+    if (server.sanitize_dump_payload == SANITIZE_DUMP_CLIENTS) {
+        /* Skip sanitization when loading (an RDB), or getting a RESTORE command
+         * from either the master or a client using an ACL user with the skip-sanitize-payload flag. */
+        int skip = server.loading ||
+            (server.current_client && (server.current_client->flags & CLIENT_MASTER));
+        if (!skip && server.current_client && server.current_client->user)
+            skip = !!(server.current_client->user->flags & USER_FLAG_SANITIZE_PAYLOAD_SKIP);
+        deep_integrity_validation = !skip;
+    }
+
     if (rdbtype == RDB_TYPE_STRING) {
         /* Read string value */
         if ((o = rdbLoadEncodedStringObject(rdb)) == NULL) return NULL;
@@ -1685,12 +1706,20 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                             server.list_compress_depth);
 
         while (len--) {
+            size_t encoded_len;
             unsigned char *zl =
-                rdbGenericLoadStringObject(rdb,RDB_LOAD_PLAIN,NULL);
+                rdbGenericLoadStringObject(rdb,RDB_LOAD_PLAIN,&encoded_len);
             if (zl == NULL) {
                 decrRefCount(o);
                 return NULL;
             }
+            if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
+            if (!ziplistValidateIntegrity(zl, encoded_len, deep_integrity_validation)) {
+                rdbExitReportCorruptRDB("Ziplist integrity check failed.");
+                decrRefCount(o);
+                zfree(zl);
+                return NULL;
+            }
             quicklistAppendZiplist(o->ptr, zl);
         }
     } else if (rdbtype == RDB_TYPE_HASH_ZIPMAP  ||
@@ -1699,9 +1728,32 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                rdbtype == RDB_TYPE_ZSET_ZIPLIST ||
                rdbtype == RDB_TYPE_HASH_ZIPLIST)
     {
+        size_t encoded_len;
         unsigned char *encoded =
-            rdbGenericLoadStringObject(rdb,RDB_LOAD_PLAIN,NULL);
+            rdbGenericLoadStringObject(rdb,RDB_LOAD_PLAIN,&encoded_len);
         if (encoded == NULL) return NULL;
+        if (rdbtype == RDB_TYPE_HASH_ZIPMAP) {
+            /* Since we don't keep zipmaps anymore, the rdb loading for these
+             * is O(n) anyway, use `deep` validation. */
+            if (!zipmapValidateIntegrity(encoded, encoded_len, 1)) {
+                rdbExitReportCorruptRDB("Zipmap integrity check failed.");
+                zfree(encoded);
+                return NULL;
+            }
+        } else if (rdbtype == RDB_TYPE_SET_INTSET) {
+            if (!intsetValidateIntegrity(encoded, encoded_len)) {
+                rdbExitReportCorruptRDB("Intset integrity check failed.");
+                zfree(encoded);
+                return NULL;
+            }
+        } else { /* ziplist */
+            if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
+            if (!ziplistValidateIntegrity(encoded, encoded_len, deep_integrity_validation)) {
+                rdbExitReportCorruptRDB("Ziplist integrity check failed.");
+                zfree(encoded);
+                return NULL;
+            }
+        }
         o = createObject(OBJ_STRING,encoded); /* Obj type fixed below. */
 
         /* Fix the object encoding, and make sure to convert the encoded
@@ -1794,14 +1846,24 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             }
 
             /* Load the listpack. */
+            size_t lp_size;
             unsigned char *lp =
-                rdbGenericLoadStringObject(rdb,RDB_LOAD_PLAIN,NULL);
+                rdbGenericLoadStringObject(rdb,RDB_LOAD_PLAIN,&lp_size);
             if (lp == NULL) {
                 rdbReportReadError("Stream listpacks loading failed.");
                 sdsfree(nodekey);
                 decrRefCount(o);
                 return NULL;
             }
+            if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
+            if (!streamValidateListpackIntegrity(lp, lp_size, deep_integrity_validation)) {
+                rdbExitReportCorruptRDB("Stream listpack integrity check failed.");
+                sdsfree(nodekey);
+                decrRefCount(o);
+                zfree(lp);
+                return NULL;
+            }
+
             unsigned char *first = lpFirst(lp);
             if (first == NULL) {
                 /* Serialized listpacks should never be empty, since on
@@ -2389,6 +2451,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
                         (unsigned long long)expected,
                         (unsigned long long)cksum);
                 rdbExitReportCorruptRDB("RDB CRC error");
+                return C_ERR;
             }
         }
     }
diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c
index f399864dd..79dbf3229 100644
--- a/src/redis-check-rdb.c
+++ b/src/redis-check-rdb.c
@@ -372,6 +372,7 @@ int redis_check_rdb_main(int argc, char **argv, FILE *fp) {
     if (shared.integers[0] == NULL)
         createSharedObjects();
     server.loading_process_events_interval_bytes = 0;
+    server.sanitize_dump_payload = SANITIZE_DUMP_YES;
     rdbCheckMode = 1;
     rdbCheckInfo("Checking RDB file %s", argv[1]);
     rdbCheckSetupSignals();
diff --git a/src/server.c b/src/server.c
index c30577da4..1b6f71aef 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2934,6 +2934,7 @@ void resetServerStats(void) {
     atomicSet(server.stat_net_input_bytes, 0);
     atomicSet(server.stat_net_output_bytes, 0);
     server.stat_unexpected_error_replies = 0;
+    server.stat_dump_payload_sanitizations = 0;
     server.aof_delayed_fsync = 0;
     server.blocked_last_cron = 0;
 }
@@ -4654,6 +4655,7 @@ sds genRedisInfoString(const char *section) {
             "tracking_total_items:%lld\r\n"
             "tracking_total_prefixes:%lld\r\n"
             "unexpected_error_replies:%lld\r\n"
+            "dump_payload_sanitizations:%lld\r\n"
             "total_reads_processed:%lld\r\n"
             "total_writes_processed:%lld\r\n"
             "io_threaded_reads_processed:%lld\r\n"
@@ -4689,6 +4691,7 @@ sds genRedisInfoString(const char *section) {
             (unsigned long long) trackingGetTotalItems(),
             (unsigned long long) trackingGetTotalPrefixes(),
             server.stat_unexpected_error_replies,
+            server.stat_dump_payload_sanitizations,
             stat_total_reads_processed,
             stat_total_writes_processed,
             server.stat_io_reads_processed,
diff --git a/src/server.h b/src/server.h
index 5bbe7876c..8a35d48df 100644
--- a/src/server.h
+++ b/src/server.h
@@ -378,6 +378,11 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
 #define TLS_CLIENT_AUTH_YES 1
 #define TLS_CLIENT_AUTH_OPTIONAL 2
 
+/* Sanitize dump payload */
+#define SANITIZE_DUMP_NO 0
+#define SANITIZE_DUMP_YES 1
+#define SANITIZE_DUMP_CLIENTS 2
+
 /* Sets operations codes */
 #define SET_OP_UNION 0
 #define SET_OP_DIFF 1
@@ -781,6 +786,12 @@ typedef struct readyList {
                                            authenticated. */
 #define USER_FLAG_ALLCHANNELS (1<<5)    /* The user can mention any Pub/Sub
                                            channel. */
+#define USER_FLAG_SANITIZE_PAYLOAD (1<<6)       /* The user require a deep RESTORE
+                                                 * payload sanitization. */
+#define USER_FLAG_SANITIZE_PAYLOAD_SKIP (1<<7)  /* The user should skip the
+                                                 * deep sanitization of RESTORE
+                                                 * payload. */
+
 typedef struct {
     sds name;       /* The username as an SDS string. */
     uint64_t flags; /* See USER_FLAG_* */
@@ -1213,6 +1224,7 @@ struct redisServer {
     size_t stat_module_cow_bytes;   /* Copy on write bytes during module fork. */
     uint64_t stat_clients_type_memory[CLIENT_TYPE_COUNT];/* Mem usage by type */
     long long stat_unexpected_error_replies; /* Number of unexpected (aof-loading, replica to master, etc.) error replies */
+    long long stat_dump_payload_sanitizations; /* Number deep dump payloads integrity validations. */
     long long stat_io_reads_processed; /* Number of read events processed by IO / Main threads */
     long long stat_io_writes_processed; /* Number of write events processed by IO / Main threads */
     redisAtomic long long stat_total_reads_processed; /* Total number of read events processed */
@@ -1232,6 +1244,7 @@ struct redisServer {
     int active_expire_enabled;      /* Can be disabled for testing purposes. */
     int active_expire_effort;       /* From 1 (default) to 10, active effort. */
     int active_defrag_enabled;
+    int sanitize_dump_payload;      /* Enables deep sanitization for ziplist and listpack in RDB and RESTORE. */
     int jemalloc_bg_thread;         /* Enable jemalloc background thread */
     size_t active_defrag_ignore_bytes; /* minimum amount of fragmentation waste to start active defrag */
     int active_defrag_threshold_lower; /* minimum percentage of fragmentation to start active defrag */
diff --git a/src/stream.h b/src/stream.h
index 1a1e271a8..c7acee719 100644
--- a/src/stream.h
+++ b/src/stream.h
@@ -120,5 +120,6 @@ int streamIncrID(streamID *id);
 int streamDecrID(streamID *id);
 void streamPropagateConsumerCreation(client *c, robj *key, robj *groupname, sds consumername);
 robj *streamDup(robj *o);
+int streamValidateListpackIntegrity(unsigned char *lp, size_t size, int deep);
 
 #endif
diff --git a/src/t_hash.c b/src/t_hash.c
index f3fde87aa..e6f14091b 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -67,7 +67,7 @@ int hashTypeGetFromZiplist(robj *o, sds field,
     zl = o->ptr;
     fptr = ziplistIndex(zl, ZIPLIST_HEAD);
     if (fptr != NULL) {
-        fptr = ziplistFind(fptr, (unsigned char*)field, sdslen(field), 1);
+        fptr = ziplistFind(zl, fptr, (unsigned char*)field, sdslen(field), 1);
         if (fptr != NULL) {
             /* Grab pointer to the value (fptr points to the field) */
             vptr = ziplistNext(zl, fptr);
@@ -208,7 +208,7 @@ int hashTypeSet(robj *o, sds field, sds value, int flags) {
         zl = o->ptr;
         fptr = ziplistIndex(zl, ZIPLIST_HEAD);
         if (fptr != NULL) {
-            fptr = ziplistFind(fptr, (unsigned char*)field, sdslen(field), 1);
+            fptr = ziplistFind(zl, fptr, (unsigned char*)field, sdslen(field), 1);
             if (fptr != NULL) {
                 /* Grab pointer to the value (fptr points to the field) */
                 vptr = ziplistNext(zl, fptr);
@@ -285,7 +285,7 @@ int hashTypeDelete(robj *o, sds field) {
         zl = o->ptr;
         fptr = ziplistIndex(zl, ZIPLIST_HEAD);
         if (fptr != NULL) {
-            fptr = ziplistFind(fptr, (unsigned char*)field, sdslen(field), 1);
+            fptr = ziplistFind(zl, fptr, (unsigned char*)field, sdslen(field), 1);
             if (fptr != NULL) {
                 zl = ziplistDelete(zl,&fptr); /* Delete the key. */
                 zl = ziplistDelete(zl,&fptr); /* Delete the value. */
diff --git a/src/t_stream.c b/src/t_stream.c
index bb38b0fb8..70cc8bfa1 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -255,21 +255,33 @@ unsigned char *lpReplaceInteger(unsigned char *lp, unsigned char **pos, int64_t
 
 /* This is a wrapper function for lpGet() to directly get an integer value
  * from the listpack (that may store numbers as a string), converting
- * the string if needed. */
-int64_t lpGetInteger(unsigned char *ele) {
+ * the string if needed.
+ * The 'valid" argument is an optional output parameter to get an indication
+ * if the record was valid, when this parameter is NULL, the function will
+ * fail with an assertion. */
+static inline int64_t lpGetIntegerIfValid(unsigned char *ele, int *valid) {
     int64_t v;
     unsigned char *e = lpGet(ele,&v,NULL);
-    if (e == NULL) return v;
+    if (e == NULL) {
+        if (valid)
+            *valid = 1;
+        return v;
+    }
     /* The following code path should never be used for how listpacks work:
      * they should always be able to store an int64_t value in integer
      * encoded form. However the implementation may change. */
     long long ll;
-    int retval = string2ll((char*)e,v,&ll);
-    serverAssert(retval != 0);
+    int ret = string2ll((char*)e,v,&ll);
+    if (valid)
+        *valid = ret;
+    else
+        serverAssert(ret != 0);
     v = ll;
     return v;
 }
 
+#define lpGetInteger(ele) lpGetIntegerIfValid(ele, NULL)
+
 /* Debugging function to log the full content of a listpack. Useful
  * for development and debugging. */
 void streamLogListpackContent(unsigned char *lp) {
@@ -3037,3 +3049,90 @@ NULL
         addReplySubcommandSyntaxError(c);
     }
 }
+
+/* Validate the integrity stream listpack entries stracture. Both in term of a
+ * valid listpack, but also that the stracture of the entires matches a valid
+ * stream. return 1 if valid 0 if not valid. */
+int streamValidateListpackIntegrity(unsigned char *lp, size_t size, int deep) {
+    int valid_record;
+    unsigned char *p, *next;
+
+    /* Since we don't want to run validation of all records twice, we'll
+     * run the listpack validation of just the header and do the rest here. */
+    if (!lpValidateIntegrity(lp, size, 0))
+        return 0;
+
+    /* In non-deep mode we just validated the listpack header (encoded size) */
+    if (!deep) return 1;
+
+    next = p = lpFirst(lp);
+    if (!lpValidateNext(lp, &next, size)) return 0;
+    if (!p) return 0;
+
+    /* entry count */
+    int64_t entry_count = lpGetIntegerIfValid(p, &valid_record);
+    if (!valid_record) return 0;
+    p = next; if (!lpValidateNext(lp, &next, size)) return 0;
+
+    /* deleted */
+    lpGetIntegerIfValid(p, &valid_record);
+    if (!valid_record) return 0;
+    p = next; if (!lpValidateNext(lp, &next, size)) return 0;
+
+    /* num-of-fields */
+    int64_t master_fields = lpGetIntegerIfValid(p, &valid_record);
+    if (!valid_record) return 0;
+    p = next; if (!lpValidateNext(lp, &next, size)) return 0;
+
+    /* the field names */
+    for (int64_t j = 0; j < master_fields; j++) {
+        p = next; if (!lpValidateNext(lp, &next, size)) return 0;
+    }
+
+    /* the zero master entry terminator. */
+    int64_t zero = lpGetIntegerIfValid(p, &valid_record);
+    if (!valid_record || zero != 0) return 0;
+    p = next; if (!lpValidateNext(lp, &next, size)) return 0;
+
+    while (entry_count--) {
+        if (!p) return 0;
+        int64_t fields = master_fields, extra_fields = 3;
+        int64_t flags = lpGetIntegerIfValid(p, &valid_record);
+        if (!valid_record) return 0;
+        p = next; if (!lpValidateNext(lp, &next, size)) return 0;
+
+        /* entry id */
+        p = next; if (!lpValidateNext(lp, &next, size)) return 0;
+        p = next; if (!lpValidateNext(lp, &next, size)) return 0;
+
+        if (!(flags & STREAM_ITEM_FLAG_SAMEFIELDS)) {
+            /* num-of-fields */
+            fields = lpGetIntegerIfValid(p, &valid_record);
+            if (!valid_record) return 0;
+            p = next; if (!lpValidateNext(lp, &next, size)) return 0;
+
+            /* the field names */
+            for (int64_t j = 0; j < fields; j++) {
+                p = next; if (!lpValidateNext(lp, &next, size)) return 0;
+            }
+
+            extra_fields += fields + 1;
+        }
+
+        /* the values */
+        for (int64_t j = 0; j < fields; j++) {
+            p = next; if (!lpValidateNext(lp, &next, size)) return 0;
+        }
+
+        /* lp-count */
+        int64_t lp_count = lpGetIntegerIfValid(p, &valid_record);
+        if (!valid_record) return 0;
+        if (lp_count != fields + extra_fields) return 0;
+        p = next; if (!lpValidateNext(lp, &next, size)) return 0;
+    }
+
+    if (next)
+        return 0;
+
+    return 1;
+}
diff --git a/src/ziplist.c b/src/ziplist.c
index dd7452006..a8ab57cfe 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -152,6 +152,7 @@
  *
  * Copyright (c) 2009-2012, Pieter Noordhuis <pcnoordhuis at gmail dot com>
  * Copyright (c) 2009-2017, Salvatore Sanfilippo <antirez at gmail dot com>
+ * Copyright (c) 2020, Redis Labs, Inc
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -298,10 +299,34 @@ typedef struct zlentry {
 /* Extract the encoding from the byte pointed by 'ptr' and set it into
  * 'encoding' field of the zlentry structure. */
 #define ZIP_ENTRY_ENCODING(ptr, encoding) do {  \
-    (encoding) = (ptr[0]); \
+    (encoding) = ((ptr)[0]); \
     if ((encoding) < ZIP_STR_MASK) (encoding) &= ZIP_STR_MASK; \
 } while(0)
 
+#define ZIP_ENCODING_SIZE_INVALID 0xff
+/* Return the number of bytes required to encode the entry type + length.
+ * On error, return ZIP_ENCODING_SIZE_INVALID */
+static inline unsigned int zipEncodingLenSize(unsigned char encoding) {
+    if (encoding == ZIP_INT_16B || encoding == ZIP_INT_32B ||
+        encoding == ZIP_INT_24B || encoding == ZIP_INT_64B ||
+        encoding == ZIP_INT_8B)
+        return 1;
+    if (encoding >= ZIP_INT_IMM_MIN && encoding <= ZIP_INT_IMM_MAX)
+        return 1;
+    if (encoding == ZIP_STR_06B)
+        return 1;
+    if (encoding == ZIP_STR_14B)
+        return 2;
+    if (encoding == ZIP_STR_32B)
+        return 5;
+    return ZIP_ENCODING_SIZE_INVALID;
+}
+
+#define ZIP_ASSERT_ENCODING(encoding) do {                          \
+    if (zipEncodingLenSize(encoding) == ZIP_ENCODING_SIZE_INVALID)     \
+        panic("Invalid encoding 0x%02X", encoding);                 \
+} while (0)
+
 /* Return bytes needed to store integer encoded by 'encoding'. */
 unsigned int zipIntSize(unsigned char encoding) {
     switch(encoding) {
@@ -313,7 +338,7 @@ unsigned int zipIntSize(unsigned char encoding) {
     }
     if (encoding >= ZIP_INT_IMM_MIN && encoding <= ZIP_INT_IMM_MAX)
         return 0; /* 4 bit immediate */
-    panic("Invalid integer encoding 0x%02X", encoding);
+    /* bad encoding, covered by a previous call to ZIP_ASSERT_ENCODING */
     return 0;
 }
 
@@ -365,11 +390,10 @@ unsigned int zipStoreEntryEncoding(unsigned char *p, unsigned char encoding, uns
 
 /* Decode the entry encoding type and data length (string length for strings,
  * number of bytes used for the integer for integer entries) encoded in 'ptr'.
- * The 'encoding' variable will hold the entry encoding, the 'lensize'
+ * The 'encoding' variable is input, extracted by the caller, the 'lensize'
  * variable will hold the number of bytes required to encode the entry
  * length, and the 'len' variable will hold the entry length. */
 #define ZIP_DECODE_LENGTH(ptr, encoding, lensize, len) do {                    \
-    ZIP_ENTRY_ENCODING((ptr), (encoding));                                     \
     if ((encoding) < ZIP_STR_MASK) {                                           \
         if ((encoding) == ZIP_STR_06B) {                                       \
             (lensize) = 1;                                                     \
@@ -384,7 +408,8 @@ unsigned int zipStoreEntryEncoding(unsigned char *p, unsigned char encoding, uns
                     ((ptr)[3] <<  8) |                                         \
                     ((ptr)[4]);                                                \
         } else {                                                               \
-            panic("Invalid string encoding 0x%02X", (encoding));               \
+            len = 0; /* bad encoding, dead code */                             \
+            /* covered by a previous call to ZIP_ASSERT_ENCODING */            \
         }                                                                      \
     } else {                                                                   \
         (lensize) = 1;                                                         \
@@ -440,9 +465,10 @@ unsigned int zipStorePrevEntryLength(unsigned char *p, unsigned int len) {
     if ((prevlensize) == 1) {                                                  \
         (prevlen) = (ptr)[0];                                                  \
     } else if ((prevlensize) == 5) {                                           \
-        assert(sizeof((prevlen)) == 4);                                        \
-        memcpy(&(prevlen), ((char*)(ptr)) + 1, 4);                             \
-        memrev32ifbe(&prevlen);                                                \
+        (prevlen) = ((ptr)[4] << 24) |                                         \
+                    ((ptr)[3] << 16) |                                         \
+                    ((ptr)[2] <<  8) |                                         \
+                    ((ptr)[1]);                                                \
     }                                                                          \
 } while(0)
 
@@ -467,14 +493,6 @@ int zipPrevLenByteDiff(unsigned char *p, unsigned int len) {
     return zipStorePrevEntryLength(NULL, len) - prevlensize;
 }
 
-/* Return the total number of bytes used by the entry pointed to by 'p'. */
-unsigned int zipRawEntryLength(unsigned char *p) {
-    unsigned int prevlensize, encoding, lensize, len;
-    ZIP_DECODE_PREVLENSIZE(p, prevlensize);
-    ZIP_DECODE_LENGTH(p + prevlensize, encoding, lensize, len);
-    return prevlensize + lensize + len;
-}
-
 /* Check if string pointed to by 'entry' can be encoded as an integer.
  * Stores the integer value in 'v' and its encoding in 'encoding'. */
 int zipTryEncoding(unsigned char *entry, unsigned int entrylen, long long *v, unsigned char *encoding) {
@@ -565,15 +583,87 @@ int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) {
     return ret;
 }
 
-/* Return a struct with all information about an entry. */
+/* Fills a struct with all information about an entry.
+ * This function is the "unsafe" alternative to the one blow.
+ * Generally, all function that return a pointer to an element in the ziplist
+ * will assert that this element is valid, so it can be freely used.
+ * Generally functions such ziplistGet assume the input pointer is already
+ * validated (since it's the return value of another function). */
 void zipEntry(unsigned char *p, zlentry *e) {
-
     ZIP_DECODE_PREVLEN(p, e->prevrawlensize, e->prevrawlen);
+    ZIP_ENTRY_ENCODING(p + e->prevrawlensize, e->encoding);
+    ZIP_ASSERT_ENCODING(e->encoding);
     ZIP_DECODE_LENGTH(p + e->prevrawlensize, e->encoding, e->lensize, e->len);
     e->headersize = e->prevrawlensize + e->lensize;
     e->p = p;
 }
 
+/* Fills a struct with all information about an entry.
+ * This function is safe to use on untrusted pointers, it'll make sure not to
+ * try to access memory outside the ziplist payload.
+ * Returns 1 if the entry is valid, and 0 otherwise. */
+static inline int zipEntrySafe(unsigned char* zl, size_t zlbytes, unsigned char *p, zlentry *e, int validate_prevlen) {
+#define OUT_OF_RANGE(p) ( \
+        (p) < zl + ZIPLIST_HEADER_SIZE || \
+        (p) > zl + zlbytes - ZIPLIST_END_SIZE)
+
+    /* Make sure the pointer doesn't rech outside the edge of the ziplist */
+    if (OUT_OF_RANGE(p))
+        return 0;
+
+    /* Make sure the encoded prevlen header doesn't reach outside the allocation */
+    ZIP_DECODE_PREVLENSIZE(p, e->prevrawlensize);
+    if (OUT_OF_RANGE(p + e->prevrawlensize))
+        return 0;
+
+    /* Make sure encoded entry header is valid. */
+    ZIP_ENTRY_ENCODING(p + e->prevrawlensize, e->encoding);
+    e->lensize = zipEncodingLenSize(e->encoding);
+    if (e->lensize == ZIP_ENCODING_SIZE_INVALID)
+        return 0;
+
+    /* Make sure the encoded entry header doesn't reach outside the allocation */
+    if (OUT_OF_RANGE(p + e->prevrawlensize + e->lensize))
+        return 0;
+
+    /* Decode the prevlen and entry len headers. */
+    ZIP_DECODE_PREVLEN(p, e->prevrawlensize, e->prevrawlen);
+    ZIP_DECODE_LENGTH(p + e->prevrawlensize, e->encoding, e->lensize, e->len);
+    e->headersize = e->prevrawlensize + e->lensize;
+
+    /* Make sure the entry doesn't rech outside the edge of the ziplist */
+    if (OUT_OF_RANGE(p + e->headersize + e->len))
+        return 0;
+
+    /* Make sure prevlen doesn't rech outside the edge of the ziplist */
+    if (validate_prevlen && OUT_OF_RANGE(p - e->prevrawlen))
+        return 0;
+
+    e->p = p;
+    return 1;
+#undef OUT_OF_RANGE
+}
+
+/* Return the total number of bytes used by the entry pointed to by 'p'. */
+static inline unsigned int zipRawEntryLengthSafe(unsigned char* zl, size_t zlbytes, unsigned char *p) {
+    zlentry e;
+    assert(zipEntrySafe(zl, zlbytes, p, &e, 0));
+    return e.headersize + e.len;
+}
+
+/* Return the total number of bytes used by the entry pointed to by 'p'. */
+static inline unsigned int zipRawEntryLength(unsigned char *p) {
+    zlentry e;
+    zipEntry(p, &e);
+    return e.headersize + e.len;
+}
+
+/* Validate that the entry doesn't reach outside the ziplist allocation. */
+static inline void zipAssertValidEntry(unsigned char* zl, size_t zlbytes, unsigned char *p) {
+    zlentry e;
+    assert(zipEntrySafe(zl, zlbytes, p, &e, 1));
+}
+
 /* Create a new empty ziplist. */
 unsigned char *ziplistNew(void) {
     unsigned int bytes = ZIPLIST_HEADER_SIZE+ZIPLIST_END_SIZE;
@@ -625,7 +715,7 @@ unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p) {
     /* Empty ziplist */
     if (p[0] == ZIP_END) return zl;
 
-    zipEntry(p, &cur);
+    zipEntry(p, &cur); /* no need for "safe" variant since the input pointer was validated by the function that returned it. */
     firstentrylen = prevlen = cur.headersize + cur.len;
     prevlensize = zipStorePrevEntryLength(NULL, prevlen);
     prevoffset = p - zl;
@@ -633,7 +723,7 @@ unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p) {
 
     /* Iterate ziplist to find out how many extra bytes do we need to update it. */
     while (p[0] != ZIP_END) {
-        zipEntry(p, &cur);
+        assert(zipEntrySafe(zl, curlen, p, &cur, 0));
 
         /* Abort when "prevlen" has not changed. */
         if (cur.prevrawlen == prevlen) break;
@@ -690,7 +780,7 @@ unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p) {
 
     /* Iterate all entries that need to be updated tail to head. */
     while (cnt) {
-        zipEntry(zl + prevoffset, &cur);
+        zipEntry(zl + prevoffset, &cur); /* no need for "safe" variant since we already iterated on all these entries above. */
         rawlen = cur.headersize + cur.len;
         /* Move entry to tail and reset prevlen. */
         memmove(p - (rawlen - cur.prevrawlensize), 
@@ -717,15 +807,18 @@ unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsigned int
     size_t offset;
     int nextdiff = 0;
     zlentry first, tail;
+    size_t zlbytes = intrev32ifbe(ZIPLIST_BYTES(zl));
 
-    zipEntry(p, &first);
+    zipEntry(p, &first); /* no need for "safe" variant since the input pointer was validated by the function that returned it. */
     for (i = 0; p[0] != ZIP_END && i < num; i++) {
-        p += zipRawEntryLength(p);
+        p += zipRawEntryLengthSafe(zl, zlbytes, p);
         deleted++;
     }
 
+    assert(p >= first.p);
     totlen = p-first.p; /* Bytes taken by the element(s) to delete. */
     if (totlen > 0) {
+        uint32_t set_tail;
         if (p[0] != ZIP_END) {
             /* Storing `prevrawlen` in this entry may increase or decrease the
              * number of bytes required compare to the current `prevrawlen`.
@@ -738,36 +831,44 @@ unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsigned int
              * had a 5 bytes prevlen header, so there is for sure at least
              * 5 bytes free and we need just 4. */
             p -= nextdiff;
+            assert(p >= first.p && p<zl+zlbytes-1);
             zipStorePrevEntryLength(p,first.prevrawlen);
 
             /* Update offset for tail */
-            ZIPLIST_TAIL_OFFSET(zl) =
-                intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))-totlen);
+            set_tail = intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))-totlen;
 
             /* When the tail contains more than one entry, we need to take
              * "nextdiff" in account as well. Otherwise, a change in the
              * size of prevlen doesn't have an effect on the *tail* offset. */
-            zipEntry(p, &tail);
+            assert(zipEntrySafe(zl, zlbytes, p, &tail, 1));
             if (p[tail.headersize+tail.len] != ZIP_END) {
-                ZIPLIST_TAIL_OFFSET(zl) =
-                   intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff);
+                set_tail = set_tail + nextdiff;
             }
 
             /* Move tail to the front of the ziplist */
-            memmove(first.p,p,
-                intrev32ifbe(ZIPLIST_BYTES(zl))-(p-zl)-1);
+            /* since we asserted that p >= first.p. we know totlen >= 0,
+             * so we know that p > first.p and this is guaranteed not to reach
+             * beyond the allocation, even if the entries lens are corrupted. */
+            size_t bytes_to_move = zlbytes-(p-zl)-1;
+            memmove(first.p,p,bytes_to_move);
         } else {
             /* The entire tail was deleted. No need to move memory. */
-            ZIPLIST_TAIL_OFFSET(zl) =
-                intrev32ifbe((first.p-zl)-first.prevrawlen);
+            set_tail = (first.p-zl)-first.prevrawlen;
         }
 
-        /* Resize and update length */
+        /* Resize the ziplist */
         offset = first.p-zl;
-        zl = ziplistResize(zl, intrev32ifbe(ZIPLIST_BYTES(zl))-totlen+nextdiff);
-        ZIPLIST_INCR_LENGTH(zl,-deleted);
+        zlbytes -= totlen - nextdiff;
+        zl = ziplistResize(zl, zlbytes);
         p = zl+offset;
 
+        /* Update record count */
+        ZIPLIST_INCR_LENGTH(zl,-deleted);
+
+        /* Set the tail offset computed above */
+        assert(set_tail <= zlbytes - ZIPLIST_END_SIZE);
+        ZIPLIST_TAIL_OFFSET(zl) = intrev32ifbe(set_tail);
+
         /* When nextdiff != 0, the raw length of the next entry has changed, so
          * we need to cascade the update throughout the ziplist */
         if (nextdiff != 0)
@@ -778,7 +879,7 @@ unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsigned int
 
 /* Insert item at "p". */
 unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) {
-    size_t curlen = intrev32ifbe(ZIPLIST_BYTES(zl)), reqlen;
+    size_t curlen = intrev32ifbe(ZIPLIST_BYTES(zl)), reqlen, newlen;
     unsigned int prevlensize, prevlen = 0;
     size_t offset;
     int nextdiff = 0;
@@ -794,7 +895,7 @@ unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned cha
     } else {
         unsigned char *ptail = ZIPLIST_ENTRY_TAIL(zl);
         if (ptail[0] != ZIP_END) {
-            prevlen = zipRawEntryLength(ptail);
+            prevlen = zipRawEntryLengthSafe(zl, curlen, ptail);
         }
     }
 
@@ -824,7 +925,8 @@ unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned cha
 
     /* Store offset because a realloc may change the address of zl. */
     offset = p-zl;
-    zl = ziplistResize(zl,curlen+reqlen+nextdiff);
+    newlen = curlen+reqlen+nextdiff;
+    zl = ziplistResize(zl,newlen);
     p = zl+offset;
 
     /* Apply memory move when necessary and update tail offset. */
@@ -845,7 +947,7 @@ unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned cha
         /* When the tail contains more than one entry, we need to take
          * "nextdiff" in account as well. Otherwise, a change in the
          * size of prevlen doesn't have an effect on the *tail* offset. */
-        zipEntry(p+reqlen, &tail);
+        assert(zipEntrySafe(zl, newlen, p+reqlen, &tail, 1));
         if (p[reqlen+tail.headersize+tail.len] != ZIP_END) {
             ZIPLIST_TAIL_OFFSET(zl) =
                 intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff);
@@ -1002,23 +1104,35 @@ unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int sle
 unsigned char *ziplistIndex(unsigned char *zl, int index) {
     unsigned char *p;
     unsigned int prevlensize, prevlen = 0;
+    size_t zlbytes = intrev32ifbe(ZIPLIST_BYTES(zl));
     if (index < 0) {
         index = (-index)-1;
         p = ZIPLIST_ENTRY_TAIL(zl);
         if (p[0] != ZIP_END) {
+            /* No need for "safe" check: when going backwards, we know the header
+             * we're parsing is in the range, we just need to assert (below) that
+             * the size we take doesn't cause p to go outside the allocation. */
             ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);
             while (prevlen > 0 && index--) {
                 p -= prevlen;
+                assert(p >= zl + ZIPLIST_HEADER_SIZE && p < zl + zlbytes - ZIPLIST_END_SIZE);
                 ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);
             }
         }
     } else {
         p = ZIPLIST_ENTRY_HEAD(zl);
-        while (p[0] != ZIP_END && index--) {
-            p += zipRawEntryLength(p);
+        while (index--) {
+            /* Use the "safe" length: When we go forward, we need to be careful
+             * not to decode an entry header if it's past the ziplist allocation. */
+            p += zipRawEntryLengthSafe(zl, zlbytes, p);
+            if (p[0] == ZIP_END)
+                break;
         }
     }
-    return (p[0] == ZIP_END || index > 0) ? NULL : p;
+    if (p[0] == ZIP_END || index > 0)
+        return NULL;
+    zipAssertValidEntry(zl, zlbytes, p);
+    return p;
 }
 
 /* Return pointer to next entry in ziplist.
@@ -1029,6 +1143,7 @@ unsigned char *ziplistIndex(unsigned char *zl, int index) {
  * The element after 'p' is returned, otherwise NULL if we are at the end. */
 unsigned char *ziplistNext(unsigned char *zl, unsigned char *p) {
     ((void) zl);
+    size_t zlbytes = intrev32ifbe(ZIPLIST_BYTES(zl));
 
     /* "p" could be equal to ZIP_END, caused by ziplistDelete,
      * and we should return NULL. Otherwise, we should return NULL
@@ -1042,6 +1157,7 @@ unsigned char *ziplistNext(unsigned char *zl, unsigned char *p) {
         return NULL;
     }
 
+    zipAssertValidEntry(zl, zlbytes, p);
     return p;
 }
 
@@ -1060,7 +1176,10 @@ unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p) {
     } else {
         ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);
         assert(prevlen > 0);
-        return p-prevlen;
+        p-=prevlen;
+        size_t zlbytes = intrev32ifbe(ZIPLIST_BYTES(zl));
+        zipAssertValidEntry(zl, zlbytes, p);
+        return p;
     }
 }
 
@@ -1073,7 +1192,7 @@ unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *sl
     if (p == NULL || p[0] == ZIP_END) return 0;
     if (sstr) *sstr = NULL;
 
-    zipEntry(p, &entry);
+    zipEntry(p, &entry); /* no need for "safe" variant since the input pointer was validated by the function that returned it. */
     if (ZIP_IS_STR(entry.encoding)) {
         if (sstr) {
             *slen = entry.len;
@@ -1121,7 +1240,7 @@ unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int
     long long zval, sval;
     if (p[0] == ZIP_END) return 0;
 
-    zipEntry(p, &entry);
+    zipEntry(p, &entry); /* no need for "safe" variant since the input pointer was validated by the function that returned it. */
     if (ZIP_IS_STR(entry.encoding)) {
         /* Raw compare */
         if (entry.len == slen) {
@@ -1142,23 +1261,23 @@ unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int
 
 /* Find pointer to the entry equal to the specified entry. Skip 'skip' entries
  * between every comparison. Returns NULL when the field could not be found. */
-unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int vlen, unsigned int skip) {
+unsigned char *ziplistFind(unsigned char *zl, unsigned char *p, unsigned char *vstr, unsigned int vlen, unsigned int skip) {
     int skipcnt = 0;
     unsigned char vencoding = 0;
     long long vll = 0;
+    size_t zlbytes = ziplistBlobLen(zl);
 
     while (p[0] != ZIP_END) {
-        unsigned int prevlensize, encoding, lensize, len;
+        struct zlentry e;
         unsigned char *q;
 
-        ZIP_DECODE_PREVLENSIZE(p, prevlensize);
-        ZIP_DECODE_LENGTH(p + prevlensize, encoding, lensize, len);
-        q = p + prevlensize + lensize;
+        assert(zipEntrySafe(zl, zlbytes, p, &e, 1));
+        q = p + e.prevrawlensize + e.lensize;
 
         if (skipcnt == 0) {
             /* Compare current entry with specified entry */
-            if (ZIP_IS_STR(encoding)) {
-                if (len == vlen && memcmp(q, vstr, vlen) == 0) {
+            if (ZIP_IS_STR(e.encoding)) {
+                if (e.len == vlen && memcmp(q, vstr, vlen) == 0) {
                     return p;
                 }
             } else {
@@ -1180,7 +1299,7 @@ unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int v
                  * if vencoding != UCHAR_MAX because if there is no encoding
                  * possible for the field it can't be a valid integer. */
                 if (vencoding != UCHAR_MAX) {
-                    long long ll = zipLoadInteger(q, encoding);
+                    long long ll = zipLoadInteger(q, e.encoding);
                     if (ll == vll) {
                         return p;
                     }
@@ -1195,7 +1314,7 @@ unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int v
         }
 
         /* Move to next entry */
-        p = q + len;
+        p = q + e.len;
     }
 
     return NULL;
@@ -1208,8 +1327,9 @@ unsigned int ziplistLen(unsigned char *zl) {
         len = intrev16ifbe(ZIPLIST_LENGTH(zl));
     } else {
         unsigned char *p = zl+ZIPLIST_HEADER_SIZE;
+        size_t zlbytes = intrev32ifbe(ZIPLIST_BYTES(zl));
         while (*p != ZIP_END) {
-            p += zipRawEntryLength(p);
+            p += zipRawEntryLengthSafe(zl, zlbytes, p);
             len++;
         }
 
@@ -1228,6 +1348,7 @@ void ziplistRepr(unsigned char *zl) {
     unsigned char *p;
     int index = 0;
     zlentry entry;
+    size_t zlbytes = ziplistBlobLen(zl);
 
     printf(
         "{total bytes %u} "
@@ -1238,7 +1359,7 @@ void ziplistRepr(unsigned char *zl) {
         intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl)));
     p = ZIPLIST_ENTRY_HEAD(zl);
     while(*p != ZIP_END) {
-        zipEntry(p, &entry);
+        assert(zipEntrySafe(zl, zlbytes, p, &entry, 1));
         printf(
             "{\n"
                 "\taddr 0x%08lx,\n"
@@ -1282,6 +1403,63 @@ void ziplistRepr(unsigned char *zl) {
     printf("{end}\n\n");
 }
 
+/* Validate the integrity of the data stracture.
+ * when `deep` is 0, only the integrity of the header is validated.
+ * when `deep` is 1, we scan all the entries one by one. */
+int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep) {
+    /* check that we can actually read the header. (and ZIP_END) */
+    if (size < ZIPLIST_HEADER_SIZE + ZIPLIST_END_SIZE)
+        return 0;
+
+    /* check that the encoded size in the header must match the allocated size. */
+    size_t bytes = intrev32ifbe(ZIPLIST_BYTES(zl));
+    if (bytes != size)
+        return 0;
+
+    /* the last byte must be the terminator. */
+    if (zl[size - ZIPLIST_END_SIZE] != ZIP_END)
+        return 0;
+
+    /* make sure the tail offset isn't reaching outside the allocation. */
+    if (intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl)) > size - ZIPLIST_END_SIZE)
+        return 0;
+
+    if (!deep)
+        return 1;
+
+    unsigned int count = 0;
+    unsigned char *p = ZIPLIST_ENTRY_HEAD(zl);
+    unsigned char *prev = NULL;
+    size_t prev_raw_size = 0;
+    while(*p != ZIP_END) {
+        struct zlentry e;
+        /* Decode the entry headers and fail if invalid or reaches outside the allocation */
+        if (!zipEntrySafe(zl, size, p, &e, 1))
+            return 0;
+
+        /* Make sure the record stating the prev entry size is correct. */
+        if (e.prevrawlen != prev_raw_size)
+            return 0;
+
+        /* Move to the next entry */
+        prev_raw_size = e.headersize + e.len;
+        prev = p;
+        p += e.headersize + e.len;
+        count++;
+    }
+
+    /* Make sure the <zltail> entry really do point to the start of the last entry. */
+    if (prev != ZIPLIST_ENTRY_TAIL(zl))
+        return 0;
+
+    /* Check that the count in the header is correct */
+    unsigned int header_count = intrev16ifbe(ZIPLIST_LENGTH(zl));
+    if (header_count != UINT16_MAX && count != header_count)
+        return 0;
+
+    return 1;
+}
+
 #ifdef REDIS_TEST
 #include <sys/time.h>
 #include "adlist.h"
@@ -1784,6 +1962,7 @@ int ziplistTest(int argc, char **argv) {
 
     printf("Create long list and check indices:\n");
     {
+        unsigned long long start = usec();
         zl = ziplistNew();
         char buf[32];
         int i,len;
@@ -1800,7 +1979,7 @@ int ziplistTest(int argc, char **argv) {
             assert(ziplistGet(p,NULL,NULL,&value));
             assert(999-i == value);
         }
-        printf("SUCCESS\n\n");
+        printf("SUCCESS. usec=%lld\n\n", usec()-start);
         zfree(zl);
     }
 
@@ -1907,6 +2086,7 @@ int ziplistTest(int argc, char **argv) {
 
     printf("Stress with random payloads of different encoding:\n");
     {
+        unsigned long long start = usec();
         int i,j,len,where;
         unsigned char *p;
         char buf[1024];
@@ -1979,13 +2159,15 @@ int ziplistTest(int argc, char **argv) {
             zfree(zl);
             listRelease(ref);
         }
-        printf("SUCCESS\n\n");
+        printf("Done. usec=%lld\n\n", usec()-start);
     }
 
     printf("Stress with variable ziplist size:\n");
     {
+        unsigned long long start = usec();
         stress(ZIPLIST_HEAD,100000,16384,256);
         stress(ZIPLIST_TAIL,100000,16384,256);
+        printf("Done. usec=%lld\n\n", usec()-start);
     }
 
     printf("Stress __ziplistCascadeUpdate:\n");
diff --git a/src/ziplist.h b/src/ziplist.h
index 964a47f6d..8d1ac1691 100644
--- a/src/ziplist.h
+++ b/src/ziplist.h
@@ -45,10 +45,11 @@ unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char
 unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p);
 unsigned char *ziplistDeleteRange(unsigned char *zl, int index, unsigned int num);
 unsigned int ziplistCompare(unsigned char *p, unsigned char *s, unsigned int slen);
-unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int vlen, unsigned int skip);
+unsigned char *ziplistFind(unsigned char *zl, unsigned char *p, unsigned char *vstr, unsigned int vlen, unsigned int skip);
 unsigned int ziplistLen(unsigned char *zl);
 size_t ziplistBlobLen(unsigned char *zl);
 void ziplistRepr(unsigned char *zl);
+int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep);
 
 #ifdef REDIS_TEST
 int ziplistTest(int argc, char *argv[]);
diff --git a/src/zipmap.c b/src/zipmap.c
index 365c4aea4..bd41fe6a5 100644
--- a/src/zipmap.c
+++ b/src/zipmap.c
@@ -111,6 +111,10 @@ static unsigned int zipmapDecodeLength(unsigned char *p) {
     return len;
 }
 
+static unsigned int zipmapGetEncodedLengthSize(unsigned char *p) {
+    return (*p < ZIPMAP_BIGLEN) ? 1: 5;
+}
+
 /* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
  * the amount of bytes required to encode such a length. */
 static unsigned int zipmapEncodeLength(unsigned char *p, unsigned int len) {
@@ -370,6 +374,70 @@ size_t zipmapBlobLen(unsigned char *zm) {
     return totlen;
 }
 
+/* Validate the integrity of the data stracture.
+ * when `deep` is 0, only the integrity of the header is validated.
+ * when `deep` is 1, we scan all the entries one by one. */
+int zipmapValidateIntegrity(unsigned char *zm, size_t size, int deep) {
+#define OUT_OF_RANGE(p) ( \
+        (p) < zm + 2 || \
+        (p) > zm + size - 1)
+    unsigned int l, s, e;
+
+    /* check that we can actually read the header (or ZIPMAP_END). */
+    if (size < 2)
+        return 0;
+
+    /* the last byte must be the terminator. */
+    if (zm[size-1] != ZIPMAP_END)
+        return 0;
+
+    if (!deep)
+        return 1;
+
+    unsigned int count = 0;
+    unsigned char *p = zm + 1; /* skip the count */
+    while(*p != ZIPMAP_END) {
+        /* read the field name length encoding type */
+        s = zipmapGetEncodedLengthSize(p);
+        /* make sure the entry length doesn't rech outside the edge of the zipmap */
+        if (OUT_OF_RANGE(p+s))
+            return 0;
+
+        /* read the field name length */
+        l = zipmapDecodeLength(p);
+        p += s; /* skip the encoded field size */
+        p += l; /* skip the field */
+
+        /* make sure the entry doesn't rech outside the edge of the zipmap */
+        if (OUT_OF_RANGE(p))
+            return 0;
+
+        /* read the value length encoding type */
+        s = zipmapGetEncodedLengthSize(p);
+        /* make sure the entry length doesn't rech outside the edge of the zipmap */
+        if (OUT_OF_RANGE(p+s))
+            return 0;
+
+        /* read the value length */
+        l = zipmapDecodeLength(p);
+        p += s; /* skip the encoded value size*/
+        e = *p++; /* skip the encoded free space (always encoded in one byte) */
+        p += l+e; /* skip the value and free space */
+        count++;
+
+        /* make sure the entry doesn't rech outside the edge of the zipmap */
+        if (OUT_OF_RANGE(p))
+            return 0;
+    }
+
+    /* check that the count in the header is correct */
+    if (zm[0] != ZIPMAP_BIGLEN && zm[0] != count)
+        return 0;
+
+    return 1;
+#undef OUT_OF_RANGE
+}
+
 #ifdef REDIS_TEST
 static void zipmapRepr(unsigned char *p) {
     unsigned int l;
diff --git a/src/zipmap.h b/src/zipmap.h
index ac588f05a..daf8430a0 100644
--- a/src/zipmap.h
+++ b/src/zipmap.h
@@ -45,6 +45,7 @@ int zipmapExists(unsigned char *zm, unsigned char *key, unsigned int klen);
 unsigned int zipmapLen(unsigned char *zm);
 size_t zipmapBlobLen(unsigned char *zm);
 void zipmapRepr(unsigned char *p);
+int zipmapValidateIntegrity(unsigned char *zm, size_t size, int deep);
 
 #ifdef REDIS_TEST
 int zipmapTest(int argc, char *argv[]);
diff --git a/tests/assets/corrupt_ziplist.rdb b/tests/assets/corrupt_ziplist.rdb
new file mode 100644
index 0000000000000000000000000000000000000000..b40ada8d634d094ba2f20d728f20b19596e3e7ef
GIT binary patch
literal 1415
zcmaJ>yH3ME5WE!T6@e)D2D(_g=jTyTKtT%yJ;@ao7$n<4$tUnlG*No~f$uO9#!1cy
z*|V&CI%%|?ot<6X?e*2o9T6q<VPR)kIeE3k*cjuxQy=Z%hpZ}Bm$f&ovfQrU)W>~&
zFfZ+FlG}X!`94<P>MK_2^wBOaS~?d==q)zU!&yf={A4d*K4oP!@_nV$d3(I}+y1iu
z&()~2>C61Nvc;cTI$n)>btn+(lX1ELPr^eUtzCYvzbM9d*I!=`yS@i_ig{uQPjG&y
zy{3)DaZjgVn9o|!Pgo}tVIIahnXzxo0py_+*QFHeClYX>E=5=;8Qd3{ZcloCh5}D2
z%q77(nK4c&o(mJkN%B);{rtPnVE+R>+<$}TNQr=_xGrg--&n|J!OsZiGYkI2!oEp<
bM#3J39Y65RY~Brk>u~62;CyS^7jGLs8LoRE

literal 0
HcmV?d00001

diff --git a/tests/cluster/tests/04-resharding.tcl b/tests/cluster/tests/04-resharding.tcl
index cee2ec5ba..1dcdb5a2c 100644
--- a/tests/cluster/tests/04-resharding.tcl
+++ b/tests/cluster/tests/04-resharding.tcl
@@ -172,3 +172,10 @@ test "Verify slaves consistency" {
     }
     assert {$verified_masters >= 5}
 }
+
+test "Dump sanitization was skipped for migrations" {
+    set verified_masters 0
+    foreach_redis_id id {
+        assert {[RI $id dump_payload_sanitizations] == 0}
+    }
+}
diff --git a/tests/integration/corrupt-dump.tcl b/tests/integration/corrupt-dump.tcl
new file mode 100644
index 000000000..9864036d2
--- /dev/null
+++ b/tests/integration/corrupt-dump.tcl
@@ -0,0 +1,178 @@
+# tests of corrupt ziplist payload with valid CRC
+
+tags {"dump" "corruption"} {
+
+set corrupt_payload_7445 "\x0E\x01\x1D\x1D\x00\x00\x00\x16\x00\x00\x00\x03\x00\x00\x04\x43\x43\x43\x43\x06\x04\x42\x42\x42\x42\x06\x3F\x41\x41\x41\x41\xFF\x09\x00\x88\xA5\xCA\xA8\xC5\x41\xF4\x35"
+
+test {corrupt payload: #7445 - with sanitize} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload yes
+        catch {
+            r restore key 0 $corrupt_payload_7445
+        } err
+        assert_match "*Bad data format*" $err
+        verify_log_message 0 "*Ziplist integrity check failed*" 0
+    }
+}
+
+test {corrupt payload: #7445 - without sanitize - 1} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r config set crash-memcheck-enabled no ;# avoid valgrind issues
+        r restore key 0 $corrupt_payload_7445
+        catch {r lindex key 2}
+        verify_log_message 0 "*ASSERTION FAILED*" 0
+    }
+}
+
+test {corrupt payload: #7445 - without sanitize - 2} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r config set crash-memcheck-enabled no ;# avoid valgrind issues
+        r restore key 0 $corrupt_payload_7445
+        catch {r lset key 2 "BEEF"}
+        verify_log_message 0 "*ASSERTION FAILED*" 0
+    }
+}
+
+test {corrupt payload: hash with valid zip list header, invalid entry len} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r restore key 0 "\x0D\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x14\x63\x00\x04\x02\x64\x00\xFF\x09\x00\xD9\x10\x54\x92\x15\xF5\x5F\x52"
+        r config set crash-memcheck-enabled no ;# avoid valgrind issues
+        r config set hash-max-ziplist-entries 1
+        catch {r hset key b b}
+        verify_log_message 0 "*zipEntrySafe*" 0
+    }
+}
+
+test {corrupt payload: invalid zlbytes header} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        catch {
+            r restore key 0 "\x0D\x1B\x25\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x02\x63\x00\x04\x02\x64\x00\xFF\x09\x00\xB7\xF7\x6E\x9F\x43\x43\x14\xC6"
+        } err
+        assert_match "*Bad data format*" $err
+    }
+}
+
+test {corrupt payload: valid zipped hash header, dup records} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r restore key 0 "\x0D\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x02\x61\x00\x04\x02\x64\x00\xFF\x09\x00\xA1\x98\x36\x78\xCC\x8E\x93\x2E"
+        r config set crash-memcheck-enabled no ;# avoid valgrind issues
+        r config set hash-max-ziplist-entries 1
+        # cause an assertion when converting to hash table
+        catch {r hset key b b}
+        verify_log_message 0 "*ziplist with dup elements dump*" 0
+    }
+}
+
+test {corrupt payload: quicklist big ziplist prev len} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r restore key 0 "\x0E\x01\x13\x13\x00\x00\x00\x0E\x00\x00\x00\x02\x00\x00\x02\x61\x00\x0E\x02\x62\x00\xFF\x09\x00\x49\x97\x30\xB2\x0D\xA1\xED\xAA"
+        r config set crash-memcheck-enabled no ;# avoid valgrind issues
+        catch {r lindex key -2}
+        verify_log_message 0 "*ASSERTION FAILED*" 0
+    }
+}
+
+test {corrupt payload: quicklist small ziplist prev len} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload yes
+        catch {
+            r restore key 0 "\x0E\x01\x13\x13\x00\x00\x00\x0E\x00\x00\x00\x02\x00\x00\x02\x61\x00\x02\x02\x62\x00\xFF\x09\x00\xC7\x71\x03\x97\x07\x75\xB0\x63"
+        } err
+        assert_match "*Bad data format*" $err
+        verify_log_message 0 "*Ziplist integrity check failed*" 0
+    }
+}
+
+test {corrupt payload: quicklist ziplist wrong count} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r restore key 0 "\x0E\x01\x13\x13\x00\x00\x00\x0E\x00\x00\x00\x03\x00\x00\x02\x61\x00\x04\x02\x62\x00\xFF\x09\x00\x4D\xE2\x0A\x2F\x08\x25\xDF\x91"
+        r lpush key a
+        # check that the server didn't crash
+        r ping
+    }
+}
+
+test {corrupt payload: #3080 - quicklist} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        catch {
+            r RESTORE key 0 "\x0E\x01\x80\x00\x00\x00\x10\x41\x41\x41\x41\x41\x41\x41\x41\x02\x00\x00\x80\x41\x41\x41\x41\x07\x00\x03\xC7\x1D\xEF\x54\x68\xCC\xF3"
+            r DUMP key
+        }
+        assert_match "*Bad data format*" $err
+        verify_log_message 0 "*Ziplist integrity check failed*" 0
+    }
+}
+
+test {corrupt payload: #3080 - ziplist} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        catch {
+            r RESTORE key 0 "\x0A\x80\x00\x00\x00\x10\x41\x41\x41\x41\x41\x41\x41\x41\x02\x00\x00\x80\x41\x41\x41\x41\x07\x00\x39\x5B\x49\xE0\xC1\xC6\xDD\x76"
+        }
+        assert_match "*Bad data format*" $err
+        verify_log_message 0 "*Ziplist integrity check failed*" 0
+    }
+}
+
+test {corrupt payload: load corrupted rdb with no CRC - #3505} {
+    set server_path [tmpdir "server.rdb-corruption-test"]
+    exec cp tests/assets/corrupt_ziplist.rdb $server_path
+    set srv [start_server [list overrides [list "dir" $server_path "dbfilename" "corrupt_ziplist.rdb" loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no]]]
+
+    # wait for termination
+    wait_for_condition 100 50 {
+        ! [is_alive $srv]
+    } else {
+        fail "rdb loading didn't fail"
+    }
+
+    set stdout [dict get $srv stdout]
+    assert_equal [count_message_lines $stdout "Terminating server after rdb file reading failure."]  1
+    assert_lessthan 1 [count_message_lines $stdout "Ziplist integrity check failed"]
+    kill_server $srv ;# let valgrind look for issues
+}
+
+test {corrupt payload: listpack invalid size header} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        catch {
+            r restore key 0 "\x0F\x01\x10\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x40\x55\x5F\x00\x00\x00\x0F\x00\x01\x01\x00\x01\x02\x01\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x00\x01\x00\x01\x00\x01\x00\x01\x02\x02\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x61\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x88\x62\x00\x00\x00\x00\x00\x00\x00\x09\x08\x01\xFF\x0A\x01\x00\x00\x09\x00\x45\x91\x0A\x87\x2F\xA5\xF9\x2E"
+        } err
+        assert_match "*Bad data format*" $err
+        verify_log_message 0 "*Stream listpack integrity check failed*" 0
+    }
+}
+
+test {corrupt payload: listpack too long entry len} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r restore key 0 "\x0F\x01\x10\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x40\x55\x55\x00\x00\x00\x0F\x00\x01\x01\x00\x01\x02\x01\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x00\x01\x00\x01\x00\x01\x00\x01\x02\x02\x89\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x61\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x88\x62\x00\x00\x00\x00\x00\x00\x00\x09\x08\x01\xFF\x0A\x01\x00\x00\x09\x00\x40\x63\xC9\x37\x03\xA2\xE5\x68"
+        catch {
+            r xinfo stream key full
+        } err
+        verify_log_message 0 "*ASSERTION FAILED*" 0
+    }
+}
+
+test {corrupt payload: listpack very long entry len} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r restore key 0 "\x0F\x01\x10\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x40\x55\x55\x00\x00\x00\x0F\x00\x01\x01\x00\x01\x02\x01\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x00\x01\x00\x01\x00\x01\x00\x01\x02\x02\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x61\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x9C\x62\x00\x00\x00\x00\x00\x00\x00\x09\x08\x01\xFF\x0A\x01\x00\x00\x09\x00\x63\x6F\x42\x8E\x7C\xB5\xA2\x9D"
+        catch {
+            r xinfo stream key full
+        } err
+        verify_log_message 0 "*ASSERTION FAILED*" 0
+    }
+}
+
+test {corrupt payload: listpack too long entry prev len} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload yes
+        catch {
+            r restore key 0 "\x0F\x01\x10\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x40\x55\x55\x00\x00\x00\x0F\x00\x01\x01\x00\x15\x02\x01\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x00\x01\x00\x01\x00\x01\x00\x01\x02\x02\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x61\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x88\x62\x00\x00\x00\x00\x00\x00\x00\x09\x08\x01\xFF\x0A\x01\x00\x00\x09\x00\x06\xFB\x44\x24\x0A\x8E\x75\xEA"
+        } err
+        assert_match "*Bad data format*" $err
+        verify_log_message 0 "*Stream listpack integrity check failed*" 0
+    }
+}
+
+} ;# tags
+
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index 3268bc974..aadec7cd1 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -112,6 +112,22 @@ proc count_log_lines {srv_idx} {
     set _ [exec wc -l < [srv $srv_idx stdout]]
 }
 
+# returns the number of times a line with that pattern appears in a file
+proc count_message_lines {file pattern} {
+    set res 0
+    # exec fails when grep exists with status other than 0 (when the patter wasn't found)
+    catch {
+        set res [exec grep $pattern $file 2> /dev/null | wc -l]
+    }
+    return $res
+}
+
+# returns the number of times a line with that pattern appears in the log
+proc count_log_message {srv_idx pattern} {
+    set stdout [srv $srv_idx stdout]
+    return [count_message_lines $stdout $pattern]
+}
+
 # verify pattern exists in server's sdtout after a certain line number
 proc verify_log_message {srv_idx pattern from_line} {
     incr from_line
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 6a38cc001..998e82bd9 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -44,6 +44,7 @@ set ::all_tests {
     integration/replication-psync
     integration/aof
     integration/rdb
+    integration/corrupt-dump
     integration/convert-zipmap-hash-on-load
     integration/logging
     integration/psync2

From 01c13bddea6c09e0a60678fa48becd42b0913515 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 22 Nov 2020 20:58:33 +0200
Subject: [PATCH 0843/1098] Sanitize dump payload: improve tests of ziplist and
 stream encodings

- improve stream rdb encoding test to include more types of stream metadata
- add test to cover various ziplist encoding entries (although it does
  look like the stress test above it is able to find some too
- add another test for ziplist encoding for hash with full sanitization
- add similar ziplist encoding tests for list
---
 tests/integration/rdb.tcl | 13 ++++++++--
 tests/unit/type/hash.tcl  | 51 +++++++++++++++++++++++++++++++++++++++
 tests/unit/type/list.tcl  | 48 ++++++++++++++++++++++++++++++++++++
 3 files changed, 110 insertions(+), 2 deletions(-)

diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl
index 58dc6c968..d29f9d88e 100644
--- a/tests/integration/rdb.tcl
+++ b/tests/integration/rdb.tcl
@@ -49,13 +49,22 @@ start_server [list overrides [list "dir" $server_path] keep_persistence true] {
             }
         }
         r xgroup create stream mygroup 0
-        r xreadgroup GROUP mygroup Alice COUNT 1 STREAMS stream >
+        set records [r xreadgroup GROUP mygroup Alice COUNT 2 STREAMS stream >]
+        r xack stream mygroup [lindex [lindex [lindex [lindex $records 0] 1] 0] 0]
         set digest [r debug digest]
+        r config set sanitize-dump-payload no
         r debug reload
         set newdigest [r debug digest]
         assert {$digest eq $newdigest}
-        r del stream
     }
+    test {Test RDB stream encoding - sanitize dump} {
+        r config set sanitize-dump-payload yes
+        r debug reload
+        set newdigest [r debug digest]
+        assert {$digest eq $newdigest}
+    }
+    # delete the stream, maybe valgrind will find something
+    r del stream
 }
 
 # Helper function to start a server and kill it, just to check the error
diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl
index 257404ea3..79e58301a 100644
--- a/tests/unit/type/hash.tcl
+++ b/tests/unit/type/hash.tcl
@@ -540,4 +540,55 @@ start_server {tags {"hash"}} {
             assert {[r hincrbyfloat myhash float -0.1] eq {1.9}}
         }
     }
+
+    test {Hash ziplist of various encodings} {
+        r del k
+        r config set hash-max-ziplist-entries 1000000000
+        r config set hash-max-ziplist-value 1000000000
+        r hset k ZIP_INT_8B 127
+        r hset k ZIP_INT_16B 32767
+        r hset k ZIP_INT_32B 2147483647
+        r hset k ZIP_INT_64B 9223372036854775808
+        r hset k ZIP_INT_IMM_MIN 0
+        r hset k ZIP_INT_IMM_MAX 12
+        r hset k ZIP_STR_06B [string repeat x 31]
+        r hset k ZIP_STR_14B [string repeat x 8191]
+        r hset k ZIP_STR_32B [string repeat x 65535]
+        set k [r hgetall k]
+        set dump [r dump k]
+
+        # will be converted to dict at RESTORE
+        r config set hash-max-ziplist-entries 2
+        r config set sanitize-dump-payload no
+        r restore kk 0 $dump
+        set kk [r hgetall kk]
+
+        # make sure the values are right
+        assert_equal [lsort $k] [lsort $kk]
+        assert_equal [dict get $k ZIP_STR_06B] [string repeat x 31]
+        set k [dict remove $k ZIP_STR_06B]
+        assert_equal [dict get $k ZIP_STR_14B] [string repeat x 8191]
+        set k [dict remove $k ZIP_STR_14B]
+        assert_equal [dict get $k ZIP_STR_32B] [string repeat x 65535]
+        set k [dict remove $k ZIP_STR_32B]
+        set _ $k
+    } {ZIP_INT_8B 127 ZIP_INT_16B 32767 ZIP_INT_32B 2147483647 ZIP_INT_64B 9223372036854775808 ZIP_INT_IMM_MIN 0 ZIP_INT_IMM_MAX 12}
+
+    test {Hash ziplist of various encodings - sanitize dump} {
+        r config set sanitize-dump-payload yes
+        r restore kk 0 $dump replace
+        set k [r hgetall k]
+        set kk [r hgetall kk]
+
+        # make sure the values are right
+        assert_equal [lsort $k] [lsort $kk]
+        assert_equal [dict get $k ZIP_STR_06B] [string repeat x 31]
+        set k [dict remove $k ZIP_STR_06B]
+        assert_equal [dict get $k ZIP_STR_14B] [string repeat x 8191]
+        set k [dict remove $k ZIP_STR_14B]
+        assert_equal [dict get $k ZIP_STR_32B] [string repeat x 65535]
+        set k [dict remove $k ZIP_STR_32B]
+        set _ $k
+    } {ZIP_INT_8B 127 ZIP_INT_16B 32767 ZIP_INT_32B 2147483647 ZIP_INT_64B 9223372036854775808 ZIP_INT_IMM_MIN 0 ZIP_INT_IMM_MAX 12}
+
 }
diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl
index 600a7ce70..61ca23377 100644
--- a/tests/unit/type/list.tcl
+++ b/tests/unit/type/list.tcl
@@ -1089,4 +1089,52 @@ start_server {
         r lpush l foo
         assert_equal {l foo} [$rd read]
     } {}
+
+    test {List ziplist of various encodings} {
+        r del k
+        r lpush k 127 ;# ZIP_INT_8B
+        r lpush k 32767 ;# ZIP_INT_16B
+        r lpush k 2147483647 ;# ZIP_INT_32B
+        r lpush k 9223372036854775808 ;# ZIP_INT_64B
+        r lpush k 0 ;# ZIP_INT_IMM_MIN
+        r lpush k 12 ;# ZIP_INT_IMM_MAX
+        r lpush k [string repeat x 31] ;# ZIP_STR_06B
+        r lpush k [string repeat x 8191] ;# ZIP_STR_14B
+        r lpush k [string repeat x 65535] ;# ZIP_STR_32B
+        set k [r lrange k 0 -1]
+        set dump [r dump k]
+
+        r config set sanitize-dump-payload no
+        r restore kk 0 $dump
+        set kk [r lrange kk 0 -1]
+
+        # try some forward and backward searches to make sure all encodings
+        # can be traversed
+        assert_equal [r lindex kk 5] {9223372036854775808}
+        assert_equal [r lindex kk -5] {0}
+        assert_equal [r lpos kk foo rank 1] {}
+        assert_equal [r lpos kk foo rank -1] {}
+
+        # make sure the values are right
+        assert_equal $k $kk
+        assert_equal [lpop k] [string repeat x 65535]
+        assert_equal [lpop k] [string repeat x 8191]
+        assert_equal [lpop k] [string repeat x 31]
+        set _ $k
+    } {12 0 9223372036854775808 2147483647 32767 127}
+
+    test {List ziplist of various encodings - sanitize dump} {
+        r config set sanitize-dump-payload yes
+        r restore kk 0 $dump replace
+        set k [r lrange k 0 -1]
+        set kk [r lrange kk 0 -1]
+
+        # make sure the values are right
+        assert_equal $k $kk
+        assert_equal [lpop k] [string repeat x 65535]
+        assert_equal [lpop k] [string repeat x 8191]
+        assert_equal [lpop k] [string repeat x 31]
+        set _ $k
+    } {12 0 9223372036854775808 2147483647 32767 127}
+
 }

From c31055db617a7fedb20c28c245ddad8ed2b11931 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 14 Aug 2020 16:05:34 +0300
Subject: [PATCH 0844/1098] Sanitize dump payload: fuzz tester and fixes for
 segfaults and leaks it exposed

The test creates keys with various encodings, DUMP them, corrupt the payload
and RESTORES it.
It utilizes the recently added use-exit-on-panic config to distinguish between
 asserts and segfaults.
If the restore succeeds, it runs random commands on the key to attempt to
trigger a crash.

It runs in two modes, one with deep sanitation enabled and one without.
In the first one we don't expect any assertions or segfaults, in the second one
we expect assertions, but no segfaults.
We also check for leaks and invalid reads using valgrind, and if we find them
we print the commands that lead to that issue.

Changes in the code (other than the test):
- Replace a few NPD (null pointer deference) flows and division by zero with an
  assertion, so that it doesn't fail the test. (since we set the server to use
  `exit` rather than `abort` on assertion).
- Fix quite a lot of flows in rdb.c that could have lead to memory leaks in
  RESTORE command (since it now responds with an error rather than panic)
- Add a DEBUG flag for SET-SKIP-CHECKSUM-VALIDATION so that the test don't need
  to bother with faking a valid checksum
- Remove a pile of code in serverLogObjectDebugInfo which is actually unsafe to
  run in the crash report (see comments in the code)
- fix a missing boundary check in lzf_decompress

test suite infra improvements:
- be able to run valgrind checks before the process terminates
- rotate log files when restarting servers
---
 src/cluster.c                             |   3 +
 src/db.c                                  |   1 +
 src/debug.c                               |  15 ++
 src/intset.c                              |   5 +-
 src/listpack.c                            |   1 +
 src/lzf_d.c                               |   6 +-
 src/networking.c                          |   1 +
 src/quicklist.c                           |   4 +-
 src/rdb.c                                 |  80 ++++--
 src/server.c                              |   1 +
 src/server.h                              |   1 +
 src/t_list.c                              |   1 +
 src/t_stream.c                            |   2 +
 src/valgrind.sup                          |   9 +
 tests/instances.tcl                       |   2 +-
 tests/integration/corrupt-dump-fuzzer.tcl | 193 ++++++++++++++
 tests/integration/corrupt-dump.tcl        | 296 ++++++++++++++++++++--
 tests/integration/psync2.tcl              |   4 +-
 tests/integration/rdb.tcl                 |   2 +-
 tests/support/server.tcl                  |  18 +-
 tests/support/util.tcl                    | 130 +++++++++-
 tests/test_helper.tcl                     |   1 +
 tests/unit/introspection.tcl              |   2 +-
 23 files changed, 722 insertions(+), 56 deletions(-)
 create mode 100644 tests/integration/corrupt-dump-fuzzer.tcl

diff --git a/src/cluster.c b/src/cluster.c
index 6116614ea..ef73b1093 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4981,6 +4981,9 @@ int verifyDumpPayload(unsigned char *p, size_t len) {
     rdbver = (footer[1] << 8) | footer[0];
     if (rdbver > RDB_VERSION) return C_ERR;
 
+    if (server.skip_checksum_validation)
+        return C_OK;
+
     /* Verify CRC64 */
     crc = crc64(0,p,len-8);
     memrev64ifbe(&crc);
diff --git a/src/db.c b/src/db.c
index 45865b3e8..12e1c2bbe 100644
--- a/src/db.c
+++ b/src/db.c
@@ -960,6 +960,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) {
          * value, or skip it if it was not filtered: we only match keys. */
         if (o && (o->type == OBJ_ZSET || o->type == OBJ_HASH)) {
             node = nextnode;
+            serverAssert(node); /* assertion for valgrind (avoid NPD) */
             nextnode = listNextNode(node);
             if (filter) {
                 kobj = listNodeValue(node);
diff --git a/src/debug.c b/src/debug.c
index d32a8ddce..7cc3c15a3 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -403,6 +403,7 @@ void debugCommand(client *c) {
 "SDSLEN <key> -- Show low level SDS string info representing key and value.",
 "SEGFAULT -- Crash the server with sigsegv.",
 "SET-ACTIVE-EXPIRE <0|1> -- Setting it to 0 disables expiring keys in background when they are not accessed (otherwise the Redis behavior). Setting it to 1 reenables back the default.",
+"SET-SKIP-CHECKSUM-VALIDATION <0|1> -- Enables or disables checksum checks for rdb or RESTORE payload.",
 "AOF-FLUSH-SLEEP <microsec> -- Server will sleep before flushing the AOF, this is used for testing",
 "SLEEP <seconds> -- Stop the server for <seconds>. Decimals allowed.",
 "STRUCTSIZE -- Return the size of different Redis core C structures.",
@@ -722,6 +723,11 @@ NULL
     {
         server.active_expire_enabled = atoi(c->argv[2]->ptr);
         addReply(c,shared.ok);
+    } else if (!strcasecmp(c->argv[1]->ptr,"set-skip-checksum-validation") &&
+               c->argc == 3)
+    {
+        server.skip_checksum_validation = atoi(c->argv[2]->ptr);
+        addReply(c,shared.ok);
     } else if (!strcasecmp(c->argv[1]->ptr,"aof-flush-sleep") &&
                c->argc == 3)
     {
@@ -880,6 +886,14 @@ void serverLogObjectDebugInfo(const robj *o) {
     serverLog(LL_WARNING,"Object type: %d", o->type);
     serverLog(LL_WARNING,"Object encoding: %d", o->encoding);
     serverLog(LL_WARNING,"Object refcount: %d", o->refcount);
+#if UNSAFE_CRASH_REPORT
+    /* This code is now disabled. o->ptr may be unreliable to print. in some
+     * cases a ziplist could have already been freed by realloc, but not yet
+     * updated to o->ptr. in other cases the call to ziplistLen may need to
+     * iterate on all the items in the list (and possibly crash again).
+     * For some cases it may be ok to crash here again, but these could cause
+     * invalid memory access which will bother valgrind and also possibly cause
+     * random memory portion to be "leaked" into the logfile. */
     if (o->type == OBJ_STRING && sdsEncodedObject(o)) {
         serverLog(LL_WARNING,"Object raw string len: %zu", sdslen(o->ptr));
         if (sdslen(o->ptr) < 4096) {
@@ -900,6 +914,7 @@ void serverLogObjectDebugInfo(const robj *o) {
     } else if (o->type == OBJ_STREAM) {
         serverLog(LL_WARNING,"Stream size: %d", (int) streamLength(o));
     }
+#endif
 }
 
 void _serverAssertPrintObject(const robj *o) {
diff --git a/src/intset.c b/src/intset.c
index 0079199cf..964a41187 100644
--- a/src/intset.c
+++ b/src/intset.c
@@ -34,6 +34,7 @@
 #include "intset.h"
 #include "zmalloc.h"
 #include "endianconv.h"
+#include "redisassert.h"
 
 /* Note that these encodings are ordered, so:
  * INTSET_ENC_INT16 < INTSET_ENC_INT32 < INTSET_ENC_INT64. */
@@ -258,7 +259,9 @@ uint8_t intsetFind(intset *is, int64_t value) {
 
 /* Return random member */
 int64_t intsetRandom(intset *is) {
-    return _intsetGet(is,rand()%intrev32ifbe(is->length));
+    uint32_t len = intrev32ifbe(is->length);
+    assert(len); /* avoid division by zero on corrupt intset payload. */
+    return _intsetGet(is,rand()%len);
 }
 
 /* Get the value at the given position. When this position is
diff --git a/src/listpack.c b/src/listpack.c
index 580954075..b403a1200 100644
--- a/src/listpack.c
+++ b/src/listpack.c
@@ -536,6 +536,7 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) {
     int64_t val;
     uint64_t uval, negstart, negmax;
 
+    assert(p); /* assertion for valgrind (avoid NPD) */
     if (LP_ENCODING_IS_7BIT_UINT(p[0])) {
         negstart = UINT64_MAX; /* 7 bit ints are always positive. */
         negmax = 0;
diff --git a/src/lzf_d.c b/src/lzf_d.c
index 5f55d471d..dbf1ae772 100644
--- a/src/lzf_d.c
+++ b/src/lzf_d.c
@@ -65,9 +65,10 @@ lzf_decompress (const void *const in_data,  unsigned int in_len,
   u8 const *const in_end  = ip + in_len;
   u8       *const out_end = op + out_len;
 
-  do
+  while (ip < in_end)
     {
-      unsigned int ctrl = *ip++;
+      unsigned int ctrl;
+      ctrl = *ip++;
 
       if (ctrl < (1 << 5)) /* literal run */
         {
@@ -182,7 +183,6 @@ lzf_decompress (const void *const in_data,  unsigned int in_len,
 #endif
         }
     }
-  while (ip < in_end);
 
   return op - (u8 *)out_data;
 }
diff --git a/src/networking.c b/src/networking.c
index 626b117a1..82b6428ac 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -674,6 +674,7 @@ void addReplyLongLong(client *c, long long ll) {
 }
 
 void addReplyAggregateLen(client *c, long length, int prefix) {
+    serverAssert(length >= 0);
     if (prefix == '*' && length < OBJ_SHARED_BULKHDR_LEN)
         addReply(c,shared.mbulkhdr[length]);
     else
diff --git a/src/quicklist.c b/src/quicklist.c
index 0916d1f1b..8d68aa9ae 100644
--- a/src/quicklist.c
+++ b/src/quicklist.c
@@ -34,6 +34,7 @@
 #include "ziplist.h"
 #include "util.h" /* for ll2string */
 #include "lzf.h"
+#include "redisassert.h"
 
 #if defined(REDIS_TEST) || defined(REDIS_TEST_VERBOSE)
 #include <stdio.h> /* for printf (debug printing), snprintf (genstr) */
@@ -1289,7 +1290,8 @@ int quicklistIndex(const quicklist *quicklist, const long long idx,
 
     quicklistDecompressNodeForUse(entry->node);
     entry->zi = ziplistIndex(entry->node->zl, entry->offset);
-    ziplistGet(entry->zi, &entry->value, &entry->sz, &entry->longval);
+    if (!ziplistGet(entry->zi, &entry->value, &entry->sz, &entry->longval))
+        assert(0); /* This can happen on corrupt ziplist with fake entry count. */
     /* The caller will use our result, so we don't re-compress here.
      * The caller can recompress or delete the node as needed. */
     return 1;
diff --git a/src/rdb.c b/src/rdb.c
index af527f94c..e88cabb5d 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -399,8 +399,9 @@ void *rdbLoadLzfStringObject(rio *rdb, int flags, size_t *lenptr) {
 
     /* Load the compressed representation and uncompress it to target. */
     if (rioRead(rdb,c,clen) == 0) goto err;
-    if (lzf_decompress(c,clen,val,len) == 0) {
+    if (lzf_decompress(c,clen,val,len) != len) {
         rdbExitReportCorruptRDB("Invalid LZF compressed string");
+        goto err;
     }
     zfree(c);
 
@@ -504,6 +505,8 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags, size_t *lenptr) {
     unsigned long long len;
 
     len = rdbLoadLen(rdb,&isencoded);
+    if (len == RDB_LENERR) return NULL;
+
     if (isencoded) {
         switch(len) {
         case RDB_ENC_INT8:
@@ -518,7 +521,6 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags, size_t *lenptr) {
         }
     }
 
-    if (len == RDB_LENERR) return NULL;
     if (plain || sds) {
         void *buf = plain ? zmalloc(len) : sdsnewlen(SDS_NOINIT,len);
         if (lenptr) *lenptr = len;
@@ -604,7 +606,7 @@ int rdbLoadDoubleValue(rio *rdb, double *val) {
     default:
         if (rioRead(rdb,buf,len) == 0) return -1;
         buf[len] = '\0';
-        sscanf(buf, "%lg", val);
+        if (sscanf(buf, "%lg", val)!=1) return -1;
         return 0;
     }
 }
@@ -1572,7 +1574,12 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             /* This will also be called when the set was just converted
              * to a regular hash table encoded set. */
             if (o->encoding == OBJ_ENCODING_HT) {
-                dictAdd((dict*)o->ptr,sdsele,NULL);
+                if (dictAdd((dict*)o->ptr,sdsele,NULL) != DICT_OK) {
+                    rdbExitReportCorruptRDB("Duplicate set members detected");
+                    decrRefCount(o);
+                    sdsfree(sdsele);
+                    return NULL;
+                }
             } else {
                 sdsfree(sdsele);
             }
@@ -1693,7 +1700,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             /* Add pair to hash table */
             ret = dictAdd((dict*)o->ptr, field, value);
             if (ret == DICT_ERR) {
-                rdbExitReportCorruptRDB("Duplicate keys detected");
+                rdbExitReportCorruptRDB("Duplicate hash fields detected");
+                sdsfree(value);
+                sdsfree(field);
+                decrRefCount(o);
+                return NULL;
             }
         }
 
@@ -1843,6 +1854,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             if (sdslen(nodekey) != sizeof(streamID)) {
                 rdbExitReportCorruptRDB("Stream node key entry is not the "
                                         "size of a stream ID");
+                sdsfree(nodekey);
+                decrRefCount(o);
+                return NULL;
             }
 
             /* Load the listpack. */
@@ -1870,14 +1884,22 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                  * deletion we should remove the radix tree key if the
                  * resulting listpack is empty. */
                 rdbExitReportCorruptRDB("Empty listpack inside stream");
+                sdsfree(nodekey);
+                decrRefCount(o);
+                zfree(lp);
+                return NULL;
             }
 
             /* Insert the key in the radix tree. */
             int retval = raxInsert(s->rax,
                 (unsigned char*)nodekey,sizeof(streamID),lp,NULL);
             sdsfree(nodekey);
-            if (!retval)
+            if (!retval) {
                 rdbExitReportCorruptRDB("Listpack re-added with existing key");
+                decrRefCount(o);
+                zfree(lp);
+                return NULL;
+            }
         }
         /* Load total number of items inside the stream. */
         s->length = rdbLoadLen(rdb,NULL);
@@ -1922,9 +1944,13 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             }
 
             streamCG *cgroup = streamCreateCG(s,cgname,sdslen(cgname),&cg_id);
-            if (cgroup == NULL)
+            if (cgroup == NULL) {
                 rdbExitReportCorruptRDB("Duplicated consumer group name %s",
                                          cgname);
+                decrRefCount(o);
+                sdsfree(cgname);
+                return NULL;
+            }
             sdsfree(cgname);
 
             /* Load the global PEL for this consumer group, however we'll
@@ -1954,9 +1980,13 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                     streamFreeNACK(nack);
                     return NULL;
                 }
-                if (!raxInsert(cgroup->pel,rawid,sizeof(rawid),nack,NULL))
+                if (!raxInsert(cgroup->pel,rawid,sizeof(rawid),nack,NULL)) {
                     rdbExitReportCorruptRDB("Duplicated global PEL entry "
                                             "loading stream consumer group");
+                    decrRefCount(o);
+                    streamFreeNACK(nack);
+                    return NULL;
+                }
             }
 
             /* Now that we loaded our global PEL, we need to load the
@@ -2003,18 +2033,24 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                         return NULL;
                     }
                     streamNACK *nack = raxFind(cgroup->pel,rawid,sizeof(rawid));
-                    if (nack == raxNotFound)
+                    if (nack == raxNotFound) {
                         rdbExitReportCorruptRDB("Consumer entry not found in "
                                                 "group global PEL");
+                        decrRefCount(o);
+                        return NULL;
+                    }
 
                     /* Set the NACK consumer, that was left to NULL when
                      * loading the global PEL. Then set the same shared
                      * NACK structure also in the consumer-specific PEL. */
                     nack->consumer = consumer;
-                    if (!raxInsert(consumer->pel,rawid,sizeof(rawid),nack,NULL))
+                    if (!raxInsert(consumer->pel,rawid,sizeof(rawid),nack,NULL)) {
                         rdbExitReportCorruptRDB("Duplicated consumer PEL entry "
                                                 " loading a stream consumer "
                                                 "group");
+                        decrRefCount(o);
+                        return NULL;
+                    }
                 }
             }
         }
@@ -2034,8 +2070,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
 
         if (mt == NULL) {
             moduleTypeNameByID(name,moduleid);
-            serverLog(LL_WARNING,"The RDB file contains module data I can't load: no matching module '%s'", name);
-            exit(1);
+            rdbExitReportCorruptRDB("The RDB file contains module data I can't load: no matching module '%s'", name);
+            return NULL;
         }
         RedisModuleIO io;
         robj keyobj;
@@ -2054,20 +2090,26 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
         if (io.ver == 2) {
             uint64_t eof = rdbLoadLen(rdb,NULL);
             if (eof == RDB_LENERR) {
-                o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */
-                decrRefCount(o);
+                if (ptr) {
+                    o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */
+                    decrRefCount(o);
+                }
                 return NULL;
             }
             if (eof != RDB_MODULE_OPCODE_EOF) {
-                serverLog(LL_WARNING,"The RDB file contains module data for the module '%s' that is not terminated by the proper module value EOF marker", name);
-                exit(1);
+                rdbExitReportCorruptRDB("The RDB file contains module data for the module '%s' that is not terminated by the proper module value EOF marker", name);
+                if (ptr) {
+                    o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */
+                    decrRefCount(o);
+                }
+                return NULL;
             }
         }
 
         if (ptr == NULL) {
             moduleTypeNameByID(name,moduleid);
-            serverLog(LL_WARNING,"The RDB file contains module data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name);
-            exit(1);
+            rdbExitReportCorruptRDB("The RDB file contains module data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name);
+            return NULL;
         }
         o = createModuleObject(mt,ptr);
     } else {
@@ -2441,7 +2483,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
         uint64_t cksum, expected = rdb->cksum;
 
         if (rioRead(rdb,&cksum,8) == 0) goto eoferr;
-        if (server.rdb_checksum) {
+        if (server.rdb_checksum && !server.skip_checksum_validation) {
             memrev64ifbe(&cksum);
             if (cksum == 0) {
                 serverLog(LL_WARNING,"RDB file was saved with checksum disabled: no check performed.");
diff --git a/src/server.c b/src/server.c
index 1b6f71aef..0c3bbff03 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2506,6 +2506,7 @@ void initServerConfig(void) {
     server.tlsfd_count = 0;
     server.sofd = -1;
     server.active_expire_enabled = 1;
+    server.skip_checksum_validation = 0;
     server.client_max_querybuf_len = PROTO_MAX_QUERYBUF_LEN;
     server.saveparams = NULL;
     server.loading = 0;
diff --git a/src/server.h b/src/server.h
index 8a35d48df..47f054b3f 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1245,6 +1245,7 @@ struct redisServer {
     int active_expire_effort;       /* From 1 (default) to 10, active effort. */
     int active_defrag_enabled;
     int sanitize_dump_payload;      /* Enables deep sanitization for ziplist and listpack in RDB and RESTORE. */
+    int skip_checksum_validation;   /* Disables checksum validateion for RDB and RESTORE payload. */
     int jemalloc_bg_thread;         /* Enable jemalloc background thread */
     size_t active_defrag_ignore_bytes; /* minimum amount of fragmentation waste to start active defrag */
     int active_defrag_threshold_lower; /* minimum percentage of fragmentation to start active defrag */
diff --git a/src/t_list.c b/src/t_list.c
index acabe7be2..42b4f92df 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -728,6 +728,7 @@ void lmoveGenericCommand(client *c, int wherefrom, int whereto) {
 
         if (checkType(c,dobj,OBJ_LIST)) return;
         value = listTypePop(sobj,wherefrom);
+        serverAssert(value); /* assertion for valgrind (avoid NPD) */
         /* We saved touched key, and protect it, since lmoveHandlePush
          * may change the client command argument vector (it does not
          * currently). */
diff --git a/src/t_stream.c b/src/t_stream.c
index 70cc8bfa1..320c5a2fe 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -787,6 +787,7 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) {
                 *numfields = lpGetInteger(si->lp_ele);
                 si->lp_ele = lpNext(si->lp,si->lp_ele);
             }
+            serverAssert(*numfields>=0);
 
             /* If current >= start, and the entry is not marked as
              * deleted, emit it. */
@@ -2879,6 +2880,7 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) {
                     addReplyStreamID(c,&id);
 
                     /* Consumer name. */
+                    serverAssert(nack->consumer); /* assertion for valgrind (avoid NPD) */
                     addReplyBulkCBuffer(c,nack->consumer->name,
                                         sdslen(nack->consumer->name));
 
diff --git a/src/valgrind.sup b/src/valgrind.sup
index b05843d8c..5d6367e3f 100644
--- a/src/valgrind.sup
+++ b/src/valgrind.sup
@@ -15,3 +15,12 @@
    Memcheck:Value8
    fun:lzf_compress
 }
+
+{
+   <negative size allocatoin, see integration/corrupt-dump>
+   Memcheck:FishyValue
+   malloc(size)
+   fun:malloc
+   fun:ztrymalloc_usable
+   fun:ztrymalloc
+}
diff --git a/tests/instances.tcl b/tests/instances.tcl
index d3b1b50cd..156c92706 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -155,7 +155,7 @@ proc log_crashes {} {
 
     set logs [glob */err.txt]
     foreach log $logs {
-        set res [find_valgrind_errors $log]
+        set res [find_valgrind_errors $log true]
         if {$res != ""} {
             puts $res
             incr ::failed
diff --git a/tests/integration/corrupt-dump-fuzzer.tcl b/tests/integration/corrupt-dump-fuzzer.tcl
new file mode 100644
index 000000000..d41ed2941
--- /dev/null
+++ b/tests/integration/corrupt-dump-fuzzer.tcl
@@ -0,0 +1,193 @@
+# tests of corrupt ziplist payload with valid CRC
+
+tags {"dump" "corruption"} {
+
+proc generate_collections {suffix elements} {
+    set rd [redis_deferring_client]
+    for {set j 0} {$j < $elements} {incr j} {
+        # add both string values and integers
+        if {$j % 2 == 0} {set val $j} else {set val "_$j"}
+        $rd hset hash$suffix $j $val
+        $rd lpush list$suffix $val
+        $rd zadd zset$suffix $j $val
+        $rd sadd set$suffix $val
+        $rd xadd stream$suffix * item 1 value $val
+    }
+    for {set j 0} {$j < $elements * 5} {incr j} {
+        $rd read ; # Discard replies
+    }
+    $rd close
+}
+
+# generate keys with various types and encodings
+proc generate_types {} {
+    r config set list-max-ziplist-size 5
+    r config set hash-max-ziplist-entries 5
+    r config set zset-max-ziplist-entries 5
+    r config set stream-node-max-entries 5
+
+    # create small (ziplist / listpack encoded) objects with 3 items
+    generate_collections "" 3
+
+    # add some metadata to the stream
+    r xgroup create stream mygroup 0
+    set records [r xreadgroup GROUP mygroup Alice COUNT 2 STREAMS stream >]
+    r xack stream mygroup [lindex [lindex [lindex [lindex $records 0] 1] 0] 0]
+
+    # create other non-collection types
+    r incr int
+    r set string str
+
+    # create bigger objects with 10 items (more than a single ziplist / listpack)
+    generate_collections big 10
+
+    # make sure our big stream also has a listpack record that has different
+    # field names than the master recored
+    r xadd streambig * item 1 value 1
+    r xadd streambig * item 1 unique value
+}
+
+proc corrupt_payload {payload} {
+    set len [string length $payload]
+    set count 1 ;# usually corrupt only one byte
+    if {rand() > 0.9} { set count 2 }
+    while { $count > 0 } {
+        set idx [expr {int(rand() * $len)}]
+        set ch [binary format c [expr {int(rand()*255)}]]
+        set payload [string replace $payload $idx $idx $ch]
+        incr count -1
+    }
+    return $payload
+}
+
+# fuzzy tester for corrupt RESTORE payloads
+# valgrind will make sure there were no leaks in the rdb loader error handling code
+foreach sanitize_dump {no yes} {
+    if {$::accurate} {
+        set min_duration [expr {60 * 10}] ;# run at least 10 minutes
+        set min_cycles 1000 ;# run at least 1k cycles (max 16 minutes)
+    } else {
+        set min_duration 10 ; # run at least 10 seconds
+        set min_cycles 10 ; # run at least 10 cycles
+    }
+
+    test "Fuzzer corrupt restore payloads - sanitize_dump: $sanitize_dump" {
+        if {$min_duration * 2 > $::timeout} {
+            fail "insufficient timeout"
+        }
+        # start a server, fill with data and save an RDB file once (avoid re-save)
+        start_server [list overrides [list "save" "" use-exit-on-panic yes crash-memcheck-enabled no loglevel verbose] ] {
+            set stdout [srv 0 stdout]
+            r config set sanitize-dump-payload $sanitize_dump
+            r debug set-skip-checksum-validation 1
+            set start_time [clock seconds]
+            generate_types
+            r save
+            set cycle 0
+            set stat_terminated_in_restore 0
+            set stat_terminated_in_traffic 0
+            set stat_terminated_by_signal 0
+            set stat_successful_restore 0
+            set stat_rejected_restore 0
+            set stat_traffic_commands_sent 0
+            # repeatedly DUMP a random key, corrupt it and try RESTORE into a new key
+            while true {
+                set k [r randomkey]
+                set dump [r dump $k]
+                set dump [corrupt_payload $dump]
+                set printable_dump [string2printable $dump]
+                set restore_failed false
+                set report_and_restart false
+                set sent {}
+                # RESTORE can fail, but hopefully not terminate
+                if { [catch { r restore "_$k" 0 $dump REPLACE } err] } {
+                    set restore_failed true
+                    # skip if return failed with an error response.
+                    if {[string match "ERR*" $err]} {
+                        incr stat_rejected_restore
+                    } else {
+                        set report_and_restart true
+                        incr stat_terminated_in_restore
+                        write_log_line 0 "corrupt payload: $printable_dump"
+                        if {$sanitize_dump == 1} {
+                            puts "Server crashed in RESTORE with payload: $printable_dump"
+                        }
+                    }
+                } else {
+                    r ping ;# an attempt to check if the server didn't terminate (this will throw an error that will terminate the tests)
+                }
+
+                set print_commands false
+                if {!$restore_failed} {
+                    # if RESTORE didn't fail or terminate, run some random traffic on the new key
+                    incr stat_successful_restore
+                    if { [ catch {
+                        set sent [generate_fuzzy_traffic_on_key "_$k" 1] ;# traffic for 1 second
+                        incr stat_traffic_commands_sent [llength $sent]
+                        r del "_$k" ;# in case the server terminated, here's where we'll detect it.
+                    } err ] } {
+                        # if the server terminated update stats and restart it
+                        set report_and_restart true
+                        incr stat_terminated_in_traffic
+                        set by_signal [count_log_message 0 "crashed by signal"]
+                        incr stat_terminated_by_signal $by_signal
+
+                        if {$by_signal != 0 || $sanitize_dump == 1 } {
+                            puts "Server crashed (by signal: $by_signal), with payload: $printable_dump"
+                            set print_commands true
+                        }
+                    }
+                }
+
+                # check valgrind report for invalid reads after each RESTORE
+                # payload so that we have a report that is easier to reproduce
+                set valgrind_errors [find_valgrind_errors [srv 0 stderr] false]
+                if {$valgrind_errors != ""} {
+                    puts "valgrind found an issue for payload: $printable_dump"
+                    set report_and_restart true
+                    set print_commands true
+                }
+
+                if {$report_and_restart} {
+                    if {$print_commands} {
+                        puts "violating commands:"
+                        foreach cmd $sent {
+                            foreach arg $cmd {
+                                puts -nonewline "[string2printable $arg] "
+                            }
+                            puts ""
+                        }
+                    }
+
+                    # restart the server and re-apply debug configuration
+                    write_log_line 0 "corrupt payload: $printable_dump"
+                    restart_server 0 true true
+                    r config set sanitize-dump-payload $sanitize_dump
+                    r debug set-skip-checksum-validation 1
+                }
+
+                incr cycle
+                if { ([clock seconds]-$start_time) >= $min_duration && $cycle >= $min_cycles} {
+                    break
+                }
+            }
+            if {$::verbose} {
+                puts "Done $cycle cycles in [expr {[clock seconds]-$start_time}] seconds."
+                puts "RESTORE: successful: $stat_successful_restore, rejected: $stat_rejected_restore"
+                puts "Total commands sent in traffic: $stat_traffic_commands_sent, crashes during traffic: $stat_terminated_in_traffic ($stat_terminated_by_signal by signal)."
+            }
+        }
+        # if we run sanitization we never expect the server to crash at runtime
+        if { $sanitize_dump == 1} {
+            assert_equal $stat_terminated_in_restore 0
+            assert_equal $stat_terminated_in_traffic 0
+        }
+        # make sure all terminations where due to assertion and not a SIGSEGV
+        assert_equal $stat_terminated_by_signal 0
+    }
+}
+
+
+
+} ;# tags
+
diff --git a/tests/integration/corrupt-dump.tcl b/tests/integration/corrupt-dump.tcl
index 9864036d2..b8828fd94 100644
--- a/tests/integration/corrupt-dump.tcl
+++ b/tests/integration/corrupt-dump.tcl
@@ -1,4 +1,10 @@
 # tests of corrupt ziplist payload with valid CRC
+# * setting crash-memcheck-enabled to no to avoid issues with valgrind
+# * setting use-exit-on-panic to yes so that valgrind can search for leaks
+# * settng debug set-skip-checksum-validation to 1 on some tests for which we
+#   didn't bother to fake a valid checksum
+# * some tests set sanitize-dump-payload to no and some to yet, depending on
+#   what we want to test
 
 tags {"dump" "corruption"} {
 
@@ -18,27 +24,27 @@ test {corrupt payload: #7445 - with sanitize} {
 test {corrupt payload: #7445 - without sanitize - 1} {
     start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
         r config set sanitize-dump-payload no
-        r config set crash-memcheck-enabled no ;# avoid valgrind issues
         r restore key 0 $corrupt_payload_7445
         catch {r lindex key 2}
-        verify_log_message 0 "*ASSERTION FAILED*" 0
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
     }
 }
 
 test {corrupt payload: #7445 - without sanitize - 2} {
     start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
         r config set sanitize-dump-payload no
-        r config set crash-memcheck-enabled no ;# avoid valgrind issues
         r restore key 0 $corrupt_payload_7445
         catch {r lset key 2 "BEEF"}
-        verify_log_message 0 "*ASSERTION FAILED*" 0
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
     }
 }
 
 test {corrupt payload: hash with valid zip list header, invalid entry len} {
     start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
         r restore key 0 "\x0D\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x14\x63\x00\x04\x02\x64\x00\xFF\x09\x00\xD9\x10\x54\x92\x15\xF5\x5F\x52"
-        r config set crash-memcheck-enabled no ;# avoid valgrind issues
         r config set hash-max-ziplist-entries 1
         catch {r hset key b b}
         verify_log_message 0 "*zipEntrySafe*" 0
@@ -47,6 +53,7 @@ test {corrupt payload: hash with valid zip list header, invalid entry len} {
 
 test {corrupt payload: invalid zlbytes header} {
     start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
         catch {
             r restore key 0 "\x0D\x1B\x25\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x02\x63\x00\x04\x02\x64\x00\xFF\x09\x00\xB7\xF7\x6E\x9F\x43\x43\x14\xC6"
         } err
@@ -56,8 +63,8 @@ test {corrupt payload: invalid zlbytes header} {
 
 test {corrupt payload: valid zipped hash header, dup records} {
     start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
         r restore key 0 "\x0D\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x02\x61\x00\x04\x02\x64\x00\xFF\x09\x00\xA1\x98\x36\x78\xCC\x8E\x93\x2E"
-        r config set crash-memcheck-enabled no ;# avoid valgrind issues
         r config set hash-max-ziplist-entries 1
         # cause an assertion when converting to hash table
         catch {r hset key b b}
@@ -67,10 +74,11 @@ test {corrupt payload: valid zipped hash header, dup records} {
 
 test {corrupt payload: quicklist big ziplist prev len} {
     start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
         r restore key 0 "\x0E\x01\x13\x13\x00\x00\x00\x0E\x00\x00\x00\x02\x00\x00\x02\x61\x00\x0E\x02\x62\x00\xFF\x09\x00\x49\x97\x30\xB2\x0D\xA1\xED\xAA"
-        r config set crash-memcheck-enabled no ;# avoid valgrind issues
         catch {r lindex key -2}
-        verify_log_message 0 "*ASSERTION FAILED*" 0
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
     }
 }
 
@@ -87,19 +95,24 @@ test {corrupt payload: quicklist small ziplist prev len} {
 
 test {corrupt payload: quicklist ziplist wrong count} {
     start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
         r restore key 0 "\x0E\x01\x13\x13\x00\x00\x00\x0E\x00\x00\x00\x03\x00\x00\x02\x61\x00\x04\x02\x62\x00\xFF\x09\x00\x4D\xE2\x0A\x2F\x08\x25\xDF\x91"
-        r lpush key a
-        # check that the server didn't crash
-        r ping
+        # we'll be able to push, but iterating on the list will assert
+        r lpush key header
+        r rpush key footer
+        catch { [r lrange key -1 -1] }
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
     }
 }
 
 test {corrupt payload: #3080 - quicklist} {
     start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
         catch {
             r RESTORE key 0 "\x0E\x01\x80\x00\x00\x00\x10\x41\x41\x41\x41\x41\x41\x41\x41\x02\x00\x00\x80\x41\x41\x41\x41\x07\x00\x03\xC7\x1D\xEF\x54\x68\xCC\xF3"
-            r DUMP key
-        }
+            r DUMP key ;# DUMP was used in the original issue, but now even with shallow sanitization restore safely fails, so this is dead code
+        } err
         assert_match "*Bad data format*" $err
         verify_log_message 0 "*Ziplist integrity check failed*" 0
     }
@@ -107,9 +120,11 @@ test {corrupt payload: #3080 - quicklist} {
 
 test {corrupt payload: #3080 - ziplist} {
     start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        # shallow sanitization is enough for restore to safely reject the payload with wrong size
+        r config set sanitize-dump-payload no
         catch {
             r RESTORE key 0 "\x0A\x80\x00\x00\x00\x10\x41\x41\x41\x41\x41\x41\x41\x41\x02\x00\x00\x80\x41\x41\x41\x41\x07\x00\x39\x5B\x49\xE0\xC1\xC6\xDD\x76"
-        }
+        } err
         assert_match "*Bad data format*" $err
         verify_log_message 0 "*Ziplist integrity check failed*" 0
     }
@@ -118,7 +133,7 @@ test {corrupt payload: #3080 - ziplist} {
 test {corrupt payload: load corrupted rdb with no CRC - #3505} {
     set server_path [tmpdir "server.rdb-corruption-test"]
     exec cp tests/assets/corrupt_ziplist.rdb $server_path
-    set srv [start_server [list overrides [list "dir" $server_path "dbfilename" "corrupt_ziplist.rdb" loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no]]]
+    set srv [start_server [list overrides [list "dir" $server_path "dbfilename" "corrupt_ziplist.rdb" loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no sanitize-dump-payload no]]]
 
     # wait for termination
     wait_for_condition 100 50 {
@@ -135,6 +150,7 @@ test {corrupt payload: load corrupted rdb with no CRC - #3505} {
 
 test {corrupt payload: listpack invalid size header} {
     start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
         catch {
             r restore key 0 "\x0F\x01\x10\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x40\x55\x5F\x00\x00\x00\x0F\x00\x01\x01\x00\x01\x02\x01\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x00\x01\x00\x01\x00\x01\x00\x01\x02\x02\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x61\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x88\x62\x00\x00\x00\x00\x00\x00\x00\x09\x08\x01\xFF\x0A\x01\x00\x00\x09\x00\x45\x91\x0A\x87\x2F\xA5\xF9\x2E"
         } err
@@ -145,21 +161,25 @@ test {corrupt payload: listpack invalid size header} {
 
 test {corrupt payload: listpack too long entry len} {
     start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
         r restore key 0 "\x0F\x01\x10\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x40\x55\x55\x00\x00\x00\x0F\x00\x01\x01\x00\x01\x02\x01\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x00\x01\x00\x01\x00\x01\x00\x01\x02\x02\x89\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x61\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x88\x62\x00\x00\x00\x00\x00\x00\x00\x09\x08\x01\xFF\x0A\x01\x00\x00\x09\x00\x40\x63\xC9\x37\x03\xA2\xE5\x68"
         catch {
             r xinfo stream key full
         } err
-        verify_log_message 0 "*ASSERTION FAILED*" 0
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
     }
 }
 
 test {corrupt payload: listpack very long entry len} {
     start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
         r restore key 0 "\x0F\x01\x10\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x40\x55\x55\x00\x00\x00\x0F\x00\x01\x01\x00\x01\x02\x01\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x00\x01\x00\x01\x00\x01\x00\x01\x02\x02\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x61\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x9C\x62\x00\x00\x00\x00\x00\x00\x00\x09\x08\x01\xFF\x0A\x01\x00\x00\x09\x00\x63\x6F\x42\x8E\x7C\xB5\xA2\x9D"
         catch {
             r xinfo stream key full
         } err
-        verify_log_message 0 "*ASSERTION FAILED*" 0
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
     }
 }
 
@@ -174,5 +194,247 @@ test {corrupt payload: listpack too long entry prev len} {
     }
 }
 
+test {corrupt payload: fuzzer findings - NPD in streamIteratorGetID} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        catch {
+            r RESTORE key 0 "\x0F\x01\x10\x00\x00\x01\x73\xBD\x68\x48\x71\x00\x00\x00\x00\x00\x00\x00\x00\x40\x42\x42\x00\x00\x00\x18\x00\x03\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x01\x02\x01\x00\x01\x00\x01\x01\x01\x00\x01\x05\x01\x02\x01\x00\x01\x01\x01\x01\x01\x82\x5F\x31\x03\x05\x01\x02\x01\x00\x01\x02\x01\x01\x01\x02\x01\x48\x01\xFF\x03\x81\x00\x00\x01\x73\xBD\x68\x48\x71\x02\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x73\xBD\x68\x48\x71\x00\x01\x00\x00\x01\x73\xBD\x68\x48\x71\x00\x00\x00\x00\x00\x00\x00\x00\x72\x48\x68\xBD\x73\x01\x00\x00\x01\x01\x05\x41\x6C\x69\x63\x65\x72\x48\x68\xBD\x73\x01\x00\x00\x01\x00\x00\x01\x73\xBD\x68\x48\x71\x00\x00\x00\x00\x00\x00\x00\x00\x09\x00\x80\xCD\xB0\xD5\x1A\xCE\xFF\x10"
+            r XREVRANGE key 725 233
+        }
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
+    }
+}
+
+test {corrupt payload: fuzzer findings - listpack NPD on invalid stream} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        catch {
+            r RESTORE _stream 0 "\x0F\x01\x10\x00\x00\x01\x73\xDC\xB6\x6B\xF1\x00\x00\x00\x00\x00\x00\x00\x00\x40\x42\x42\x00\x00\x00\x18\x00\x03\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x01\x02\x01\x00\x01\x00\x01\x01\x01\x00\x01\x05\x01\x02\x01\x1F\x01\x00\x01\x01\x01\x6D\x5F\x31\x03\x05\x01\x02\x01\x29\x01\x00\x01\x01\x01\x02\x01\x05\x01\xFF\x03\x81\x00\x00\x01\x73\xDC\xB6\x6C\x1A\x00\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x73\xDC\xB6\x6B\xF1\x00\x01\x00\x00\x01\x73\xDC\xB6\x6B\xF1\x00\x00\x00\x00\x00\x00\x00\x00\x4B\x6C\xB6\xDC\x73\x01\x00\x00\x01\x01\x05\x41\x6C\x69\x63\x65\x3D\x6C\xB6\xDC\x73\x01\x00\x00\x01\x00\x00\x01\x73\xDC\xB6\x6B\xF1\x00\x00\x00\x00\x00\x00\x00\x00\x09\x00\xC7\x7D\x1C\xD7\x04\xFF\xE6\x9D"
+            r XREAD STREAMS _stream 519389898758
+        }
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
+    }
+}
+
+test {corrupt payload: fuzzer findings - NPD in quicklistIndex} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        catch {
+            r RESTORE key 0 "\x0E\x01\x13\x13\x00\x00\x00\x10\x00\x00\x00\x03\x12\x00\xF3\x02\x02\x5F\x31\x04\xF1\xFF\x09\x00\xC9\x4B\x31\xFE\x61\xC0\x96\xFE"
+            r LSET key 290 290
+        }
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
+    }
+}
+
+test {corrupt payload: fuzzer findings - invalid read in ziplistFind} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        catch {
+            r RESTORE key 0 "\x0D\x19\x19\x00\x00\x00\x16\x00\x00\x00\x06\x00\x00\xF1\x02\xF1\x02\xF2\x02\x02\x5F\x31\x04\x99\x02\xF3\xFF\x09\x00\xC5\xB8\x10\xC0\x8A\xF9\x16\xDF"
+            r HEXISTS key -688319650333
+        }
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
+    }
+}
+
+
+test {corrupt payload: fuzzer findings - invalid ziplist encoding} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload yes
+        r debug set-skip-checksum-validation 1
+        catch {
+            r RESTORE _listbig 0 "\x0E\x02\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\x02\x5F\x39\x04\xF9\x02\x86\x5F\x37\x04\xF7\x02\x02\x5F\x35\xFF\x19\x19\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\xF5\x02\x02\x5F\x33\x04\xF3\x02\x02\x5F\x31\x04\xF1\xFF\x09\x00\x0C\xFC\x99\x2C\x23\x45\x15\x60"
+        } err
+        assert_match "*Bad data format*" $err
+        verify_log_message 0 "*Ziplist integrity check failed*" 0
+    }
+}
+
+test {corrupt payload: fuzzer findings - hash crash} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload yes
+        r debug set-skip-checksum-validation 1
+        r RESTORE _hash 0 "\x0D\x19\x19\x00\x00\x00\x16\x00\x00\x00\x06\x00\x00\xF1\x02\xF1\x02\xF2\x02\x02\x5F\x31\x04\xF3\x02\xF3\xFF\x09\x00\x38\xB8\x10\xC0\x8A\xF9\x16\xDF"
+        r HSET _hash 394891450 1635910264
+        r HMGET _hash 887312884855
+    }
+}
+
+test {corrupt payload: fuzzer findings - uneven entry count in hash} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        r RESTORE _hashbig 0 "\x0D\x3D\x3D\x00\x00\x00\x38\x00\x00\x00\x14\x00\x00\xF2\x02\x02\x5F\x31\x04\x1C\x02\xF7\x02\xF1\x02\xF1\x02\xF5\x02\xF5\x02\xF4\x02\x02\x5F\x33\x04\xF6\x02\x02\x5F\x35\x04\xF8\x02\x02\x5F\x37\x04\xF9\x02\xF9\x02\xF3\x02\xF3\x02\xFA\x02\x02\x5F\x39\xFF\x09\x00\x73\xB7\x68\xC8\x97\x24\x8E\x88"
+        catch { r HSCAN _hashbig -250 }
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
+    }
+}
+
+test {corrupt payload: fuzzer findings - invalid read in lzf_decompress} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        catch { r RESTORE _setbig 0 "\x02\x03\x02\x5F\x31\xC0\x02\xC3\x00\x09\x00\xE6\xDC\x76\x44\xFF\xEB\x3D\xFE" } err
+        assert_match "*Bad data format*" $err
+    }
+}
+
+test {corrupt payload: fuzzer findings - leak in rdbloading due to dup entry in set} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        catch { r RESTORE _setbig 0 "\x02\x0A\x02\x5F\x39\xC0\x06\x02\x5F\x31\xC0\x00\xC0\x04\x02\x5F\x35\xC0\x02\xC0\x08\x02\x5F\x31\x02\x5F\x33\x09\x00\x7A\x5A\xFB\x90\x3A\xE9\x3C\xBE" } err
+        assert_match "*Bad data format*" $err
+    }
+}
+
+test {corrupt payload: fuzzer findings - empty intset div by zero} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        r RESTORE _setbig 0 "\x02\xC0\xC0\x06\x02\x5F\x39\xC0\x02\x02\x5F\x33\xC0\x00\x02\x5F\x31\xC0\x04\xC0\x08\x02\x5F\x37\x02\x5F\x35\x09\x00\xC5\xD4\x6D\xBA\xAD\x14\xB7\xE7"
+        catch {r SRANDMEMBER _setbig }
+    }
+}
+
+test {corrupt payload: fuzzer findings - valgrind ziplist - crash report prints freed memory} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        r RESTORE _zsetbig 0 "\x0C\x3D\x3D\x00\x00\x00\x3A\x00\x00\x00\x14\x00\x00\xF1\x02\xF1\x02\x02\x5F\x31\x04\xF2\x02\xF3\x02\xF3\x02\x02\x5F\x33\x04\xF4\x02\xEE\x02\xF5\x02\x02\x5F\x35\x04\xF6\x02\xF7\x02\xF7\x02\x02\x5F\x37\x04\xF8\x02\xF9\x02\xF9\x02\x02\x5F\x39\x04\xFA\xFF\x09\x00\xAE\xF9\x77\x2A\x47\x24\x33\xF6"
+        catch { r ZREMRANGEBYSCORE _zsetbig -1050966020 724 }
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
+    }
+}
+
+test {corrupt payload: fuzzer findings - valgrind ziplist prevlen reaches outside the ziplist} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        r RESTORE _listbig 0 "\x0E\x02\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\x02\x5F\x39\x04\xF9\x02\x02\x5F\x37\x04\xF7\x02\x02\x5F\x35\xFF\x19\x19\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\xF5\x02\x02\x5F\x33\x04\xF3\x95\x02\x5F\x31\x04\xF1\xFF\x09\x00\x0C\xFC\x99\x2C\x23\x45\x15\x60"
+        catch { r RPOP _listbig }
+        catch { r RPOP _listbig }
+        catch { r RPUSH _listbig 949682325 }
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
+    }
+}
+
+test {corrupt payload: fuzzer findings - valgrind - bad rdbLoadDoubleValue} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        catch { r RESTORE _list 0 "\x03\x01\x11\x11\x00\x00\x00\x0A\x00\x00\x00\x01\x00\x00\xD0\x07\x1A\xE9\x02\xFF\x09\x00\x1A\x06\x07\x32\x41\x28\x3A\x46" } err
+        assert_match "*Bad data format*" $err
+    }
+}
+
+test {corrupt payload: fuzzer findings - valgrind ziplist prev too big} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        r RESTORE _list 0 "\x0E\x01\x13\x13\x00\x00\x00\x10\x00\x00\x00\x03\x00\x00\xF3\x02\x02\x5F\x31\xC1\xF1\xFF\x09\x00\xC9\x4B\x31\xFE\x61\xC0\x96\xFE"
+        catch { r RPUSHX _list -45 }
+        catch { r LREM _list -748 -840}
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
+    }
+}
+
+test {corrupt payload: fuzzer findings - lzf decompression fails, avoid valgrind invalid read} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        catch {r RESTORE _stream 0 "\x0F\x02\x10\x00\x00\x01\x73\xDD\xAA\x2A\xB9\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x40\x4B\x40\x5C\x18\x5C\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x00\x01\x20\x03\x00\x05\x20\x1C\x40\x07\x05\x01\x01\x82\x5F\x31\x03\x80\x0D\x40\x00\x00\x02\x60\x19\x40\x27\x40\x19\x00\x33\x60\x19\x40\x29\x02\x01\x01\x04\x20\x19\x00\xFF\x10\x00\x00\x01\x73\xDD\xAA\x2A\xBC\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x40\x4D\x40\x5E\x18\x5E\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x06\x01\x01\x82\x5F\x35\x03\x05\x20\x1E\x17\x0B\x03\x01\x01\x06\x01\x40\x0B\x00\x01\x60\x0D\x02\x82\x5F\x37\x60\x19\x80\x00\x00\x08\x60\x19\x80\x27\x02\x82\x5F\x39\x20\x19\x00\xFF\x0A\x81\x00\x00\x01\x73\xDD\xAA\x2A\xBE\x00\x00\x09\x00\x21\x85\x77\x43\x71\x7B\x17\x88"} err
+        assert_match "*Bad data format*" $err
+    }
+}
+
+test {corrupt payload: fuzzer findings - stream bad lp_count} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload yes
+        r debug set-skip-checksum-validation 1
+        catch { r RESTORE _stream 0 "\x0F\x01\x10\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x00\x00\x00\x00\x00\x00\x00\x40\x42\x42\x00\x00\x00\x18\x00\x03\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x01\x02\x01\x00\x01\x00\x01\x01\x01\x00\x01\x56\x01\x02\x01\x22\x01\x00\x01\x01\x01\x82\x5F\x31\x03\x05\x01\x02\x01\x2C\x01\x00\x01\x01\x01\x02\x01\x05\x01\xFF\x03\x81\x00\x00\x01\x73\xDE\xDF\x7D\xC7\x00\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x01\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x00\x00\x00\x00\x00\x00\x00\xF9\x7D\xDF\xDE\x73\x01\x00\x00\x01\x01\x05\x41\x6C\x69\x63\x65\xEB\x7D\xDF\xDE\x73\x01\x00\x00\x01\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x00\x00\x00\x00\x00\x00\x00\x09\x00\xB2\xA8\xA7\x5F\x1B\x61\x72\xD5"} err
+        assert_match "*Bad data format*" $err
+        r ping
+    }
+}
+
+test {corrupt payload: fuzzer findings - stream bad lp_count - unsanitized} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        r RESTORE _stream 0 "\x0F\x01\x10\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x00\x00\x00\x00\x00\x00\x00\x40\x42\x42\x00\x00\x00\x18\x00\x03\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x01\x02\x01\x00\x01\x00\x01\x01\x01\x00\x01\x56\x01\x02\x01\x22\x01\x00\x01\x01\x01\x82\x5F\x31\x03\x05\x01\x02\x01\x2C\x01\x00\x01\x01\x01\x02\x01\x05\x01\xFF\x03\x81\x00\x00\x01\x73\xDE\xDF\x7D\xC7\x00\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x01\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x00\x00\x00\x00\x00\x00\x00\xF9\x7D\xDF\xDE\x73\x01\x00\x00\x01\x01\x05\x41\x6C\x69\x63\x65\xEB\x7D\xDF\xDE\x73\x01\x00\x00\x01\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x00\x00\x00\x00\x00\x00\x00\x09\x00\xB2\xA8\xA7\x5F\x1B\x61\x72\xD5"
+        catch { r XREVRANGE _stream 638932639 738}
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
+    }
+}
+
+test {corrupt payload: fuzzer findings - stream integrity check issue} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload yes
+        r debug set-skip-checksum-validation 1
+        catch { r RESTORE _stream 0 "\x0F\x02\x10\x00\x00\x01\x75\x2D\xA2\x90\x67\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x40\x4F\x40\x5C\x18\x5C\x00\x00\x00\x24\x00\x05\x01\x00\x01\x4A\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x00\x01\x20\x03\x00\x05\x20\x1C\x40\x09\x05\x01\x01\x82\x5F\x31\x03\x80\x0D\x00\x02\x20\x0D\x00\x02\xA0\x19\x00\x03\x20\x0B\x02\x82\x5F\x33\xA0\x19\x00\x04\x20\x0D\x00\x04\x20\x19\x00\xFF\x10\x00\x00\x01\x75\x2D\xA2\x90\x67\x00\x00\x00\x00\x00\x00\x00\x05\xC3\x40\x56\x40\x60\x18\x60\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x06\x01\x01\x82\x5F\x35\x03\x05\x20\x1E\x40\x0B\x03\x01\x01\x06\x01\x80\x0B\x00\x02\x20\x0B\x02\x82\x5F\x37\x60\x19\x03\x01\x01\xDF\xFB\x20\x05\x00\x08\x60\x1A\x20\x0C\x00\xFC\x20\x05\x02\x82\x5F\x39\x20\x1B\x00\xFF\x0A\x81\x00\x00\x01\x75\x2D\xA2\x90\x68\x01\x00\x09\x00\x1D\x6F\xC0\x69\x8A\xDE\xF7\x92" } err
+        assert_match "*Bad data format*" $err
+    }
+}
+
+test {corrupt payload: fuzzer findings - infinite loop} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        r RESTORE _stream 0 "\x0F\x01\x10\x00\x00\x01\x75\x3A\xA6\xD0\x93\x00\x00\x00\x00\x00\x00\x00\x00\x40\x42\x42\x00\x00\x00\x18\x00\x03\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x01\x02\x01\x00\x01\x00\x01\x01\x01\x00\x01\x05\x01\x02\x01\x00\x01\x01\x01\x01\x01\x82\x5F\x31\x03\xFD\x01\x02\x01\x00\x01\x02\x01\x01\x01\x02\x01\x05\x01\xFF\x03\x81\x00\x00\x01\x75\x3A\xA6\xD0\x93\x02\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x75\x3A\xA6\xD0\x93\x00\x01\x00\x00\x01\x75\x3A\xA6\xD0\x93\x00\x00\x00\x00\x00\x00\x00\x00\x94\xD0\xA6\x3A\x75\x01\x00\x00\x01\x01\x05\x41\x6C\x69\x63\x65\x94\xD0\xA6\x3A\x75\x01\x00\x00\x01\x00\x00\x01\x75\x3A\xA6\xD0\x93\x00\x00\x00\x00\x00\x00\x00\x00\x09\x00\xC4\x09\xAD\x69\x7E\xEE\xA6\x2F"
+        catch { r XREVRANGE _stream 288270516 971031845 }
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
+    }
+}
+
+test {corrupt payload: fuzzer findings - invalid tail offset after removal} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        r RESTORE _zset 0 "\x0C\x19\x19\x00\x00\x00\x02\x00\x00\x00\x06\x00\x00\xF1\x02\xF1\x02\x02\x5F\x31\x04\xF2\x02\xF3\x02\xF3\xFF\x09\x00\x4D\x72\x7B\x97\xCD\x9A\x70\xC1"
+        catch {r ZPOPMIN _zset}
+        catch {r ZPOPMAX _zset}
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
+    }
+}
+
+test {corrupt payload: fuzzer findings - negative reply length} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        r RESTORE _stream 0 "\x0F\x01\x10\x00\x00\x01\x75\xCF\xA1\x16\xA7\x00\x00\x00\x00\x00\x00\x00\x00\x40\x42\x42\x00\x00\x00\x18\x00\x03\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x01\x02\x01\x00\x01\x00\x01\x01\x01\x00\x01\x05\x01\x02\x01\x00\x01\x01\x01\x01\x01\x14\x5F\x31\x03\x05\x01\x02\x01\x00\x01\x02\x01\x01\x01\x02\x01\x05\x01\xFF\x03\x81\x00\x00\x01\x75\xCF\xA1\x16\xA7\x02\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x75\xCF\xA1\x16\xA7\x01\x01\x00\x00\x01\x75\xCF\xA1\x16\xA7\x00\x00\x00\x00\x00\x00\x00\x01\xA7\x16\xA1\xCF\x75\x01\x00\x00\x01\x01\x05\x41\x6C\x69\x63\x65\xA7\x16\xA1\xCF\x75\x01\x00\x00\x01\x00\x00\x01\x75\xCF\xA1\x16\xA7\x00\x00\x00\x00\x00\x00\x00\x01\x09\x00\x1B\x42\x52\xB8\xDD\x5C\xE5\x4E"
+        catch {r XADD _stream * -956 -2601503852}
+        catch {r XINFO STREAM _stream FULL}
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
+    }
+}
+
+test {corrupt payload: fuzzer findings - valgrind negative malloc} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload yes
+        r debug set-skip-checksum-validation 1
+        catch {r RESTORE _key 0 "\x0E\x01\x81\xD6\xD6\x00\x00\x00\x0A\x00\x00\x00\x01\x00\x00\x40\xC8\x6F\x2F\x36\xE2\xDF\xE3\x2E\x26\x64\x8B\x87\xD1\x7A\xBD\xFF\xEF\xEF\x63\x65\xF6\xF8\x8C\x4E\xEC\x96\x89\x56\x88\xF8\x3D\x96\x5A\x32\xBD\xD1\x36\xD8\x02\xE6\x66\x37\xCB\x34\x34\xC4\x52\xA7\x2A\xD5\x6F\x2F\x7E\xEE\xA2\x94\xD9\xEB\xA9\x09\x38\x3B\xE1\xA9\x60\xB6\x4E\x09\x44\x1F\x70\x24\xAA\x47\xA8\x6E\x30\xE1\x13\x49\x4E\xA1\x92\xC4\x6C\xF0\x35\x83\xD9\x4F\xD9\x9C\x0A\x0D\x7A\xE7\xB1\x61\xF5\xC1\x2D\xDC\xC3\x0E\x87\xA6\x80\x15\x18\xBA\x7F\x72\xDD\x14\x75\x46\x44\x0B\xCA\x9C\x8F\x1C\x3C\xD7\xDA\x06\x62\x18\x7E\x15\x17\x24\xAB\x45\x21\x27\xC2\xBC\xBB\x86\x6E\xD8\xBD\x8E\x50\xE0\xE0\x88\xA4\x9B\x9D\x15\x2A\x98\xFF\x5E\x78\x6C\x81\xFC\xA8\xC9\xC8\xE6\x61\xC8\xD1\x4A\x7F\x81\xD6\xA6\x1A\xAD\x4C\xC1\xA2\x1C\x90\x68\x15\x2A\x8A\x36\xC0\x58\xC3\xCC\xA6\x54\x19\x12\x0F\xEB\x46\xFF\x6E\xE3\xA7\x92\xF8\xFF\x09\x00\xD0\x71\xF7\x9F\xF7\x6A\xD6\x2E"} err
+        assert_match "*Bad data format*" $err
+        r ping
+    }
+}
+
 } ;# tags
 
diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl
index 1b996ffd4..8f8db5a92 100644
--- a/tests/integration/psync2.tcl
+++ b/tests/integration/psync2.tcl
@@ -280,7 +280,7 @@ start_server {} {
         set sync_partial_err [status $R($master_id) sync_partial_err]
         catch {
             $R($slave_id) config rewrite
-            restart_server [expr {0-$slave_id}] true
+            restart_server [expr {0-$slave_id}] true false
             set R($slave_id) [srv [expr {0-$slave_id}] client]
         }
         # note: just waiting for connected_slaves==4 has a race condition since
@@ -329,7 +329,7 @@ start_server {} {
 
         catch {
             $R($slave_id) config rewrite
-            restart_server [expr {0-$slave_id}] true
+            restart_server [expr {0-$slave_id}] true false
             set R($slave_id) [srv [expr {0-$slave_id}] client]
         }
 
diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl
index d29f9d88e..aadfe281f 100644
--- a/tests/integration/rdb.tcl
+++ b/tests/integration/rdb.tcl
@@ -164,7 +164,7 @@ test {client freed during loading} {
         # 100mb of rdb, 100k keys will load in more than 1 second
         r debug populate 100000 key 1000
 
-        restart_server 0 false
+        restart_server 0 false false
 
         # make sure it's still loading
         assert_equal [s loading] 1
diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index ae93ad007..e5b167a35 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -13,7 +13,7 @@ proc start_server_error {config_file error} {
 }
 
 proc check_valgrind_errors stderr {
-    set res [find_valgrind_errors $stderr]
+    set res [find_valgrind_errors $stderr true]
     if {$res != ""} {
         send_data_packet $::test_server_fd err "Valgrind error: $res\n"
     }
@@ -437,7 +437,7 @@ proc start_server {options {code undefined}} {
 
         while 1 {
             # check that the server actually started and is ready for connections
-            if {[exec grep -i "Ready to accept" | wc -l < $stdout] > 0} {
+            if {[count_message_lines $stdout "Ready to accept"] > 0} {
                 break
             }
             after 10
@@ -511,13 +511,19 @@ proc start_server {options {code undefined}} {
     }
 }
 
-proc restart_server {level wait_ready} {
+proc restart_server {level wait_ready rotate_logs} {
     set srv [lindex $::servers end+$level]
     kill_server $srv
 
+    set pid [dict get $srv "pid"]
     set stdout [dict get $srv "stdout"]
     set stderr [dict get $srv "stderr"]
-    set config_file [dict get $srv "config_file"]
+    if {$rotate_logs} {
+        set ts [clock format [clock seconds] -format %y%m%d%H%M%S]
+        file rename $stdout $stdout.$ts.$pid
+        file rename $stderr $stderr.$ts.$pid
+    }
+    set prev_ready_count [count_message_lines $stdout "Ready to accept"]
 
     # if we're inside a test, write the test name to the server log file
     if {[info exists ::cur_test]} {
@@ -526,7 +532,7 @@ proc restart_server {level wait_ready} {
         close $fd
     }
 
-    set prev_ready_count [exec grep -i "Ready to accept" | wc -l < $stdout]
+    set config_file [dict get $srv "config_file"]
 
     set pid [spawn_server $config_file $stdout $stderr]
 
@@ -541,7 +547,7 @@ proc restart_server {level wait_ready} {
     if {$wait_ready} {
         while 1 {
             # check that the server actually started and is ready for connections
-            if {[exec grep -i "Ready to accept" | wc -l < $stdout] > $prev_ready_count + 1} {
+            if {[count_message_lines $stdout "Ready to accept"] > $prev_ready_count} {
                 break
             }
             after 10
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index aadec7cd1..040a7f7d9 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -454,22 +454,31 @@ proc colorstr {color str} {
     }
 }
 
-proc find_valgrind_errors {stderr} {
+proc find_valgrind_errors {stderr on_termination} {
     set fd [open $stderr]
     set buf [read $fd]
     close $fd
 
     # Look for stack trace (" at 0x") and other errors (Invalid, Mismatched, etc).
     # Look for "Warnings", but not the "set address range perms". These don't indicate any real concern.
-    # Look for the absense of a leak free summary (happens when redis isn't terminated properly).
+    # corrupt-dump unit, not sure why but it seems they don't indicate any real concern.
     if {[regexp -- { at 0x} $buf] ||
         [regexp -- {^(?=.*Warning)(?:(?!set address range perms).)*$} $buf] ||
         [regexp -- {Invalid} $buf] ||
         [regexp -- {Mismatched} $buf] ||
         [regexp -- {uninitialized} $buf] ||
         [regexp -- {has a fishy} $buf] ||
-        [regexp -- {overlap} $buf] ||
-        (![regexp -- {definitely lost: 0 bytes} $buf] &&
+        [regexp -- {overlap} $buf]} {
+        return $buf
+    }
+
+    # If the process didn't terminate yet, we can't look for the summary report
+    if {!$on_termination} {
+        return ""
+    }
+
+    # Look for the absense of a leak free summary (happens when redis isn't terminated properly).
+    if {(![regexp -- {definitely lost: 0 bytes} $buf] &&
          ![regexp -- {no leaks are possible} $buf])} {
         return $buf
     }
@@ -547,3 +556,116 @@ proc cmdrstat {cmd r} {
         set _ $value
     }
 }
+
+proc generate_fuzzy_traffic_on_key {key duration} {
+    # Commands per type, blocking commands removed
+    # TODO: extract these from help.h or elsewhere, and improve to include other types
+    set string_commands {APPEND BITCOUNT BITFIELD BITOP BITPOS DECR DECRBY GET GETBIT GETRANGE GETSET INCR INCRBY INCRBYFLOAT MGET MSET MSETNX PSETEX SET SETBIT SETEX SETNX SETRANGE STRALGO STRLEN}
+    set hash_commands {HDEL HEXISTS HGET HGETALL HINCRBY HINCRBYFLOAT HKEYS HLEN HMGET HMSET HSCAN HSET HSETNX HSTRLEN HVALS}
+    set zset_commands {ZADD ZCARD ZCOUNT ZINCRBY ZINTERSTORE ZLEXCOUNT ZPOPMAX ZPOPMIN ZRANGE ZRANGEBYLEX ZRANGEBYSCORE ZRANK ZREM ZREMRANGEBYLEX ZREMRANGEBYRANK ZREMRANGEBYSCORE ZREVRANGE ZREVRANGEBYLEX ZREVRANGEBYSCORE ZREVRANK ZSCAN ZSCORE ZUNIONSTORE}
+    set list_commands {LINDEX LINSERT LLEN LPOP LPOS LPUSH LPUSHX LRANGE LREM LSET LTRIM RPOP RPOPLPUSH RPUSH RPUSHX}
+    set set_commands {SADD SCARD SDIFF SDIFFSTORE SINTER SINTERSTORE SISMEMBER SMEMBERS SMOVE SPOP SRANDMEMBER SREM SSCAN SUNION SUNIONSTORE}
+    set stream_commands {XACK XADD XCLAIM XDEL XGROUP XINFO XLEN XPENDING XRANGE XREAD XREADGROUP XREVRANGE XTRIM}
+    set commands [dict create string $string_commands hash $hash_commands zset $zset_commands list $list_commands set $set_commands stream $stream_commands]
+
+    set type [r type $key]
+    set cmds [dict get $commands $type]
+    set start_time [clock seconds]
+    set sent {}
+    set succeeded 0
+    while {([clock seconds]-$start_time) < $duration} {
+        # find a random command for our key type
+        set cmd_idx [expr {int(rand()*[llength $cmds])}]
+        set cmd [lindex $cmds $cmd_idx]
+        # get the command details from redis
+        if { [ catch {
+            set cmd_info [lindex [r command info $cmd] 0]
+        } err ] } {
+            # if we failed, it means redis crashed after the previous command
+            return $sent
+        }
+        # try to build a valid command argument
+        set arity [lindex $cmd_info 1]
+        set arity [expr $arity < 0 ? - $arity: $arity]
+        set firstkey [lindex $cmd_info 3]
+        set i 1
+        if {$cmd == "XINFO"} {
+            lappend cmd "STREAM"
+            lappend cmd $key
+            lappend cmd "FULL"
+            incr i 3
+        }
+        if {$cmd == "XREAD"} {
+            lappend cmd "STREAMS"
+            lappend cmd $key
+            randpath {
+                lappend cmd \$
+            } {
+                lappend cmd [randomValue]
+            }
+            incr i 3
+        }
+        if {$cmd == "XADD"} {
+            lappend cmd $key
+            randpath {
+                lappend cmd "*"
+            } {
+                lappend cmd [randomValue]
+            }
+            lappend cmd [randomValue]
+            lappend cmd [randomValue]
+            incr i 4
+        }
+        for {} {$i < $arity} {incr i} {
+            if {$i == $firstkey} {
+                lappend cmd $key
+            } else {
+                lappend cmd [randomValue]
+            }
+        }
+        # execute the command, we expect commands to fail on syntax errors
+        lappend sent $cmd
+        if { ! [ catch {
+            r {*}$cmd
+        } err ] } {
+            incr succeeded
+        }
+    }
+
+    # print stats so that we know if we managed to generate commands that actually made senes
+    #if {$::verbose} {
+    #    set count [llength $sent]
+    #    puts "Fuzzy traffic sent: $count, succeeded: $succeeded"
+    #}
+
+    # return the list of commands we sent
+    return $sent
+}
+
+# write line to server log file
+proc write_log_line {srv_idx msg} {
+    set logfile [srv $srv_idx stdout]
+    set fd [open $logfile "a+"]
+    puts $fd "### $msg"
+    close $fd
+}
+
+proc string2printable s {
+    set res {}
+    set has_special_chars false
+    foreach i [split $s {}] {
+        scan $i %c int
+        # non printable characters, including space and excluding: " \ $ { }
+        if {$int < 32 || $int > 122 || $int == 34 || $int == 36 || $int == 92} {
+            set has_special_chars true
+        }
+        # TCL8.5 has issues mixing \x notation and normal chars in the same
+        # source code string, so we'll convert the entire string.
+        append res \\x[format %02X $int]
+    }
+    if {!$has_special_chars} {
+        return $s
+    }
+    set res "\"$res\""
+    return $res
+}
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 998e82bd9..a1fae8d75 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -45,6 +45,7 @@ set ::all_tests {
     integration/aof
     integration/rdb
     integration/corrupt-dump
+    integration/corrupt-dump-fuzzer
     integration/convert-zipmap-hash-on-load
     integration/logging
     integration/psync2
diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl
index 2d50b417b..86da17657 100644
--- a/tests/unit/introspection.tcl
+++ b/tests/unit/introspection.tcl
@@ -160,7 +160,7 @@ start_server {tags {"introspection"}} {
         # Rewrite entire configuration, restart and confirm the
         # server is able to parse it and start.
         assert_equal [r debug config-rewrite-force-all] "OK"
-        restart_server 0 0
+        restart_server 0 false false
         assert_equal [r ping] "PONG"
 
         # Verify no changes were introduced

From 5b44631397787a65327fcab77f7df37862286ed9 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 22 Nov 2020 21:21:42 +0200
Subject: [PATCH 0845/1098] testsuite: fix fd leak, prevent port clashing when
 using --baseport

when using --baseport to run two tests suite in parallel (different
folders), we need to also make sure the port used by the testsuite to
communicate with it's workers is unique. otherwise the attept to find a
free port connects to the other test suite and messes it.

maybe one day we need to attempt to bind, instead of connect when tring
to find a free port.
---
 tests/support/redis.tcl | 2 ++
 tests/test_helper.tcl   | 6 +++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/support/redis.tcl b/tests/support/redis.tcl
index 847f68e63..26b4510ac 100644
--- a/tests/support/redis.tcl
+++ b/tests/support/redis.tcl
@@ -109,6 +109,7 @@ proc ::redis::__dispatch__raw__ {id method argv} {
         }
         ::redis::redis_write $fd $cmd
         if {[catch {flush $fd}]} {
+            catch {close $fd}
             set ::redis::fd($id) {}
             return -code error "I/O error reading reply"
         }
@@ -251,6 +252,7 @@ proc ::redis::redis_read_reply {id fd} {
         % {redis_read_map $id $fd}
         default {
             if {$type eq {}} {
+                catch {close $fd}
                 set ::redis::fd($id) {}
                 return -code error "I/O error reading reply"
             }
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index a1fae8d75..29ebdd7bb 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -180,6 +180,10 @@ proc reconnect {args} {
     set port [dict get $srv "port"]
     set config [dict get $srv "config"]
     set client [redis $host $port 0 $::tls]
+    if {[dict exists $srv "client"]} {
+        set old [dict get $srv "client"]
+        $old close
+    }
     dict set srv "client" $client
 
     # select the right db when we don't have to authenticate
@@ -257,7 +261,7 @@ proc test_server_main {} {
     set tclsh [info nameofexecutable]
     # Open a listening socket, trying different ports in order to find a
     # non busy one.
-    set clientport [find_available_port 11111 32]
+    set clientport [find_available_port [expr {$::baseport - 32}] 32]
     if {!$::quiet} {
         puts "Starting test server at port $clientport"
     }

From 3716950cfc389c0f7ed13fac5bd205173c2d8189 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 2 Nov 2020 09:35:37 +0200
Subject: [PATCH 0846/1098] Sanitize dump payload: validate no duplicate
 records in hash/zset/intset

If RESTORE passes successfully with full sanitization, we can't affort
to crash later on assertion due to duplicate records in a hash when
converting it form ziplist to dict.
This means that when doing full sanitization, we must make sure there
are no duplicate records in any of the collections.
---
 src/intset.c                       |  23 +++-
 src/intset.h                       |   2 +-
 src/rdb.c                          | 181 ++++++++++++++++++++---------
 src/server.h                       |   2 +
 src/t_hash.c                       |  49 ++++++++
 src/t_zset.c                       |  49 ++++++++
 src/ziplist.c                      |   7 +-
 src/ziplist.h                      |   4 +-
 tests/integration/corrupt-dump.tcl |  54 ++++++++-
 9 files changed, 308 insertions(+), 63 deletions(-)

diff --git a/src/intset.c b/src/intset.c
index 964a41187..54afae875 100644
--- a/src/intset.c
+++ b/src/intset.c
@@ -284,8 +284,11 @@ size_t intsetBlobLen(intset *is) {
     return sizeof(intset)+intrev32ifbe(is->length)*intrev32ifbe(is->encoding);
 }
 
-int intsetValidateIntegrity(const unsigned char *p, size_t size) {
-    const intset *is = (const intset *)p;
+/* Validate the integrity of the data stracture.
+ * when `deep` is 0, only the integrity of the header is validated.
+ * when `deep` is 1, we make sure there are no duplicate or out of order records. */
+int intsetValidateIntegrity(const unsigned char *p, size_t size, int deep) {
+    intset *is = (intset *)p;
     /* check that we can actually read the header. */
     if (size < sizeof(*is))
         return 0;
@@ -308,6 +311,22 @@ int intsetValidateIntegrity(const unsigned char *p, size_t size) {
     if (sizeof(*is) + count*record_size != size)
         return 0;
 
+    /* check that the set is not empty. */
+    if (count==0)
+        return 0;
+
+    if (!deep)
+        return 1;
+
+    /* check that there are no dup or out of order records. */
+    int64_t prev = _intsetGet(is,0);
+    for (uint32_t i=1; i<count; i++) {
+        int64_t cur = _intsetGet(is,i);
+        if (cur <= prev)
+            return 0;
+        prev = cur;
+    }
+
     return 1;
 }
 
diff --git a/src/intset.h b/src/intset.h
index 4fba339d5..08260fc94 100644
--- a/src/intset.h
+++ b/src/intset.h
@@ -46,7 +46,7 @@ int64_t intsetRandom(intset *is);
 uint8_t intsetGet(intset *is, uint32_t pos, int64_t *value);
 uint32_t intsetLen(const intset *is);
 size_t intsetBlobLen(intset *is);
-int intsetValidateIntegrity(const unsigned char *is, size_t size);
+int intsetValidateIntegrity(const unsigned char *is, size_t size, int deep);
 
 #ifdef REDIS_TEST
 int intsetTest(int argc, char *argv[]);
diff --git a/src/rdb.c b/src/rdb.c
index e88cabb5d..e8501ee50 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -44,7 +44,7 @@
 #include <sys/param.h>
 
 /* This macro is called when the internal RDB structure is corrupt */
-#define rdbExitReportCorruptRDB(...) rdbReportError(1, __LINE__,__VA_ARGS__)
+#define rdbReportCorruptRDB(...) rdbReportError(1, __LINE__,__VA_ARGS__)
 /* This macro is called when RDB read failed (possibly a short read) */
 #define rdbReportReadError(...) rdbReportError(0, __LINE__,__VA_ARGS__)
 
@@ -228,7 +228,7 @@ int rdbLoadLenByRef(rio *rdb, int *isencoded, uint64_t *lenptr) {
         if (rioRead(rdb,&len,8) == 0) return -1;
         *lenptr = ntohu64(len);
     } else {
-        rdbExitReportCorruptRDB(
+        rdbReportCorruptRDB(
             "Unknown length encoding %d in rdbLoadLen()",type);
         return -1; /* Never reached. */
     }
@@ -296,7 +296,7 @@ void *rdbLoadIntegerObject(rio *rdb, int enctype, int flags, size_t *lenptr) {
         v = enc[0]|(enc[1]<<8)|(enc[2]<<16)|(enc[3]<<24);
         val = (int32_t)v;
     } else {
-        rdbExitReportCorruptRDB("Unknown RDB integer encoding type %d",enctype);
+        rdbReportCorruptRDB("Unknown RDB integer encoding type %d",enctype);
         return NULL; /* Never reached. */
     }
     if (plain || sds) {
@@ -400,7 +400,7 @@ void *rdbLoadLzfStringObject(rio *rdb, int flags, size_t *lenptr) {
     /* Load the compressed representation and uncompress it to target. */
     if (rioRead(rdb,c,clen) == 0) goto err;
     if (lzf_decompress(c,clen,val,len) != len) {
-        rdbExitReportCorruptRDB("Invalid LZF compressed string");
+        rdbReportCorruptRDB("Invalid LZF compressed string");
         goto err;
     }
     zfree(c);
@@ -516,7 +516,7 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags, size_t *lenptr) {
         case RDB_ENC_LZF:
             return rdbLoadLzfStringObject(rdb,flags,lenptr);
         default:
-            rdbExitReportCorruptRDB("Unknown RDB string encoding type %llu",len);
+            rdbReportCorruptRDB("Unknown RDB string encoding type %llu",len);
             return NULL;
         }
     }
@@ -1467,26 +1467,26 @@ robj *rdbLoadCheckModuleValue(rio *rdb, char *modulename) {
         {
             uint64_t len;
             if (rdbLoadLenByRef(rdb,NULL,&len) == -1) {
-                rdbExitReportCorruptRDB(
+                rdbReportCorruptRDB(
                     "Error reading integer from module %s value", modulename);
             }
         } else if (opcode == RDB_MODULE_OPCODE_STRING) {
             robj *o = rdbGenericLoadStringObject(rdb,RDB_LOAD_NONE,NULL);
             if (o == NULL) {
-                rdbExitReportCorruptRDB(
+                rdbReportCorruptRDB(
                     "Error reading string from module %s value", modulename);
             }
             decrRefCount(o);
         } else if (opcode == RDB_MODULE_OPCODE_FLOAT) {
             float val;
             if (rdbLoadBinaryFloatValue(rdb,&val) == -1) {
-                rdbExitReportCorruptRDB(
+                rdbReportCorruptRDB(
                     "Error reading float from module %s value", modulename);
             }
         } else if (opcode == RDB_MODULE_OPCODE_DOUBLE) {
             double val;
             if (rdbLoadBinaryDoubleValue(rdb,&val) == -1) {
-                rdbExitReportCorruptRDB(
+                rdbReportCorruptRDB(
                     "Error reading double from module %s value", modulename);
             }
         }
@@ -1564,7 +1564,14 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             if (o->encoding == OBJ_ENCODING_INTSET) {
                 /* Fetch integer value from element. */
                 if (isSdsRepresentableAsLongLong(sdsele,&llval) == C_OK) {
-                    o->ptr = intsetAdd(o->ptr,llval,NULL);
+                    uint8_t success;
+                    o->ptr = intsetAdd(o->ptr,llval,&success);
+                    if (!success) {
+                        rdbReportCorruptRDB("Duplicate set members detected");
+                        decrRefCount(o);
+                        sdsfree(sdsele);
+                        return NULL;
+                    }
                 } else {
                     setTypeConvert(o,OBJ_ENCODING_HT);
                     dictExpand(o->ptr,len);
@@ -1575,7 +1582,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
              * to a regular hash table encoded set. */
             if (o->encoding == OBJ_ENCODING_HT) {
                 if (dictAdd((dict*)o->ptr,sdsele,NULL) != DICT_OK) {
-                    rdbExitReportCorruptRDB("Duplicate set members detected");
+                    rdbReportCorruptRDB("Duplicate set members detected");
                     decrRefCount(o);
                     sdsfree(sdsele);
                     return NULL;
@@ -1626,7 +1633,12 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             if (sdslen(sdsele) > maxelelen) maxelelen = sdslen(sdsele);
 
             znode = zslInsert(zs->zsl,score,sdsele);
-            dictAdd(zs->dict,sdsele,&znode->score);
+            if (dictAdd(zs->dict,sdsele,&znode->score) != DICT_OK) {
+                rdbReportCorruptRDB("Duplicate zset fields detected");
+                decrRefCount(o);
+                sdsfree(sdsele);
+                return NULL;
+            }
         }
 
         /* Convert *after* loading, since sorted sets are not stored ordered. */
@@ -1637,15 +1649,24 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
         uint64_t len;
         int ret;
         sds field, value;
+        dict *dupSearchDict = NULL;
 
         len = rdbLoadLen(rdb, NULL);
         if (len == RDB_LENERR) return NULL;
 
         o = createHashObject();
 
-        /* Too many entries? Use a hash table. */
+        /* Too many entries? Use a hash table right from the start. */
         if (len > server.hash_max_ziplist_entries)
             hashTypeConvert(o, OBJ_ENCODING_HT);
+        else if (deep_integrity_validation) {
+            /* In this mode, we need to guarantee that the server won't crash
+             * later when the ziplist is converted to a dict.
+             * Create a set (dict with no values) to for a dup search.
+             * We can dismiss it as soon as we convert the ziplist to a hash. */
+            dupSearchDict = dictCreate(&hashDictType, NULL);
+        }
+
 
         /* Load every field and value into the ziplist */
         while (o->encoding == OBJ_ENCODING_ZIPLIST && len > 0) {
@@ -1653,14 +1674,29 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             /* Load raw strings */
             if ((field = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) {
                 decrRefCount(o);
+                if (dupSearchDict) dictRelease(dupSearchDict);
                 return NULL;
             }
             if ((value = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) {
                 sdsfree(field);
                 decrRefCount(o);
+                if (dupSearchDict) dictRelease(dupSearchDict);
                 return NULL;
             }
 
+            if (dupSearchDict) {
+                sds field_dup = sdsdup(field);
+                if (dictAdd(dupSearchDict, field_dup, NULL) != DICT_OK) {
+                    rdbReportCorruptRDB("Hash with dup elements");
+                    dictRelease(dupSearchDict);
+                    decrRefCount(o);
+                    sdsfree(field_dup);
+                    sdsfree(field);
+                    sdsfree(value);
+                    return NULL;
+                }
+            }
+
             /* Add pair to ziplist */
             o->ptr = ziplistPush(o->ptr, (unsigned char*)field,
                     sdslen(field), ZIPLIST_TAIL);
@@ -1680,6 +1716,13 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             sdsfree(value);
         }
 
+        if (dupSearchDict) {
+            /* We no longer need this, from now on the entries are added
+             * to a dict so the check is performed implicitly. */
+            dictRelease(dupSearchDict);
+            dupSearchDict = NULL;
+        }
+
         if (o->encoding == OBJ_ENCODING_HT && len > DICT_HT_INITIAL_SIZE)
             dictExpand(o->ptr,len);
 
@@ -1700,7 +1743,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             /* Add pair to hash table */
             ret = dictAdd((dict*)o->ptr, field, value);
             if (ret == DICT_ERR) {
-                rdbExitReportCorruptRDB("Duplicate hash fields detected");
+                rdbReportCorruptRDB("Duplicate hash fields detected");
                 sdsfree(value);
                 sdsfree(field);
                 decrRefCount(o);
@@ -1725,8 +1768,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                 return NULL;
             }
             if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
-            if (!ziplistValidateIntegrity(zl, encoded_len, deep_integrity_validation)) {
-                rdbExitReportCorruptRDB("Ziplist integrity check failed.");
+            if (!ziplistValidateIntegrity(zl, encoded_len, deep_integrity_validation, NULL, NULL)) {
+                rdbReportCorruptRDB("Ziplist integrity check failed.");
                 decrRefCount(o);
                 zfree(zl);
                 return NULL;
@@ -1743,28 +1786,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
         unsigned char *encoded =
             rdbGenericLoadStringObject(rdb,RDB_LOAD_PLAIN,&encoded_len);
         if (encoded == NULL) return NULL;
-        if (rdbtype == RDB_TYPE_HASH_ZIPMAP) {
-            /* Since we don't keep zipmaps anymore, the rdb loading for these
-             * is O(n) anyway, use `deep` validation. */
-            if (!zipmapValidateIntegrity(encoded, encoded_len, 1)) {
-                rdbExitReportCorruptRDB("Zipmap integrity check failed.");
-                zfree(encoded);
-                return NULL;
-            }
-        } else if (rdbtype == RDB_TYPE_SET_INTSET) {
-            if (!intsetValidateIntegrity(encoded, encoded_len)) {
-                rdbExitReportCorruptRDB("Intset integrity check failed.");
-                zfree(encoded);
-                return NULL;
-            }
-        } else { /* ziplist */
-            if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
-            if (!ziplistValidateIntegrity(encoded, encoded_len, deep_integrity_validation)) {
-                rdbExitReportCorruptRDB("Ziplist integrity check failed.");
-                zfree(encoded);
-                return NULL;
-            }
-        }
+
         o = createObject(OBJ_STRING,encoded); /* Obj type fixed below. */
 
         /* Fix the object encoding, and make sure to convert the encoded
@@ -1775,6 +1797,15 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
          * converted. */
         switch(rdbtype) {
             case RDB_TYPE_HASH_ZIPMAP:
+                /* Since we don't keep zipmaps anymore, the rdb loading for these
+                 * is O(n) anyway, use `deep` validation. */
+                if (!zipmapValidateIntegrity(encoded, encoded_len, 1)) {
+                    rdbReportCorruptRDB("Zipmap integrity check failed.");
+                    zfree(encoded);
+                    o->ptr = NULL;
+                    decrRefCount(o);
+                    return NULL;
+                }
                 /* Convert to ziplist encoded hash. This must be deprecated
                  * when loading dumps created by Redis 2.4 gets deprecated. */
                 {
@@ -1783,14 +1814,28 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                     unsigned char *fstr, *vstr;
                     unsigned int flen, vlen;
                     unsigned int maxlen = 0;
+                    dict *dupSearchDict = dictCreate(&hashDictType, NULL);
 
                     while ((zi = zipmapNext(zi, &fstr, &flen, &vstr, &vlen)) != NULL) {
                         if (flen > maxlen) maxlen = flen;
                         if (vlen > maxlen) maxlen = vlen;
                         zl = ziplistPush(zl, fstr, flen, ZIPLIST_TAIL);
                         zl = ziplistPush(zl, vstr, vlen, ZIPLIST_TAIL);
+
+                        /* search for duplicate records */
+                        sds field = sdsnewlen(fstr, flen);
+                        if (dictAdd(dupSearchDict, field, NULL) != DICT_OK) {
+                            rdbReportCorruptRDB("Hash zipmap with dup elements");
+                            dictRelease(dupSearchDict);
+                            sdsfree(field);
+                            zfree(encoded);
+                            o->ptr = NULL;
+                            decrRefCount(o);
+                            return NULL;
+                        }
                     }
 
+                    dictRelease(dupSearchDict);
                     zfree(o->ptr);
                     o->ptr = zl;
                     o->type = OBJ_HASH;
@@ -1804,23 +1849,55 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                 }
                 break;
             case RDB_TYPE_LIST_ZIPLIST:
+                if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
+                if (!ziplistValidateIntegrity(encoded, encoded_len, deep_integrity_validation, NULL, NULL)) {
+                    rdbReportCorruptRDB("List ziplist integrity check failed.");
+                    zfree(encoded);
+                    o->ptr = NULL;
+                    decrRefCount(o);
+                    return NULL;
+                }
                 o->type = OBJ_LIST;
                 o->encoding = OBJ_ENCODING_ZIPLIST;
                 listTypeConvert(o,OBJ_ENCODING_QUICKLIST);
                 break;
             case RDB_TYPE_SET_INTSET:
+                if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
+                if (!intsetValidateIntegrity(encoded, encoded_len, deep_integrity_validation)) {
+                    rdbReportCorruptRDB("Intset integrity check failed.");
+                    zfree(encoded);
+                    o->ptr = NULL;
+                    decrRefCount(o);
+                    return NULL;
+                }
                 o->type = OBJ_SET;
                 o->encoding = OBJ_ENCODING_INTSET;
                 if (intsetLen(o->ptr) > server.set_max_intset_entries)
                     setTypeConvert(o,OBJ_ENCODING_HT);
                 break;
             case RDB_TYPE_ZSET_ZIPLIST:
+                if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
+                if (!zsetZiplistValidateIntegrity(encoded, encoded_len, deep_integrity_validation)) {
+                    rdbReportCorruptRDB("Zset ziplist integrity check failed.");
+                    zfree(encoded);
+                    o->ptr = NULL;
+                    decrRefCount(o);
+                    return NULL;
+                }
                 o->type = OBJ_ZSET;
                 o->encoding = OBJ_ENCODING_ZIPLIST;
                 if (zsetLength(o) > server.zset_max_ziplist_entries)
                     zsetConvert(o,OBJ_ENCODING_SKIPLIST);
                 break;
             case RDB_TYPE_HASH_ZIPLIST:
+                if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
+                if (!hashZiplistValidateIntegrity(encoded, encoded_len, deep_integrity_validation)) {
+                    rdbReportCorruptRDB("Hash ziplist integrity check failed.");
+                    zfree(encoded);
+                    o->ptr = NULL;
+                    decrRefCount(o);
+                    return NULL;
+                }
                 o->type = OBJ_HASH;
                 o->encoding = OBJ_ENCODING_ZIPLIST;
                 if (hashTypeLength(o) > server.hash_max_ziplist_entries)
@@ -1828,7 +1905,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                 break;
             default:
                 /* totally unreachable */
-                rdbExitReportCorruptRDB("Unknown RDB encoding type %d",rdbtype);
+                rdbReportCorruptRDB("Unknown RDB encoding type %d",rdbtype);
                 break;
         }
     } else if (rdbtype == RDB_TYPE_STREAM_LISTPACKS) {
@@ -1852,7 +1929,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                 return NULL;
             }
             if (sdslen(nodekey) != sizeof(streamID)) {
-                rdbExitReportCorruptRDB("Stream node key entry is not the "
+                rdbReportCorruptRDB("Stream node key entry is not the "
                                         "size of a stream ID");
                 sdsfree(nodekey);
                 decrRefCount(o);
@@ -1871,7 +1948,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             }
             if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
             if (!streamValidateListpackIntegrity(lp, lp_size, deep_integrity_validation)) {
-                rdbExitReportCorruptRDB("Stream listpack integrity check failed.");
+                rdbReportCorruptRDB("Stream listpack integrity check failed.");
                 sdsfree(nodekey);
                 decrRefCount(o);
                 zfree(lp);
@@ -1883,7 +1960,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                 /* Serialized listpacks should never be empty, since on
                  * deletion we should remove the radix tree key if the
                  * resulting listpack is empty. */
-                rdbExitReportCorruptRDB("Empty listpack inside stream");
+                rdbReportCorruptRDB("Empty listpack inside stream");
                 sdsfree(nodekey);
                 decrRefCount(o);
                 zfree(lp);
@@ -1895,7 +1972,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                 (unsigned char*)nodekey,sizeof(streamID),lp,NULL);
             sdsfree(nodekey);
             if (!retval) {
-                rdbExitReportCorruptRDB("Listpack re-added with existing key");
+                rdbReportCorruptRDB("Listpack re-added with existing key");
                 decrRefCount(o);
                 zfree(lp);
                 return NULL;
@@ -1945,7 +2022,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
 
             streamCG *cgroup = streamCreateCG(s,cgname,sdslen(cgname),&cg_id);
             if (cgroup == NULL) {
-                rdbExitReportCorruptRDB("Duplicated consumer group name %s",
+                rdbReportCorruptRDB("Duplicated consumer group name %s",
                                          cgname);
                 decrRefCount(o);
                 sdsfree(cgname);
@@ -1981,7 +2058,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                     return NULL;
                 }
                 if (!raxInsert(cgroup->pel,rawid,sizeof(rawid),nack,NULL)) {
-                    rdbExitReportCorruptRDB("Duplicated global PEL entry "
+                    rdbReportCorruptRDB("Duplicated global PEL entry "
                                             "loading stream consumer group");
                     decrRefCount(o);
                     streamFreeNACK(nack);
@@ -2034,7 +2111,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                     }
                     streamNACK *nack = raxFind(cgroup->pel,rawid,sizeof(rawid));
                     if (nack == raxNotFound) {
-                        rdbExitReportCorruptRDB("Consumer entry not found in "
+                        rdbReportCorruptRDB("Consumer entry not found in "
                                                 "group global PEL");
                         decrRefCount(o);
                         return NULL;
@@ -2045,7 +2122,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                      * NACK structure also in the consumer-specific PEL. */
                     nack->consumer = consumer;
                     if (!raxInsert(consumer->pel,rawid,sizeof(rawid),nack,NULL)) {
-                        rdbExitReportCorruptRDB("Duplicated consumer PEL entry "
+                        rdbReportCorruptRDB("Duplicated consumer PEL entry "
                                                 " loading a stream consumer "
                                                 "group");
                         decrRefCount(o);
@@ -2070,7 +2147,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
 
         if (mt == NULL) {
             moduleTypeNameByID(name,moduleid);
-            rdbExitReportCorruptRDB("The RDB file contains module data I can't load: no matching module '%s'", name);
+            rdbReportCorruptRDB("The RDB file contains module data I can't load: no matching module '%s'", name);
             return NULL;
         }
         RedisModuleIO io;
@@ -2097,7 +2174,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                 return NULL;
             }
             if (eof != RDB_MODULE_OPCODE_EOF) {
-                rdbExitReportCorruptRDB("The RDB file contains module data for the module '%s' that is not terminated by the proper module value EOF marker", name);
+                rdbReportCorruptRDB("The RDB file contains module data for the module '%s' that is not terminated by the proper module value EOF marker", name);
                 if (ptr) {
                     o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */
                     decrRefCount(o);
@@ -2108,7 +2185,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
 
         if (ptr == NULL) {
             moduleTypeNameByID(name,moduleid);
-            rdbExitReportCorruptRDB("The RDB file contains module data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name);
+            rdbReportCorruptRDB("The RDB file contains module data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name);
             return NULL;
         }
         o = createModuleObject(mt,ptr);
@@ -2327,7 +2404,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
             } else if (!strcasecmp(auxkey->ptr,"lua")) {
                 /* Load the script back in memory. */
                 if (luaCreateFunction(NULL,server.lua,auxval) == NULL) {
-                    rdbExitReportCorruptRDB(
+                    rdbReportCorruptRDB(
                         "Can't load Lua script from RDB file! "
                         "BODY: %s", (char*)auxval->ptr);
                 }
@@ -2492,7 +2569,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
                     "got (%llx). Aborting now.",
                         (unsigned long long)expected,
                         (unsigned long long)cksum);
-                rdbExitReportCorruptRDB("RDB CRC error");
+                rdbReportCorruptRDB("RDB CRC error");
                 return C_ERR;
             }
         }
diff --git a/src/server.h b/src/server.h
index 47f054b3f..37197c0b3 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2071,6 +2071,7 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore);
 long zsetRank(robj *zobj, sds ele, int reverse);
 int zsetDel(robj *zobj, sds ele);
 robj *zsetDup(robj *o);
+int zsetZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep);
 void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey, robj *countarg);
 sds ziplistGetObject(unsigned char *sptr);
 int zslValueGteMin(double value, zrangespec *spec);
@@ -2177,6 +2178,7 @@ robj *hashTypeLookupWriteOrCreate(client *c, robj *key);
 robj *hashTypeGetValueObject(robj *o, sds field);
 int hashTypeSet(robj *o, sds field, sds value, int flags);
 robj *hashTypeDup(robj *o);
+int hashZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep);
 
 /* Pub / Sub */
 int pubsubUnsubscribeAllChannels(client *c, int notify);
diff --git a/src/t_hash.c b/src/t_hash.c
index e6f14091b..ff9ac742e 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -549,6 +549,55 @@ robj *hashTypeDup(robj *o) {
     return hobj;
 }
 
+/* callback for to check the ziplist doesn't have duplicate recoreds */
+static int _hashZiplistEntryValidation(unsigned char *p, void *userdata) {
+    struct {
+        long count;
+        dict *fields;
+    } *data = userdata;
+
+    /* Odd records are field names, add to dict and check that's not a dup */
+    if (((data->count) & 1) == 0) {
+        unsigned char *str;
+        unsigned int slen;
+        long long vll;
+        if (!ziplistGet(p, &str, &slen, &vll))
+            return 0;
+        sds field = str? sdsnewlen(str, slen): sdsfromlonglong(vll);;
+        if (dictAdd(data->fields, field, NULL) != DICT_OK) {
+            /* Duplicate, return an error */
+            sdsfree(field);
+            return 0;
+        }
+    }
+
+    (data->count)++;
+    return 1;
+}
+
+/* Validate the integrity of the data stracture.
+ * when `deep` is 0, only the integrity of the header is validated.
+ * when `deep` is 1, we scan all the entries one by one. */
+int hashZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep) {
+    if (!deep)
+        return ziplistValidateIntegrity(zl, size, 0, NULL, NULL);
+
+    /* Keep track of the field names to locate duplicate ones */
+    struct {
+        long count;
+        dict *fields;
+    } data = {0, dictCreate(&hashDictType, NULL)};
+
+    int ret = ziplistValidateIntegrity(zl, size, 1, _hashZiplistEntryValidation, &data);
+
+    /* make sure we have an even number of records. */
+    if (data.count & 1)
+        ret = 0;
+
+    dictRelease(data.fields);
+    return ret;
+}
+
 /*-----------------------------------------------------------------------------
  * Hash type commands
  *----------------------------------------------------------------------------*/
diff --git a/src/t_zset.c b/src/t_zset.c
index 0b3cc061e..d4ffe1318 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -1604,6 +1604,55 @@ robj *zsetDup(robj *o) {
     return zobj;
 }
 
+/* callback for to check the ziplist doesn't have duplicate recoreds */
+static int _zsetZiplistValidateIntegrity(unsigned char *p, void *userdata) {
+    struct {
+        long count;
+        dict *fields;
+    } *data = userdata;
+
+    /* Even records are field names, add to dict and check that's not a dup */
+    if (((data->count) & 1) == 1) {
+        unsigned char *str;
+        unsigned int slen;
+        long long vll;
+        if (!ziplistGet(p, &str, &slen, &vll))
+            return 0;
+        sds field = str? sdsnewlen(str, slen): sdsfromlonglong(vll);;
+        if (dictAdd(data->fields, field, NULL) != DICT_OK) {
+            /* Duplicate, return an error */
+            sdsfree(field);
+            return 0;
+        }
+    }
+
+    (data->count)++;
+    return 1;
+}
+
+/* Validate the integrity of the data stracture.
+ * when `deep` is 0, only the integrity of the header is validated.
+ * when `deep` is 1, we scan all the entries one by one. */
+int zsetZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep) {
+    if (!deep)
+        return ziplistValidateIntegrity(zl, size, 0, NULL, NULL);
+
+    /* Keep track of the field names to locate duplicate ones */
+    struct {
+        long count;
+        dict *fields;
+    } data = {0, dictCreate(&hashDictType, NULL)};
+
+    int ret = ziplistValidateIntegrity(zl, size, 1, _zsetZiplistValidateIntegrity, &data);
+
+    /* make sure we have an even number of records. */
+    if (data.count & 1)
+        ret = 0;
+
+    dictRelease(data.fields);
+    return ret;
+}
+
 /*-----------------------------------------------------------------------------
  * Sorted set commands
  *----------------------------------------------------------------------------*/
diff --git a/src/ziplist.c b/src/ziplist.c
index a8ab57cfe..a4d8d4afe 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -1406,7 +1406,8 @@ void ziplistRepr(unsigned char *zl) {
 /* Validate the integrity of the data stracture.
  * when `deep` is 0, only the integrity of the header is validated.
  * when `deep` is 1, we scan all the entries one by one. */
-int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep) {
+int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep,
+    ziplistValidateEntryCB entry_cb, void *cb_userdata) {
     /* check that we can actually read the header. (and ZIP_END) */
     if (size < ZIPLIST_HEADER_SIZE + ZIPLIST_END_SIZE)
         return 0;
@@ -1441,6 +1442,10 @@ int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep) {
         if (e.prevrawlen != prev_raw_size)
             return 0;
 
+        /* Optionally let the caller validate the entry too. */
+        if (entry_cb && !entry_cb(p, cb_userdata))
+            return 0;
+
         /* Move to the next entry */
         prev_raw_size = e.headersize + e.len;
         prev = p;
diff --git a/src/ziplist.h b/src/ziplist.h
index 8d1ac1691..5153951dc 100644
--- a/src/ziplist.h
+++ b/src/ziplist.h
@@ -49,7 +49,9 @@ unsigned char *ziplistFind(unsigned char *zl, unsigned char *p, unsigned char *v
 unsigned int ziplistLen(unsigned char *zl);
 size_t ziplistBlobLen(unsigned char *zl);
 void ziplistRepr(unsigned char *zl);
-int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep);
+typedef int (*ziplistValidateEntryCB)(unsigned char* p, void* userdata);
+int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep,
+                             ziplistValidateEntryCB entry_cb, void *cb_userdata);
 
 #ifdef REDIS_TEST
 int ziplistTest(int argc, char *argv[]);
diff --git a/tests/integration/corrupt-dump.tcl b/tests/integration/corrupt-dump.tcl
index b8828fd94..fbcca2f2a 100644
--- a/tests/integration/corrupt-dump.tcl
+++ b/tests/integration/corrupt-dump.tcl
@@ -17,7 +17,7 @@ test {corrupt payload: #7445 - with sanitize} {
             r restore key 0 $corrupt_payload_7445
         } err
         assert_match "*Bad data format*" $err
-        verify_log_message 0 "*Ziplist integrity check failed*" 0
+        verify_log_message 0 "*integrity check failed*" 0
     }
 }
 
@@ -89,7 +89,7 @@ test {corrupt payload: quicklist small ziplist prev len} {
             r restore key 0 "\x0E\x01\x13\x13\x00\x00\x00\x0E\x00\x00\x00\x02\x00\x00\x02\x61\x00\x02\x02\x62\x00\xFF\x09\x00\xC7\x71\x03\x97\x07\x75\xB0\x63"
         } err
         assert_match "*Bad data format*" $err
-        verify_log_message 0 "*Ziplist integrity check failed*" 0
+        verify_log_message 0 "*integrity check failed*" 0
     }
 }
 
@@ -114,7 +114,7 @@ test {corrupt payload: #3080 - quicklist} {
             r DUMP key ;# DUMP was used in the original issue, but now even with shallow sanitization restore safely fails, so this is dead code
         } err
         assert_match "*Bad data format*" $err
-        verify_log_message 0 "*Ziplist integrity check failed*" 0
+        verify_log_message 0 "*integrity check failed*" 0
     }
 }
 
@@ -126,7 +126,7 @@ test {corrupt payload: #3080 - ziplist} {
             r RESTORE key 0 "\x0A\x80\x00\x00\x00\x10\x41\x41\x41\x41\x41\x41\x41\x41\x02\x00\x00\x80\x41\x41\x41\x41\x07\x00\x39\x5B\x49\xE0\xC1\xC6\xDD\x76"
         } err
         assert_match "*Bad data format*" $err
-        verify_log_message 0 "*Ziplist integrity check failed*" 0
+        verify_log_message 0 "*integrity check failed*" 0
     }
 }
 
@@ -144,7 +144,7 @@ test {corrupt payload: load corrupted rdb with no CRC - #3505} {
 
     set stdout [dict get $srv stdout]
     assert_equal [count_message_lines $stdout "Terminating server after rdb file reading failure."]  1
-    assert_lessthan 1 [count_message_lines $stdout "Ziplist integrity check failed"]
+    assert_lessthan 1 [count_message_lines $stdout "integrity check failed"]
     kill_server $srv ;# let valgrind look for issues
 }
 
@@ -194,6 +194,36 @@ test {corrupt payload: listpack too long entry prev len} {
     }
 }
 
+test {corrupt payload: hash ziplist with duplicate records} {
+    # when we do perform full sanitization, we expect duplicate records to fail the restore
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload yes
+        r debug set-skip-checksum-validation 1
+        catch { r RESTORE _hash 0 "\x0D\x3D\x3D\x00\x00\x00\x3A\x00\x00\x00\x14\x13\x00\xF5\x02\xF5\x02\xF2\x02\x53\x5F\x31\x04\xF3\x02\xF3\x02\xF7\x02\xF7\x02\xF8\x02\x02\x5F\x37\x04\xF1\x02\xF1\x02\xF6\x02\x02\x5F\x35\x04\xF4\x02\x02\x5F\x33\x04\xFA\x02\x02\x5F\x39\x04\xF9\x02\xF9\xFF\x09\x00\xB5\x48\xDE\x62\x31\xD0\xE5\x63" } err
+        assert_match "*Bad data format*" $err
+    }
+}
+
+test {corrupt payload: hash ziplist uneven record count} {
+    # when we do perform full sanitization, we expect duplicate records to fail the restore
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload yes
+        r debug set-skip-checksum-validation 1
+        catch { r RESTORE _hash 0 "\r\x1b\x1b\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02a\x00\x04\x02b\x00\x04\x02a\x00\x04\x02d\x00\xff\t\x00\xa1\x98\x36x\xcc\x8e\x93\x2e" } err
+        assert_match "*Bad data format*" $err
+    }
+}
+
+test {corrupt payload: hash dupliacte records} {
+    # when we do perform full sanitization, we expect duplicate records to fail the restore
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload yes
+        r debug set-skip-checksum-validation 1
+        catch { r RESTORE _hash 0 "\x04\x02\x01a\x01b\x01a\x01d\t\x00\xc6\x9c\xab\xbc\bk\x0c\x06" } err
+        assert_match "*Bad data format*" $err
+    }
+}
+
 test {corrupt payload: fuzzer findings - NPD in streamIteratorGetID} {
     start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
         r config set sanitize-dump-payload no
@@ -255,7 +285,7 @@ test {corrupt payload: fuzzer findings - invalid ziplist encoding} {
             r RESTORE _listbig 0 "\x0E\x02\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\x02\x5F\x39\x04\xF9\x02\x86\x5F\x37\x04\xF7\x02\x02\x5F\x35\xFF\x19\x19\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\xF5\x02\x02\x5F\x33\x04\xF3\x02\x02\x5F\x31\x04\xF1\xFF\x09\x00\x0C\xFC\x99\x2C\x23\x45\x15\x60"
         } err
         assert_match "*Bad data format*" $err
-        verify_log_message 0 "*Ziplist integrity check failed*" 0
+        verify_log_message 0 "*integrity check failed*" 0
     }
 }
 
@@ -402,6 +432,18 @@ test {corrupt payload: fuzzer findings - infinite loop} {
     }
 }
 
+test {corrupt payload: fuzzer findings - hash convert asserts on RESTORE with shallow sanitization} {
+    # if we don't perform full sanitization, and the next command can assert on converting
+    # a ziplist to hash records, then we're ok with that happning in RESTORE too
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        catch { r RESTORE _hash 0 "\x0D\x3D\x3D\x00\x00\x00\x3A\x00\x00\x00\x14\x13\x00\xF5\x02\xF5\x02\xF2\x02\x53\x5F\x31\x04\xF3\x02\xF3\x02\xF7\x02\xF7\x02\xF8\x02\x02\x5F\x37\x04\xF1\x02\xF1\x02\xF6\x02\x02\x5F\x35\x04\xF4\x02\x02\x5F\x33\x04\xFA\x02\x02\x5F\x39\x04\xF9\x02\xF9\xFF\x09\x00\xB5\x48\xDE\x62\x31\xD0\xE5\x63" }
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
+    }
+}
+
 test {corrupt payload: fuzzer findings - invalid tail offset after removal} {
     start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
         r config set sanitize-dump-payload no

From 7ca00d694d44be13a3ff9ff1c96b49222ac9463b Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 22 Nov 2020 21:22:49 +0200
Subject: [PATCH 0847/1098] Sanitize dump payload: fail RESTORE if memory
 allocation fails

When RDB input attempts to make a huge memory allocation that fails,
RESTORE should fail gracefully rather than die with panic
---
 src/dict.c                         |  29 +++++-
 src/dict.h                         |   1 +
 src/rdb.c                          |  56 +++++++---
 src/sds.c                          |  14 ++-
 src/sds.h                          |   1 +
 src/sdsalloc.h                     |   4 +
 src/zmalloc.c                      | 158 +++++++++++++++++------------
 src/zmalloc.h                      |   6 ++
 tests/integration/corrupt-dump.tcl |  19 ++++
 9 files changed, 202 insertions(+), 86 deletions(-)

diff --git a/src/dict.c b/src/dict.c
index cca5aa921..4736dacd5 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -143,9 +143,13 @@ int dictResize(dict *d)
     return dictExpand(d, minimal);
 }
 
-/* Expand or create the hash table */
-int dictExpand(dict *d, unsigned long size)
+/* Expand or create the hash table,
+ * when malloc_failed is non-NULL, it'll avoid panic if malloc fails (in which case it'll be set to 1).
+ * Returns DICT_OK if expand was performed, and DICT_ERR if skipped. */
+int _dictExpand(dict *d, unsigned long size, int* malloc_failed)
 {
+    if (malloc_failed) *malloc_failed = 0;
+
     /* the size is invalid if it is smaller than the number of
      * elements already inside the hash table */
     if (dictIsRehashing(d) || d->ht[0].used > size)
@@ -160,7 +164,14 @@ int dictExpand(dict *d, unsigned long size)
     /* Allocate the new hash table and initialize all pointers to NULL */
     n.size = realsize;
     n.sizemask = realsize-1;
-    n.table = zcalloc(realsize*sizeof(dictEntry*));
+    if (malloc_failed) {
+        n.table = ztrycalloc(realsize*sizeof(dictEntry*));
+        *malloc_failed = n.table == NULL;
+        if (*malloc_failed)
+            return DICT_ERR;
+    } else
+        n.table = zcalloc(realsize*sizeof(dictEntry*));
+
     n.used = 0;
 
     /* Is this the first initialization? If so it's not really a rehashing
@@ -176,6 +187,18 @@ int dictExpand(dict *d, unsigned long size)
     return DICT_OK;
 }
 
+/* return DICT_ERR if expand was not performed */
+int dictExpand(dict *d, unsigned long size) {
+    return _dictExpand(d, size, NULL);
+}
+
+/* return DICT_ERR if expand failed due to memory allocation failure */
+int dictTryExpand(dict *d, unsigned long size) {
+    int malloc_failed;
+    _dictExpand(d, size, &malloc_failed);
+    return malloc_failed? DICT_ERR : DICT_OK;
+}
+
 /* Performs N steps of incremental rehashing. Returns 1 if there are still
  * keys to move from the old to the new hash table, otherwise 0 is returned.
  *
diff --git a/src/dict.h b/src/dict.h
index fca924abe..f7515e905 100644
--- a/src/dict.h
+++ b/src/dict.h
@@ -151,6 +151,7 @@ typedef void (dictScanBucketFunction)(void *privdata, dictEntry **bucketref);
 /* API */
 dict *dictCreate(dictType *type, void *privDataPtr);
 int dictExpand(dict *d, unsigned long size);
+int dictTryExpand(dict *d, unsigned long size);
 int dictAdd(dict *d, void *key, void *val);
 dictEntry *dictAddRaw(dict *d, void *key, dictEntry **existing);
 dictEntry *dictAddOrFind(dict *d, void *key);
diff --git a/src/rdb.c b/src/rdb.c
index e8501ee50..0878be831 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -387,14 +387,22 @@ void *rdbLoadLzfStringObject(rio *rdb, int flags, size_t *lenptr) {
 
     if ((clen = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL;
     if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL;
-    if ((c = zmalloc(clen)) == NULL) goto err;
+    if ((c = ztrymalloc(clen)) == NULL) {
+        serverLog(server.loading? LL_WARNING: LL_VERBOSE, "rdbLoadLzfStringObject failed allocating %llu bytes", (unsigned long long)clen);
+        goto err;
+    }
 
     /* Allocate our target according to the uncompressed size. */
     if (plain) {
-        val = zmalloc(len);
+        val = ztrymalloc(len);
     } else {
-        val = sdsnewlen(SDS_NOINIT,len);
+        val = sdstrynewlen(SDS_NOINIT,len);
     }
+    if (!val) {
+        serverLog(server.loading? LL_WARNING: LL_VERBOSE, "rdbLoadLzfStringObject failed allocating %llu bytes", (unsigned long long)len);
+        goto err;
+    }
+
     if (lenptr) *lenptr = len;
 
     /* Load the compressed representation and uncompress it to target. */
@@ -522,7 +530,11 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags, size_t *lenptr) {
     }
 
     if (plain || sds) {
-        void *buf = plain ? zmalloc(len) : sdsnewlen(SDS_NOINIT,len);
+        void *buf = plain ? ztrymalloc(len) : sdstrynewlen(SDS_NOINIT,len);
+        if (!buf) {
+            serverLog(server.loading? LL_WARNING: LL_VERBOSE, "rdbGenericLoadStringObject failed allocating %llu bytes", len);
+            return NULL;
+        }
         if (lenptr) *lenptr = len;
         if (len && rioRead(rdb,buf,len) == 0) {
             if (plain)
@@ -1545,8 +1557,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             o = createSetObject();
             /* It's faster to expand the dict to the right size asap in order
              * to avoid rehashing */
-            if (len > DICT_HT_INITIAL_SIZE)
-                dictExpand(o->ptr,len);
+            if (len > DICT_HT_INITIAL_SIZE && dictTryExpand(o->ptr,len) != DICT_OK) {
+                rdbReportCorruptRDB("OOM in dictTryExpand %llu", (unsigned long long)len);
+                decrRefCount(o);
+                return NULL;
+            }
         } else {
             o = createIntsetObject();
         }
@@ -1574,7 +1589,12 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                     }
                 } else {
                     setTypeConvert(o,OBJ_ENCODING_HT);
-                    dictExpand(o->ptr,len);
+                    if (dictTryExpand(o->ptr,len) != DICT_OK) {
+                        rdbReportCorruptRDB("OOM in dictTryExpand %llu", (unsigned long long)len);
+                        sdsfree(sdsele);
+                        decrRefCount(o);
+                        return NULL;
+                    }
                 }
             }
 
@@ -1601,8 +1621,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
         o = createZsetObject();
         zs = o->ptr;
 
-        if (zsetlen > DICT_HT_INITIAL_SIZE)
-            dictExpand(zs->dict,zsetlen);
+        if (zsetlen > DICT_HT_INITIAL_SIZE && dictTryExpand(zs->dict,zsetlen) != DICT_OK) {
+            rdbReportCorruptRDB("OOM in dictTryExpand %llu", (unsigned long long)zsetlen);
+            decrRefCount(o);
+            return NULL;
+        }
 
         /* Load every single element of the sorted set. */
         while(zsetlen--) {
@@ -1723,8 +1746,13 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             dupSearchDict = NULL;
         }
 
-        if (o->encoding == OBJ_ENCODING_HT && len > DICT_HT_INITIAL_SIZE)
-            dictExpand(o->ptr,len);
+        if (o->encoding == OBJ_ENCODING_HT && len > DICT_HT_INITIAL_SIZE) {
+            if (dictTryExpand(o->ptr,len) != DICT_OK) {
+                rdbReportCorruptRDB("OOM in dictTryExpand %llu", (unsigned long long)len);
+                decrRefCount(o);
+                return NULL;
+            }
+        }
 
         /* Load remaining fields and values into the hash table */
         while (o->encoding == OBJ_ENCODING_HT && len > 0) {
@@ -1823,9 +1851,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
                         zl = ziplistPush(zl, vstr, vlen, ZIPLIST_TAIL);
 
                         /* search for duplicate records */
-                        sds field = sdsnewlen(fstr, flen);
-                        if (dictAdd(dupSearchDict, field, NULL) != DICT_OK) {
-                            rdbReportCorruptRDB("Hash zipmap with dup elements");
+                        sds field = sdstrynewlen(fstr, flen);
+                        if (!field || dictAdd(dupSearchDict, field, NULL) != DICT_OK) {
+                            rdbReportCorruptRDB("Hash zipmap with dup elements, or big length (%u)", flen);
                             dictRelease(dupSearchDict);
                             sdsfree(field);
                             zfree(encoded);
diff --git a/src/sds.c b/src/sds.c
index f72eac921..6a85eb4aa 100644
--- a/src/sds.c
+++ b/src/sds.c
@@ -100,7 +100,7 @@ static inline size_t sdsTypeMaxSize(char type) {
  * You can print the string with printf() as there is an implicit \0 at the
  * end of the string. However the string is binary safe and can contain
  * \0 characters in the middle, as the length is stored in the sds header. */
-sds sdsnewlen(const void *init, size_t initlen) {
+sds _sdsnewlen(const void *init, size_t initlen, int trymalloc) {
     void *sh;
     sds s;
     char type = sdsReqType(initlen);
@@ -111,7 +111,9 @@ sds sdsnewlen(const void *init, size_t initlen) {
     unsigned char *fp; /* flags pointer. */
     size_t usable;
 
-    sh = s_malloc_usable(hdrlen+initlen+1, &usable);
+    sh = trymalloc?
+        s_trymalloc_usable(hdrlen+initlen+1, &usable) :
+        s_malloc_usable(hdrlen+initlen+1, &usable);
     if (sh == NULL) return NULL;
     if (init==SDS_NOINIT)
         init = NULL;
@@ -162,6 +164,14 @@ sds sdsnewlen(const void *init, size_t initlen) {
     return s;
 }
 
+sds sdsnewlen(const void *init, size_t initlen) {
+    return _sdsnewlen(init, initlen, 0);
+}
+
+sds sdstrynewlen(const void *init, size_t initlen) {
+    return _sdsnewlen(init, initlen, 1);
+}
+
 /* Create an empty (zero length) sds string. Even in this case the string
  * always has an implicit null term. */
 sds sdsempty(void) {
diff --git a/src/sds.h b/src/sds.h
index adcc12c0a..3a9e4cefe 100644
--- a/src/sds.h
+++ b/src/sds.h
@@ -216,6 +216,7 @@ static inline void sdssetalloc(sds s, size_t newlen) {
 }
 
 sds sdsnewlen(const void *init, size_t initlen);
+sds sdstrynewlen(const void *init, size_t initlen);
 sds sdsnew(const char *init);
 sds sdsempty(void);
 sds sdsdup(const sds s);
diff --git a/src/sdsalloc.h b/src/sdsalloc.h
index 725ad79e3..a1c5584f0 100644
--- a/src/sdsalloc.h
+++ b/src/sdsalloc.h
@@ -42,9 +42,13 @@
 #include "zmalloc.h"
 #define s_malloc zmalloc
 #define s_realloc zrealloc
+#define s_trymalloc ztrymalloc
+#define s_tryrealloc ztryrealloc
 #define s_free zfree
 #define s_malloc_usable zmalloc_usable
 #define s_realloc_usable zrealloc_usable
+#define s_trymalloc_usable ztrymalloc_usable
+#define s_tryrealloc_usable ztryrealloc_usable
 #define s_free_usable zfree_usable
 
 #endif
diff --git a/src/zmalloc.c b/src/zmalloc.c
index 7425198fe..a04ecacae 100644
--- a/src/zmalloc.c
+++ b/src/zmalloc.c
@@ -85,33 +85,44 @@ static void zmalloc_default_oom(size_t size) {
 
 static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom;
 
-void *zmalloc(size_t size) {
+/* Try allocating memory, and return NULL if failed.
+ * '*usable' is set to the usable size if non NULL. */
+void *ztrymalloc_usable(size_t size, size_t *usable) {
     void *ptr = malloc(size+PREFIX_SIZE);
 
-    if (!ptr) zmalloc_oom_handler(size);
+    if (!ptr) return NULL;
 #ifdef HAVE_MALLOC_SIZE
-    update_zmalloc_stat_alloc(zmalloc_size(ptr));
+    size = zmalloc_size(ptr);
+    update_zmalloc_stat_alloc(size);
+    if (usable) *usable = size;
     return ptr;
 #else
     *((size_t*)ptr) = size;
     update_zmalloc_stat_alloc(size+PREFIX_SIZE);
+    if (usable) *usable = size;
     return (char*)ptr+PREFIX_SIZE;
 #endif
 }
 
-/* Similar to zmalloc, '*usable' is set to the usable size. */
-void *zmalloc_usable(size_t size, size_t *usable) {
-    void *ptr = malloc(size+PREFIX_SIZE);
-
+/* Allocate memory or panic */
+void *zmalloc(size_t size) {
+    void *ptr = ztrymalloc_usable(size, NULL);
+    if (!ptr) zmalloc_oom_handler(size);
+    return ptr;
+}
+
+/* Try allocating memory, and return NULL if failed. */
+void *ztrymalloc(size_t size) {
+    void *ptr = ztrymalloc_usable(size, NULL);
+    return ptr;
+}
+
+/* Allocate memory or panic.
+ * '*usable' is set to the usable size if non NULL. */
+void *zmalloc_usable(size_t size, size_t *usable) {
+    void *ptr = ztrymalloc_usable(size, usable);
     if (!ptr) zmalloc_oom_handler(size);
-#ifdef HAVE_MALLOC_SIZE
-    update_zmalloc_stat_alloc(*usable = zmalloc_size(ptr));
     return ptr;
-#else
-    *((size_t*)ptr) = *usable = size;
-    update_zmalloc_stat_alloc(size+PREFIX_SIZE);
-    return (char*)ptr+PREFIX_SIZE;
-#endif
 }
 
 /* Allocation and free functions that bypass the thread cache
@@ -132,101 +143,114 @@ void zfree_no_tcache(void *ptr) {
 }
 #endif
 
-void *zcalloc(size_t size) {
+/* Try allocating memory and zero it, and return NULL if failed.
+ * '*usable' is set to the usable size if non NULL. */
+void *ztrycalloc_usable(size_t size, size_t *usable) {
     void *ptr = calloc(1, size+PREFIX_SIZE);
+    if (ptr == NULL) return NULL;
 
-    if (!ptr) zmalloc_oom_handler(size);
 #ifdef HAVE_MALLOC_SIZE
-    update_zmalloc_stat_alloc(zmalloc_size(ptr));
+    size = zmalloc_size(ptr);
+    update_zmalloc_stat_alloc(size);
+    if (usable) *usable = size;
     return ptr;
 #else
     *((size_t*)ptr) = size;
     update_zmalloc_stat_alloc(size+PREFIX_SIZE);
+    if (usable) *usable = size;
     return (char*)ptr+PREFIX_SIZE;
 #endif
 }
 
-/* Similar to zcalloc, '*usable' is set to the usable size. */
-void *zcalloc_usable(size_t size, size_t *usable) {
-    void *ptr = calloc(1, size+PREFIX_SIZE);
-
+/* Allocate memory and zero it or panic */
+void *zcalloc(size_t size) {
+    void *ptr = ztrycalloc_usable(size, NULL);
     if (!ptr) zmalloc_oom_handler(size);
-#ifdef HAVE_MALLOC_SIZE
-    update_zmalloc_stat_alloc(*usable = zmalloc_size(ptr));
     return ptr;
-#else
-    *((size_t*)ptr) = *usable = size;
-    update_zmalloc_stat_alloc(size+PREFIX_SIZE);
-    return (char*)ptr+PREFIX_SIZE;
-#endif
 }
 
-void *zrealloc(void *ptr, size_t size) {
+/* Try allocating memory, and return NULL if failed. */
+void *ztrycalloc(size_t size) {
+    void *ptr = ztrycalloc_usable(size, NULL);
+    return ptr;
+}
+
+/* Allocate memory or panic.
+ * '*usable' is set to the usable size if non NULL. */
+void *zcalloc_usable(size_t size, size_t *usable) {
+    void *ptr = ztrycalloc_usable(size, usable);
+    if (!ptr) zmalloc_oom_handler(size);
+    return ptr;
+}
+
+/* Try reallocating memory, and return NULL if failed.
+ * '*usable' is set to the usable size if non NULL. */
+void *ztryrealloc_usable(void *ptr, size_t size, size_t *usable) {
 #ifndef HAVE_MALLOC_SIZE
     void *realptr;
 #endif
     size_t oldsize;
     void *newptr;
 
+    /* not allocating anything, just redirect to free. */
     if (size == 0 && ptr != NULL) {
         zfree(ptr);
+        if (usable) *usable = 0;
         return NULL;
     }
-    if (ptr == NULL) return zmalloc(size);
+    /* Not freeing anything, just redirect to malloc. */
+    if (ptr == NULL)
+        return ztrymalloc_usable(size, usable);
+
 #ifdef HAVE_MALLOC_SIZE
     oldsize = zmalloc_size(ptr);
     newptr = realloc(ptr,size);
-    if (!newptr) zmalloc_oom_handler(size);
+    if (newptr == NULL) {
+        if (usable) *usable = 0;
+        return NULL;
+    }
 
     update_zmalloc_stat_free(oldsize);
-    update_zmalloc_stat_alloc(zmalloc_size(newptr));
+    size = zmalloc_size(newptr);
+    update_zmalloc_stat_alloc(size);
+    if (usable) *usable = size;
     return newptr;
 #else
     realptr = (char*)ptr-PREFIX_SIZE;
     oldsize = *((size_t*)realptr);
     newptr = realloc(realptr,size+PREFIX_SIZE);
-    if (!newptr) zmalloc_oom_handler(size);
+    if (newptr == NULL) {
+        if (usable) *usable = 0;
+        return NULL;
+    }
 
     *((size_t*)newptr) = size;
-    update_zmalloc_stat_free(oldsize+PREFIX_SIZE);
-    update_zmalloc_stat_alloc(size+PREFIX_SIZE);
+    update_zmalloc_stat_free(oldsize);
+    update_zmalloc_stat_alloc(size);
+    if (usable) *usable = size;
     return (char*)newptr+PREFIX_SIZE;
 #endif
 }
 
-/* Similar to zrealloc, '*usable' is set to the new usable size. */
+/* Reallocate memory and zero it or panic */
+void *zrealloc(void *ptr, size_t size) {
+    ptr = ztryrealloc_usable(ptr, size, NULL);
+    if (!ptr && size != 0) zmalloc_oom_handler(size);
+    return ptr;
+}
+
+/* Try Reallocating memory, and return NULL if failed. */
+void *ztryrealloc(void *ptr, size_t size) {
+    ptr = ztryrealloc_usable(ptr, size, NULL);
+    return ptr;
+}
+
+/* Reallocate memory or panic.
+ * '*usable' is set to the usable size if non NULL. */
 void *zrealloc_usable(void *ptr, size_t size, size_t *usable) {
-#ifndef HAVE_MALLOC_SIZE
-    void *realptr;
-#endif
-    size_t oldsize;
-    void *newptr;
-
-    if (size == 0 && ptr != NULL) {
-        zfree(ptr);
-        *usable = 0;
-        return NULL;
-    }
-    if (ptr == NULL) return zmalloc_usable(size, usable);
-#ifdef HAVE_MALLOC_SIZE
-    oldsize = zmalloc_size(ptr);
-    newptr = realloc(ptr,size);
-    if (!newptr) zmalloc_oom_handler(size);
-
-    update_zmalloc_stat_free(oldsize);
-    update_zmalloc_stat_alloc(*usable = zmalloc_size(newptr));
-    return newptr;
-#else
-    realptr = (char*)ptr-PREFIX_SIZE;
-    oldsize = *((size_t*)realptr);
-    newptr = realloc(realptr,size+PREFIX_SIZE);
-    if (!newptr) zmalloc_oom_handler(size);
-
-    *((size_t*)newptr) = *usable = size;
-    update_zmalloc_stat_free(oldsize);
-    update_zmalloc_stat_alloc(size);
-    return (char*)newptr+PREFIX_SIZE;
-#endif
+    ptr = ztryrealloc_usable(ptr, size, usable);
+    if (!ptr && size != 0) zmalloc_oom_handler(size);
+    return ptr;
 }
 
 /* Provide zmalloc_size() for systems where this function is not provided by
diff --git a/src/zmalloc.h b/src/zmalloc.h
index df63253ef..64bc9fc76 100644
--- a/src/zmalloc.h
+++ b/src/zmalloc.h
@@ -80,10 +80,16 @@
 void *zmalloc(size_t size);
 void *zcalloc(size_t size);
 void *zrealloc(void *ptr, size_t size);
+void *ztrymalloc(size_t size);
+void *ztrycalloc(size_t size);
+void *ztryrealloc(void *ptr, size_t size);
 void zfree(void *ptr);
 void *zmalloc_usable(size_t size, size_t *usable);
 void *zcalloc_usable(size_t size, size_t *usable);
 void *zrealloc_usable(void *ptr, size_t size, size_t *usable);
+void *ztrymalloc_usable(size_t size, size_t *usable);
+void *ztrycalloc_usable(size_t size, size_t *usable);
+void *ztryrealloc_usable(void *ptr, size_t size, size_t *usable);
 void zfree_usable(void *ptr, size_t *usable);
 char *zstrdup(const char *s);
 size_t zmalloc_used_memory(void);
diff --git a/tests/integration/corrupt-dump.tcl b/tests/integration/corrupt-dump.tcl
index fbcca2f2a..4b09d45b9 100644
--- a/tests/integration/corrupt-dump.tcl
+++ b/tests/integration/corrupt-dump.tcl
@@ -444,6 +444,25 @@ test {corrupt payload: fuzzer findings - hash convert asserts on RESTORE with sh
     }
 }
 
+test {corrupt payload: OOM in rdbGenericLoadStringObject} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        catch { r RESTORE x 0 "\x0A\x81\x7F\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x13\x00\x00\x00\x0E\x00\x00\x00\x02\x00\x00\x02\x61\x00\x04\x02\x62\x00\xFF\x09\x00\x57\x04\xE5\xCD\xD4\x37\x6C\x57" } err
+        assert_match "*Bad data format*" $err
+        r ping
+    }
+}
+
+test {corrupt payload: fuzzer findings - OOM in dictExpand} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload no
+        r debug set-skip-checksum-validation 1
+        catch { r RESTORE x 0 "\x02\x81\x02\x5F\x31\xC0\x00\xC0\x02\x09\x00\xCD\x84\x2C\xB7\xE8\xA4\x49\x57" } err
+        assert_match "*Bad data format*" $err
+        r ping
+    }
+}
+
 test {corrupt payload: fuzzer findings - invalid tail offset after removal} {
     start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
         r config set sanitize-dump-payload no

From e288430c05359706bcb0f78730af2997ab0db07f Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 8 Sep 2020 15:51:20 +0300
Subject: [PATCH 0848/1098] Sanitize dump payload: performance optimizations
 and tuning

First, if the ziplist header is surely inside the ziplist, do fast path
decoding rather than the careful one.

In that case, streamline the encoding if-else chain to be executed only
once, and the encoding validity tested at the end.

encourage inlining

likely / unlikely hints for speculative execution

Assertion used _exit(1) to tell the compiler that the code after them is
not reachable and get rid of warnings.

But in some cases assertions are placed inside tight loops, and any
piece of code in them can slow down execution (code cache and other
reasons), instead using either abort() or better yet, unreachable
builtin.
---
 src/config.h      |  14 ++++++
 src/quicklist.c   |   9 +---
 src/redisassert.h |   6 +--
 src/server.h      |   6 +--
 src/ziplist.c     | 111 +++++++++++++++++++++++++++++++++++++++-------
 5 files changed, 116 insertions(+), 30 deletions(-)

diff --git a/src/config.h b/src/config.h
index 2cd44f927..b8723e7ce 100644
--- a/src/config.h
+++ b/src/config.h
@@ -97,6 +97,20 @@
 #define redis_fsync fsync
 #endif
 
+#if __GNUC__ >= 4
+#define redis_unreachable __builtin_unreachable
+#else
+#define redis_unreachable abort
+#endif
+
+#if __GNUC__ >= 3
+#define likely(x) __builtin_expect(!!(x), 1)
+#define unlikely(x) __builtin_expect(!!(x), 0)
+#else
+#define likely(x) (x)
+#define unlikely(x) (x)
+#endif
+
 /* Define rdb_fsync_range to sync_file_range() on Linux, otherwise we use
  * the plain fsync() call. */
 #ifdef __linux__
diff --git a/src/quicklist.c b/src/quicklist.c
index 8d68aa9ae..5ed9b6a5d 100644
--- a/src/quicklist.c
+++ b/src/quicklist.c
@@ -31,6 +31,7 @@
 #include <string.h> /* for memcpy */
 #include "quicklist.h"
 #include "zmalloc.h"
+#include "config.h"
 #include "ziplist.h"
 #include "util.h" /* for ll2string */
 #include "lzf.h"
@@ -88,14 +89,6 @@ void _quicklistBookmarkDelete(quicklist *ql, quicklistBookmark *bm);
         (e)->sz = 0;                                                           \
     } while (0)
 
-#if __GNUC__ >= 3
-#define likely(x) __builtin_expect(!!(x), 1)
-#define unlikely(x) __builtin_expect(!!(x), 0)
-#else
-#define likely(x) (x)
-#define unlikely(x) (x)
-#endif
-
 /* Create a new quicklist.
  * Free with quicklistRelease(). */
 quicklist *quicklistCreate(void) {
diff --git a/src/redisassert.h b/src/redisassert.h
index 61ab35a14..dd7920554 100644
--- a/src/redisassert.h
+++ b/src/redisassert.h
@@ -38,10 +38,10 @@
 #ifndef __REDIS_ASSERT_H__
 #define __REDIS_ASSERT_H__
 
-#include <unistd.h> /* for _exit() */
+#include "config.h"
 
-#define assert(_e) ((_e)?(void)0 : (_serverAssert(#_e,__FILE__,__LINE__),_exit(1)))
-#define panic(...) _serverPanic(__FILE__,__LINE__,__VA_ARGS__),_exit(1)
+#define assert(_e) (likely((_e))?(void)0 : (_serverAssert(#_e,__FILE__,__LINE__),redis_unreachable()))
+#define panic(...) _serverPanic(__FILE__,__LINE__,__VA_ARGS__),redis_unreachable()
 
 void _serverAssert(char *estr, char *file, int line);
 void _serverPanic(const char *file, int line, const char *msg, ...);
diff --git a/src/server.h b/src/server.h
index 37197c0b3..42ce6d32a 100644
--- a/src/server.h
+++ b/src/server.h
@@ -468,9 +468,9 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
 #define run_with_period(_ms_) if ((_ms_ <= 1000/server.hz) || !(server.cronloops%((_ms_)/(1000/server.hz))))
 
 /* We can print the stacktrace, so our assert is defined this way: */
-#define serverAssertWithInfo(_c,_o,_e) ((_e)?(void)0 : (_serverAssertWithInfo(_c,_o,#_e,__FILE__,__LINE__),_exit(1)))
-#define serverAssert(_e) ((_e)?(void)0 : (_serverAssert(#_e,__FILE__,__LINE__),_exit(1)))
-#define serverPanic(...) _serverPanic(__FILE__,__LINE__,__VA_ARGS__),_exit(1)
+#define serverAssertWithInfo(_c,_o,_e) ((_e)?(void)0 : (_serverAssertWithInfo(_c,_o,#_e,__FILE__,__LINE__),redis_unreachable()))
+#define serverAssert(_e) ((_e)?(void)0 : (_serverAssert(#_e,__FILE__,__LINE__),redis_unreachable()))
+#define serverPanic(...) _serverPanic(__FILE__,__LINE__,__VA_ARGS__),redis_unreachable()
 
 /*-----------------------------------------------------------------------------
  * Data types
diff --git a/src/ziplist.c b/src/ziplist.c
index a4d8d4afe..866078613 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -188,6 +188,7 @@
 #include "zmalloc.h"
 #include "util.h"
 #include "ziplist.h"
+#include "config.h"
 #include "endianconv.h"
 #include "redisassert.h"
 
@@ -322,13 +323,12 @@ static inline unsigned int zipEncodingLenSize(unsigned char encoding) {
     return ZIP_ENCODING_SIZE_INVALID;
 }
 
-#define ZIP_ASSERT_ENCODING(encoding) do {                          \
-    if (zipEncodingLenSize(encoding) == ZIP_ENCODING_SIZE_INVALID)     \
-        panic("Invalid encoding 0x%02X", encoding);                 \
+#define ZIP_ASSERT_ENCODING(encoding) do {                                     \
+    assert(zipEncodingLenSize(encoding) != ZIP_ENCODING_SIZE_INVALID);         \
 } while (0)
 
-/* Return bytes needed to store integer encoded by 'encoding'. */
-unsigned int zipIntSize(unsigned char encoding) {
+/* Return bytes needed to store integer encoded by 'encoding' */
+static inline unsigned int zipIntSize(unsigned char encoding) {
     switch(encoding) {
     case ZIP_INT_8B:  return 1;
     case ZIP_INT_16B: return 2;
@@ -339,6 +339,7 @@ unsigned int zipIntSize(unsigned char encoding) {
     if (encoding >= ZIP_INT_IMM_MIN && encoding <= ZIP_INT_IMM_MAX)
         return 0; /* 4 bit immediate */
     /* bad encoding, covered by a previous call to ZIP_ASSERT_ENCODING */
+    redis_unreachable();
     return 0;
 }
 
@@ -392,7 +393,8 @@ unsigned int zipStoreEntryEncoding(unsigned char *p, unsigned char encoding, uns
  * number of bytes used for the integer for integer entries) encoded in 'ptr'.
  * The 'encoding' variable is input, extracted by the caller, the 'lensize'
  * variable will hold the number of bytes required to encode the entry
- * length, and the 'len' variable will hold the entry length. */
+ * length, and the 'len' variable will hold the entry length.
+ * On invalid encoding error, lensize is set to 0. */
 #define ZIP_DECODE_LENGTH(ptr, encoding, lensize, len) do {                    \
     if ((encoding) < ZIP_STR_MASK) {                                           \
         if ((encoding) == ZIP_STR_06B) {                                       \
@@ -408,12 +410,21 @@ unsigned int zipStoreEntryEncoding(unsigned char *p, unsigned char encoding, uns
                     ((ptr)[3] <<  8) |                                         \
                     ((ptr)[4]);                                                \
         } else {                                                               \
-            len = 0; /* bad encoding, dead code */                             \
-            /* covered by a previous call to ZIP_ASSERT_ENCODING */            \
+            (lensize) = 0; /* bad encoding, should be covered by a previous */ \
+            (len) = 0;     /* ZIP_ASSERT_ENCODING / zipEncodingLenSize, or  */ \
+                           /* match the lensize after this macro with 0.    */ \
         }                                                                      \
     } else {                                                                   \
         (lensize) = 1;                                                         \
-        (len) = zipIntSize(encoding);                                          \
+        if ((encoding) == ZIP_INT_8B)  (len) = 1;                              \
+        else if ((encoding) == ZIP_INT_16B) (len) = 2;                         \
+        else if ((encoding) == ZIP_INT_24B) (len) = 3;                         \
+        else if ((encoding) == ZIP_INT_32B) (len) = 4;                         \
+        else if ((encoding) == ZIP_INT_64B) (len) = 8;                         \
+        else if (encoding >= ZIP_INT_IMM_MIN && encoding <= ZIP_INT_IMM_MAX)   \
+            (len) = 0; /* 4 bit immediate */                                   \
+        else                                                                   \
+            (lensize) = (len) = 0; /* bad encoding */                          \
     }                                                                          \
 } while(0)
 
@@ -464,7 +475,7 @@ unsigned int zipStorePrevEntryLength(unsigned char *p, unsigned int len) {
     ZIP_DECODE_PREVLENSIZE(ptr, prevlensize);                                  \
     if ((prevlensize) == 1) {                                                  \
         (prevlen) = (ptr)[0];                                                  \
-    } else if ((prevlensize) == 5) {                                           \
+    } else { /* prevlensize == 5 */                                            \
         (prevlen) = ((ptr)[4] << 24) |                                         \
                     ((ptr)[3] << 16) |                                         \
                     ((ptr)[2] <<  8) |                                         \
@@ -589,11 +600,11 @@ int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) {
  * will assert that this element is valid, so it can be freely used.
  * Generally functions such ziplistGet assume the input pointer is already
  * validated (since it's the return value of another function). */
-void zipEntry(unsigned char *p, zlentry *e) {
+static inline void zipEntry(unsigned char *p, zlentry *e) {
     ZIP_DECODE_PREVLEN(p, e->prevrawlensize, e->prevrawlen);
     ZIP_ENTRY_ENCODING(p + e->prevrawlensize, e->encoding);
-    ZIP_ASSERT_ENCODING(e->encoding);
     ZIP_DECODE_LENGTH(p + e->prevrawlensize, e->encoding, e->lensize, e->len);
+    assert(e->lensize != 0); /* check that encoding was valid. */
     e->headersize = e->prevrawlensize + e->lensize;
     e->p = p;
 }
@@ -603,9 +614,29 @@ void zipEntry(unsigned char *p, zlentry *e) {
  * try to access memory outside the ziplist payload.
  * Returns 1 if the entry is valid, and 0 otherwise. */
 static inline int zipEntrySafe(unsigned char* zl, size_t zlbytes, unsigned char *p, zlentry *e, int validate_prevlen) {
-#define OUT_OF_RANGE(p) ( \
-        (p) < zl + ZIPLIST_HEADER_SIZE || \
-        (p) > zl + zlbytes - ZIPLIST_END_SIZE)
+    unsigned char *zlfirst = zl + ZIPLIST_HEADER_SIZE;
+    unsigned char *zllast = zl + zlbytes - ZIPLIST_END_SIZE;
+#define OUT_OF_RANGE(p) (unlikely((p) < zlfirst || (p) > zllast))
+
+    /* If threre's no possibility for the header to reach outside the ziplist,
+     * take the fast path. (max lensize and prevrawlensize are both 5 bytes) */
+    if (p >= zlfirst && p + 10 < zllast) {
+        ZIP_DECODE_PREVLEN(p, e->prevrawlensize, e->prevrawlen);
+        ZIP_ENTRY_ENCODING(p + e->prevrawlensize, e->encoding);
+        ZIP_DECODE_LENGTH(p + e->prevrawlensize, e->encoding, e->lensize, e->len);
+        e->headersize = e->prevrawlensize + e->lensize;
+        e->p = p;
+        /* We didn't call ZIP_ASSERT_ENCODING, so we check lensize was set to 0. */
+        if (unlikely(e->lensize == 0))
+            return 0;
+        /* Make sure the entry doesn't rech outside the edge of the ziplist */
+        if (OUT_OF_RANGE(p + e->headersize + e->len))
+            return 0;
+        /* Make sure prevlen doesn't rech outside the edge of the ziplist */
+        if (validate_prevlen && OUT_OF_RANGE(p - e->prevrawlen))
+            return 0;
+        return 1;
+    }
 
     /* Make sure the pointer doesn't rech outside the edge of the ziplist */
     if (OUT_OF_RANGE(p))
@@ -619,7 +650,7 @@ static inline int zipEntrySafe(unsigned char* zl, size_t zlbytes, unsigned char
     /* Make sure encoded entry header is valid. */
     ZIP_ENTRY_ENCODING(p + e->prevrawlensize, e->encoding);
     e->lensize = zipEncodingLenSize(e->encoding);
-    if (e->lensize == ZIP_ENCODING_SIZE_INVALID)
+    if (unlikely(e->lensize == ZIP_ENCODING_SIZE_INVALID))
         return 0;
 
     /* Make sure the encoded entry header doesn't reach outside the allocation */
@@ -2175,6 +2206,54 @@ int ziplistTest(int argc, char **argv) {
         printf("Done. usec=%lld\n\n", usec()-start);
     }
 
+    /* Benchmarks */
+    {
+        zl = ziplistNew();
+        for (int i=0; i<100000; i++) {
+            char buf[4096] = "asdf";
+            zl = ziplistPush(zl, (unsigned char*)buf, 4, ZIPLIST_TAIL);
+            zl = ziplistPush(zl, (unsigned char*)buf, 40, ZIPLIST_TAIL);
+            zl = ziplistPush(zl, (unsigned char*)buf, 400, ZIPLIST_TAIL);
+            zl = ziplistPush(zl, (unsigned char*)buf, 4000, ZIPLIST_TAIL);
+            zl = ziplistPush(zl, (unsigned char*)"1", 1, ZIPLIST_TAIL);
+            zl = ziplistPush(zl, (unsigned char*)"10", 2, ZIPLIST_TAIL);
+            zl = ziplistPush(zl, (unsigned char*)"100", 3, ZIPLIST_TAIL);
+            zl = ziplistPush(zl, (unsigned char*)"1000", 4, ZIPLIST_TAIL);
+            zl = ziplistPush(zl, (unsigned char*)"10000", 5, ZIPLIST_TAIL);
+            zl = ziplistPush(zl, (unsigned char*)"100000", 6, ZIPLIST_TAIL);
+        }
+
+        printf("Benchmark ziplistFind:\n");
+        {
+            unsigned long long start = usec();
+            for (int i = 0; i < 2000; i++) {
+                unsigned char *fptr = ziplistIndex(zl, ZIPLIST_HEAD);
+                fptr = ziplistFind(zl, fptr, (unsigned char*)"nothing", 7, 1);
+            }
+            printf("%lld\n", usec()-start);
+        }
+
+        printf("Benchmark ziplistIndex:\n");
+        {
+            unsigned long long start = usec();
+            for (int i = 0; i < 2000; i++) {
+                ziplistIndex(zl, 99999);
+            }
+            printf("%lld\n", usec()-start);
+        }
+
+        printf("Benchmark ziplistValidateIntegrity:\n");
+        {
+            unsigned long long start = usec();
+            for (int i = 0; i < 2000; i++) {
+                ziplistValidateIntegrity(zl, ziplistBlobLen(zl), 1, NULL, NULL);
+            }
+            printf("%lld\n", usec()-start);
+        }
+
+        zfree(zl);
+    }
+
     printf("Stress __ziplistCascadeUpdate:\n");
     {
         char data[ZIP_BIG_PREVLEN];

From ec951cdc1555f544abe3c5468509e2148bdd4c3c Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Sun, 6 Dec 2020 13:30:29 +0000
Subject: [PATCH 0849/1098] Solaris based system rss size report. (#8138)

---
 src/config.h  |  1 +
 src/zmalloc.c | 21 +++++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/src/config.h b/src/config.h
index b8723e7ce..1d4cb5835 100644
--- a/src/config.h
+++ b/src/config.h
@@ -87,6 +87,7 @@
 #include <sys/feature_tests.h>
 #ifdef _DTRACE_VERSION
 #define HAVE_EVPORT 1
+#define HAVE_PSINFO 1
 #endif
 #endif
 
diff --git a/src/zmalloc.c b/src/zmalloc.c
index a04ecacae..2f9b4029d 100644
--- a/src/zmalloc.c
+++ b/src/zmalloc.c
@@ -433,6 +433,27 @@ size_t zmalloc_get_rss(void) {
 
     return 0L;
 }
+#elif defined(HAVE_PSINFO)
+#include <unistd.h>
+#include <sys/procfs.h>
+#include <fcntl.h>
+
+size_t zmalloc_get_rss(void) {
+    struct prpsinfo info;
+    char filename[256];
+    int fd;
+
+    snprintf(filename,256,"/proc/%d/psinfo",getpid());
+
+    if ((fd = open(filename,O_RDONLY)) == -1) return 0;
+    if (ioctl(fd, PIOCPSINFO, &info) == -1) {
+        close(fd);
+	return 0;
+    }
+
+    close(fd);
+    return info.pr_rssize;
+}
 #else
 size_t zmalloc_get_rss(void) {
     /* If we can't get the RSS in an OS-specific way for this system just

From bccbc5509a1fe3ebee48904bd2418d9e665d2a5a Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 7 Dec 2020 14:24:05 +0200
Subject: [PATCH 0850/1098] Add CLIENT INFO and CLIENT LIST [id]. (#8113)

* Add CLIENT INFO subcommand.

The output is identical to CLIENT LIST but provides a single line for
the current client only.

* Add CLIENT LIST ID [id...].

Co-authored-by: Itamar Haber <itamar@redislabs.com>
---
 src/networking.c             | 31 +++++++++++++++++++++++++++++--
 src/redis-cli.c              |  3 ++-
 tests/unit/introspection.tcl | 10 ++++++++++
 3 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 82b6428ac..f1665825c 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2331,6 +2331,7 @@ void clientCommand(client *c) {
     if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
         const char *help[] = {
 "ID                     -- Return the ID of the current connection.",
+"INFO                   -- Return information about the current client connection.",
 "GETNAME                -- Return the name of the current connection.",
 "KILL <ip:port>         -- Kill connection made from <ip:port>.",
 "KILL <option> <value> [option value ...] -- Kill connections. Options are:",
@@ -2341,6 +2342,7 @@ void clientCommand(client *c) {
 "     SKIPME (yes|no)   -- Skip killing current connection (default: yes).",
 "LIST [options ...]     -- Return information about client connections. Options:",
 "     TYPE (normal|master|replica|pubsub) -- Return clients of specified type.",
+"     ID id [id ...]                      -- Return clients of specified IDs only.",
 "PAUSE <timeout>        -- Suspend all Redis clients for <timout> milliseconds.",
 "REPLY (on|off|skip)    -- Control the replies sent to the current connection.",
 "SETNAME <name>         -- Assign the name <name> to the current connection.",
@@ -2354,21 +2356,46 @@ NULL
     } else if (!strcasecmp(c->argv[1]->ptr,"id") && c->argc == 2) {
         /* CLIENT ID */
         addReplyLongLong(c,c->id);
+    } else if (!strcasecmp(c->argv[1]->ptr,"info") && c->argc == 2) {
+        /* CLIENT INFO */
+        sds o = catClientInfoString(sdsempty(), c);
+        o = sdscatlen(o,"\n",1);
+        addReplyVerbatim(c,o,sdslen(o),"txt");
+        sdsfree(o);
     } else if (!strcasecmp(c->argv[1]->ptr,"list")) {
         /* CLIENT LIST */
         int type = -1;
+        sds o = NULL;
         if (c->argc == 4 && !strcasecmp(c->argv[2]->ptr,"type")) {
             type = getClientTypeByName(c->argv[3]->ptr);
             if (type == -1) {
                 addReplyErrorFormat(c,"Unknown client type '%s'",
                     (char*) c->argv[3]->ptr);
                 return;
-             }
+            }
+        } else if (c->argc > 3 && !strcasecmp(c->argv[2]->ptr,"id")) {
+            int j;
+            o = sdsempty();
+            for (j = 3; j < c->argc; j++) {
+                long long cid;
+                if (getLongLongFromObjectOrReply(c, c->argv[j], &cid,
+                            "Invalid client ID")) {
+                    sdsfree(o);
+                    return;
+                }
+                client *cl = lookupClientByID(cid);
+                if (cl) {
+                    o = catClientInfoString(o, cl);
+                    o = sdscatlen(o, "\n", 1);
+                }
+            }
         } else if (c->argc != 2) {
             addReply(c,shared.syntaxerr);
             return;
         }
-        sds o = getAllClientsInfoString(type);
+
+        if (!o)
+            o = getAllClientsInfoString(type);
         addReplyVerbatim(c,o,sdslen(o),"txt");
         sdsfree(o);
     } else if (!strcasecmp(c->argv[1]->ptr,"reply") && c->argc == 3) {
diff --git a/src/redis-cli.c b/src/redis-cli.c
index e140435d8..b97eabb73 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -1304,7 +1304,8 @@ static int cliSendCommand(int argc, char **argv, long repeat) {
                       (!strcasecmp(argv[1],"nodes") ||
                        !strcasecmp(argv[1],"info"))) ||
         (argc >= 2 && !strcasecmp(command,"client") &&
-                       !strcasecmp(argv[1],"list")) ||
+                       (!strcasecmp(argv[1],"list") ||
+                        !strcasecmp(argv[1],"info"))) ||
         (argc == 3 && !strcasecmp(command,"latency") &&
                        !strcasecmp(argv[1],"graph")) ||
         (argc == 2 && !strcasecmp(command,"latency") &&
diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl
index 86da17657..a250762f2 100644
--- a/tests/unit/introspection.tcl
+++ b/tests/unit/introspection.tcl
@@ -3,6 +3,16 @@ start_server {tags {"introspection"}} {
         r client list
     } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=26 qbuf-free=* argv-mem=* obl=0 oll=0 omem=0 tot-mem=* events=r cmd=client*}
 
+    test {CLIENT LIST with IDs} {
+        set myid [r client id]
+        set cl [split [r client list id $myid] "\r\n"]
+        assert_match "id=$myid*" [lindex $cl 0]
+    }
+
+    test {CLIENT INFO} {
+        r client info
+    } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=26 qbuf-free=* argv-mem=* obl=0 oll=0 omem=0 tot-mem=* events=r cmd=client*}
+
     test {MONITOR can log executed commands} {
         set rd [redis_deferring_client]
         $rd monitor

From b5e99bd0645557b204703ec0e54aa2051fc122ee Mon Sep 17 00:00:00 2001
From: filipe oliveira <filipecosta.90@gmail.com>
Date: Mon, 7 Dec 2020 12:30:12 +0000
Subject: [PATCH 0851/1098] Enable configuring OpenSSL using the standard
 openssl.cnf (#8143)

---
 src/tls.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/tls.c b/src/tls.c
index 25580a4f1..af16d2a41 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -34,6 +34,7 @@
 
 #ifdef USE_OPENSSL
 
+#include <openssl/conf.h>
 #include <openssl/ssl.h>
 #include <openssl/err.h>
 #include <openssl/rand.h>
@@ -136,6 +137,17 @@ static void initCryptoLocks(void) {
 #endif /* USE_CRYPTO_LOCKS */
 
 void tlsInit(void) {
+    /* Enable configuring OpenSSL using the standard openssl.cnf
+     * OPENSSL_config()/OPENSSL_init_crypto() should be the first 
+     * call to the OpenSSL* library.
+     *  - OPENSSL_config() should be used for OpenSSL versions < 1.1.0
+     *  - OPENSSL_init_crypto() should be used for OpenSSL versions >= 1.1.0
+     */
+    #if OPENSSL_VERSION_NUMBER < 0x10100000L
+    OPENSSL_config(NULL);
+    #else
+    OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL);
+    #endif
     ERR_load_crypto_strings();
     SSL_load_error_strings();
     SSL_library_init();

From 6bb5503524fbc15c4d6a5ac5eb08aedee18d3189 Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Mon, 7 Dec 2020 20:31:35 +0100
Subject: [PATCH 0852/1098] More efficient self-XCLAIM (#8098)

when the same consumer re-claim an entry that it already has, there's
no need to remove-and-insert if it's the same rax.
we do need to update the idle time though.
this commit only improves efficiency (doesn't change behavior).
---
 src/t_stream.c                     | 21 +++++++++++---------
 tests/unit/type/stream-cgroups.tcl | 32 ++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 9 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 320c5a2fe..684328de2 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -2601,15 +2601,15 @@ void xclaimCommand(client *c) {
                 mstime_t this_idle = now - nack->delivery_time;
                 if (this_idle < minidle) continue;
             }
-            /* Remove the entry from the old consumer.
-             * Note that nack->consumer is NULL if we created the
-             * NACK above because of the FORCE option. */
-            if (nack->consumer)
-                raxRemove(nack->consumer->pel,buf,sizeof(buf),NULL);
-            /* Update the consumer and idle time. */
             if (consumer == NULL)
                 consumer = streamLookupConsumer(group,c->argv[3]->ptr,SLC_NONE,NULL);
-            nack->consumer = consumer;
+            if (nack->consumer != consumer) {
+                /* Remove the entry from the old consumer.
+                 * Note that nack->consumer is NULL if we created the
+                 * NACK above because of the FORCE option. */
+                if (nack->consumer)
+                    raxRemove(nack->consumer->pel,buf,sizeof(buf),NULL);
+            }
             nack->delivery_time = deliverytime;
             /* Set the delivery attempts counter if given, otherwise
              * autoincrement unless JUSTID option provided */
@@ -2618,8 +2618,11 @@ void xclaimCommand(client *c) {
             } else if (!justid) {
                 nack->delivery_count++;
             }
-            /* Add the entry in the new consumer local PEL. */
-            raxInsert(consumer->pel,buf,sizeof(buf),nack,NULL);
+            if (nack->consumer != consumer) {
+                /* Add the entry in the new consumer local PEL. */
+                raxInsert(consumer->pel,buf,sizeof(buf),nack,NULL);
+                nack->consumer = consumer;
+            }
             /* Send the reply for this entry. */
             if (justid) {
                 addReplyStreamID(c,&id);
diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl
index 41e2ab979..1d1a68a35 100644
--- a/tests/unit/type/stream-cgroups.tcl
+++ b/tests/unit/type/stream-cgroups.tcl
@@ -235,6 +235,12 @@ start_server {
         ]
         assert {[llength [lindex $reply 0 1 0 1]] == 2}
         assert {[lindex $reply 0 1 0 1] eq {a 1}}
+
+        # make sure the entry is present in both the gorup, and the right consumer
+        assert {[llength [r XPENDING mystream mygroup - + 10]] == 1}
+        assert {[llength [r XPENDING mystream mygroup - + 10 client1]] == 1}
+        assert {[llength [r XPENDING mystream mygroup - + 10 client2]] == 0}
+
         r debug sleep 0.2
         set reply [
             r XCLAIM mystream mygroup client2 10 $id1
@@ -242,6 +248,11 @@ start_server {
         assert {[llength [lindex $reply 0 1]] == 2}
         assert {[lindex $reply 0 1] eq {a 1}}
 
+        # make sure the entry is present in both the gorup, and the right consumer
+        assert {[llength [r XPENDING mystream mygroup - + 10]] == 1}
+        assert {[llength [r XPENDING mystream mygroup - + 10 client1]] == 0}
+        assert {[llength [r XPENDING mystream mygroup - + 10 client2]] == 1}
+
         # Client 1 reads another 2 items from stream
         r XREADGROUP GROUP mygroup client1 count 2 STREAMS mystream >
         r debug sleep 0.2
@@ -311,6 +322,27 @@ start_server {
         assert {[lindex $reply 0 3] == 2}
     }
 
+    test {XCLAIM same consumer} {
+        # Add 3 items into the stream, and create a consumer group
+        r del mystream
+        set id1 [r XADD mystream * a 1]
+        set id2 [r XADD mystream * b 2]
+        set id3 [r XADD mystream * c 3]
+        r XGROUP CREATE mystream mygroup 0
+
+        set reply [r XREADGROUP GROUP mygroup client1 count 1 STREAMS mystream >]
+        assert {[llength [lindex $reply 0 1 0 1]] == 2}
+        assert {[lindex $reply 0 1 0 1] eq {a 1}}
+        r debug sleep 0.2
+        # re-claim with the same consumer that already has it
+        assert {[llength [r XCLAIM mystream mygroup client1 10 $id1]] == 1}
+
+        # make sure the entry is still in the PEL
+        set reply [r XPENDING mystream mygroup - + 10]
+        assert {[llength $reply] == 1}
+        assert {[lindex $reply 0 1] eq {client1}}
+    }
+
     test {XINFO FULL output} {
         r del x
         r XADD x 100 a 1

From 09b7f6c2a10d5c50bad64953f9a31e04f81c0444 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Tue, 8 Dec 2020 04:15:39 -0500
Subject: [PATCH 0853/1098] use getPositiveLongFromObjectOrReply for positive
 check in spop (#8146)

This is a cleanup commit that doesn't fix any bug, however it does change the error response text in case the user provided a negative number.
---
 src/t_set.c | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/src/t_set.c b/src/t_set.c
index b2da14cd5..7c71dfc2f 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -457,13 +457,8 @@ void spopWithCountCommand(client *c) {
     robj *set;
 
     /* Get the count argument */
-    if (getLongFromObjectOrReply(c,c->argv[2],&l,NULL) != C_OK) return;
-    if (l >= 0) {
-        count = (unsigned long) l;
-    } else {
-        addReply(c,shared.outofrangeerr);
-        return;
-    }
+    if (getPositiveLongFromObjectOrReply(c,c->argv[2],&l,NULL) != C_OK) return;
+    count = (unsigned long) l;
 
     /* Make sure a key with the name inputted exists, and that it's type is
      * indeed a set. Otherwise, return nil */

From ec02c761aa16175eb599b2fc9d0b2792ffe8a66c Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 8 Dec 2020 11:27:30 +0200
Subject: [PATCH 0854/1098] Fix setproctitle related crashes. (#8150)

Makes spt_init more careful with assumptions about what memory regions
may be overwritten. It will now only consider a contiguous block of argv
and envp elements and mind any gaps.
---
 src/setproctitle.c | 43 +++++++++++++++++++++++++++++++++++++++----
 1 file changed, 39 insertions(+), 4 deletions(-)

diff --git a/src/setproctitle.c b/src/setproctitle.c
index 707383673..1c91570eb 100644
--- a/src/setproctitle.c
+++ b/src/setproctitle.c
@@ -178,7 +178,16 @@ static int spt_copyargs(int argc, char *argv[]) {
 	return 0;
 } /* spt_copyargs() */
 
-
+/* Initialize and populate SPT to allow a future setproctitle()
+ * call.
+ *
+ * As setproctitle() basically needs to overwrite argv[0], we're
+ * trying to determine what is the largest contiguous block
+ * starting at argv[0] we can use for this purpose.
+ *
+ * As this range will overwrite some or all of the argv and environ
+ * strings, a deep copy of these two arrays is performed.
+ */
 void spt_init(int argc, char *argv[]) {
         char **envp = environ;
 	char *base, *end, *nul, *tmp;
@@ -187,24 +196,39 @@ void spt_init(int argc, char *argv[]) {
 	if (!(base = argv[0]))
 		return;
 
+	/* We start with end pointing at the end of argv[0] */
 	nul = &base[strlen(base)];
 	end = nul + 1;
 
+	/* Attempt to extend end as far as we can, while making sure
+	 * that the range between base and end is only allocated to
+	 * argv, or anything that immediately follows argv (presumably
+	 * envp).
+	 */
 	for (i = 0; i < argc || (i >= argc && argv[i]); i++) {
 		if (!argv[i] || argv[i] < end)
 			continue;
 
-		end = argv[i] + strlen(argv[i]) + 1;
+		if (end >= argv[i] && end <= argv[i] + strlen(argv[i]))
+			end = argv[i] + strlen(argv[i]) + 1;
 	}
 
+	/* In case the envp array was not an immediate extension to argv,
+	 * scan it explicitly.
+	 */
 	for (i = 0; envp[i]; i++) {
 		if (envp[i] < end)
 			continue;
 
-		end = envp[i] + strlen(envp[i]) + 1;
+		if (end >= envp[i] && end <= envp[i] + strlen(envp[i]))
+			end = envp[i] + strlen(envp[i]) + 1;
 	}
 	envc = i;
 
+	/* We're going to deep copy argv[], but argv[0] will still point to
+	 * the old memory for the purpose of updating the title so we need
+	 * to keep the original value elsewhere.
+	 */
 	if (!(SPT.arg0 = strdup(argv[0])))
 		goto syerr;
 
@@ -225,7 +249,7 @@ void spt_init(int argc, char *argv[]) {
 	setprogname(tmp);
 #endif
 
-
+    /* Now make a full deep copy of the environment and argv[] */
 	if ((error = spt_copyenv(envc, envp)))
 		goto error;
 
@@ -294,3 +318,14 @@ error:
 
 #endif /* __linux || __APPLE__ */
 #endif /* !HAVE_SETPROCTITLE */
+
+#ifdef SETPROCTITLE_TEST_MAIN
+int main(int argc, char *argv[]) {
+	spt_init(argc, argv);
+
+	printf("SPT.arg0: [%p] '%s'\n", SPT.arg0, SPT.arg0);
+	printf("SPT.base: [%p] '%s'\n", SPT.base, SPT.base);
+	printf("SPT.end: [%p] (%d bytes after base)'\n", SPT.end, (int) (SPT.end - SPT.base));
+	return 0;
+}
+#endif

From 37f45d9e5677e75bc396af5e3d8864125f026969 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Tue, 8 Dec 2020 11:43:00 +0200
Subject: [PATCH 0855/1098] Adds exclusive range query intervals to XPENDING
 (#8130)

---
 src/t_stream.c                     | 19 ++++++++++++++++---
 tests/unit/type/stream-cgroups.tcl | 14 ++++++++++++++
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 684328de2..d61fb3eab 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -2231,8 +2231,9 @@ void xpendingCommand(client *c) {
     robj *groupname = c->argv[2];
     robj *consumername = NULL;
     streamID startid, endid;
-    long long count;
+    long long count = 0;
     long long minidle = 0;
+    int startex = 0, endex = 0;
 
     /* Start and stop, and the consumer, can be omitted. Also the IDLE modifier. */
     if (c->argc != 3 && (c->argc < 6 || c->argc > 9)) {
@@ -2256,13 +2257,25 @@ void xpendingCommand(client *c) {
             /* Search for rest of arguments after 'IDLE <idle>' */
             startidx += 2;
         }
+
+        /* count argument. */
         if (getLongLongFromObjectOrReply(c,c->argv[startidx+2],&count,NULL) == C_ERR)
             return;
         if (count < 0) count = 0;
-        if (streamParseIDOrReply(c,c->argv[startidx],&startid,0) == C_ERR)
+
+        /* start and end arguments. */
+        if (streamParseIntervalIDOrReply(c,c->argv[startidx],&startid,&startex,0) != C_OK)
             return;
-        if (streamParseIDOrReply(c,c->argv[startidx+1],&endid,UINT64_MAX) == C_ERR)
+        if (startex && streamIncrID(&startid) != C_OK) {
+            addReplyError(c,"invalid start ID for the interval");
             return;
+        }
+        if (streamParseIntervalIDOrReply(c,c->argv[startidx+1],&endid,&endex,UINT64_MAX) != C_OK)
+            return;
+        if (endex && streamDecrID(&endid) != C_OK) {
+            addReplyError(c,"invalid end ID for the interval");
+            return;
+        }
 
         if (startidx+3 < c->argc) {
             /* 'consumer' was provided */
diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl
index 1d1a68a35..91dc2245e 100644
--- a/tests/unit/type/stream-cgroups.tcl
+++ b/tests/unit/type/stream-cgroups.tcl
@@ -87,6 +87,20 @@ start_server {
         assert {[llength $pending] == 4}
     }
 
+    test {XPENDING with exclusive range intervals works as expected} {
+        set pending [r XPENDING mystream mygroup - + 10]
+        assert {[llength $pending] == 4}
+        set startid [lindex [lindex $pending 0] 0]
+        set endid [lindex [lindex $pending 3] 0]
+        set expending [r XPENDING mystream mygroup ($startid ($endid 10]
+        assert {[llength $expending] == 2}
+        for {set j 0} {$j < 2} {incr j} {
+            set itemid [lindex [lindex $expending $j] 0]
+            assert {$itemid ne $startid}
+            assert {$itemid ne $endid}
+        }
+    }
+
     test {XACK is able to remove items from the client/group PEL} {
         set pending [r XPENDING mystream mygroup - + 10 client-1]
         set id1 [lindex $pending 0 0]

From 00db1b5579d1b633a6f6ee9e99e4ee2a03fcb6dc Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 8 Dec 2020 16:22:16 +0200
Subject: [PATCH 0856/1098] Fix failing macOS tests due to wc differences.
 (#8161)

---
 tests/support/util.tcl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index 040a7f7d9..0807ebe45 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -109,7 +109,7 @@ proc wait_done_loading r {
 
 # count current log lines in server's stdout
 proc count_log_lines {srv_idx} {
-    set _ [exec wc -l < [srv $srv_idx stdout]]
+    set _ [string trim [exec wc -l < [srv $srv_idx stdout]]]
 }
 
 # returns the number of times a line with that pattern appears in a file
@@ -117,7 +117,7 @@ proc count_message_lines {file pattern} {
     set res 0
     # exec fails when grep exists with status other than 0 (when the patter wasn't found)
     catch {
-        set res [exec grep $pattern $file 2> /dev/null | wc -l]
+        set res [string trim [exec grep $pattern $file 2> /dev/null | wc -l]]
     }
     return $res
 }

From 1acc315cea882ed7b3c257179bb96eccf6eb676d Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuancode@163.com>
Date: Tue, 8 Dec 2020 22:26:38 +0800
Subject: [PATCH 0857/1098] Minor improvements for list-2 test (#8156)

had some unused variables.
now some are used to assert that they match, others were useless.
---
 tests/unit/type/list-2.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/type/list-2.tcl b/tests/unit/type/list-2.tcl
index 4c7d6d91c..5874a9028 100644
--- a/tests/unit/type/list-2.tcl
+++ b/tests/unit/type/list-2.tcl
@@ -29,11 +29,11 @@ start_server {
                     set max [expr {$min+int(rand()*$startlen)}]
                     set before_len [llength $mylist]
                     set before_len_r [r llen mylist]
+                    assert_equal $before_len $before_len_r
                     set mylist [lrange $mylist $min $max]
                     r ltrim mylist $min $max
                     assert_equal $mylist [r lrange mylist 0 -1] "failed trim"
 
-                    set starting [r llen mylist]
                     for {set j [r llen mylist]} {$j < $startlen} {incr j} {
                         set str [randomInt 9223372036854775807]
                         r rpush mylist $str

From a102b21d178453247078b355a47762bfc121ac20 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 8 Dec 2020 16:33:09 +0200
Subject: [PATCH 0858/1098] Improve stability of new CSC eviction test (#8160)

c4fdf09c0 added a test that now fails with valgrind
it fails for two resons:
1) the test samples the used memory and then limits the maxmemory to
   that value, but it turns out this is not atomic and on slow machines
   the background cron process that clean out old query buffers reduces
   the memory so that the setting doesn't cause eviction.
2) the dbsize was tested late, after reading some invalidation messages
   by that time more and more keys got evicted, partially draining the
   db. this is not the focus of this fix (still a known limitation)
---
 tests/unit/maxmemory.tcl | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/tests/unit/maxmemory.tcl b/tests/unit/maxmemory.tcl
index d4ad3ff03..ed5657860 100644
--- a/tests/unit/maxmemory.tcl
+++ b/tests/unit/maxmemory.tcl
@@ -291,6 +291,11 @@ start_server {tags {"maxmemory"}} {
             }
         }
 
+        # we need to wait one second for the client querybuf excess memory to be
+        # trimmed by cron, otherwise the INFO used_memory and CONFIG maxmemory
+        # below (on slow machines) won't be "atomic" and won't trigger eviction.
+        after 1100
+
         # set the memory limit which will cause a few keys to be evicted
         # we need to make sure to evict keynames of a total size of more than
         # 16kb since the (PROTO_REPLY_CHUNK_BYTES), only after that the
@@ -302,13 +307,20 @@ start_server {tags {"maxmemory"}} {
         # make sure some eviction happened
         set evicted [s evicted_keys]
         if {$::verbose} { puts "evicted: $evicted" }
+
+        # make sure we didn't drain the database
+        assert_range [r dbsize] 200 300
+
         assert_range $evicted 10 50
         foreach rd $clients {
             $rd read ;# make sure we have some invalidation message waiting
             $rd close
         }
 
-        # make sure we didn't drain the database
-        assert_range [r dbsize] 200 300
+        # eviction continues (known problem described in #8069)
+        # for now this test only make sures the eviction loop itself doesn't
+        # have feedback loop
+        set evicted [s evicted_keys]
+        if {$::verbose} { puts "evicted: $evicted" }
     }
 }

From 48efc25f749c3620f9245786582ac76cb40e9bf4 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 8 Dec 2020 16:41:20 +0200
Subject: [PATCH 0859/1098] Handle output buffer limits for Module blocked
 clients (#8141)

Module blocked clients cache the response in a temporary client,
the reply list in this client would be affected by the recent fix
in #7202, but when the reply is later copied into the real client,
it would have bypassed all the checks for output buffer limit, which
would have resulted in both: responding with a partial response to
the client, and also not disconnecting it at all.
---
 src/networking.c                       | 33 +++++++++-
 tests/modules/blockedclient.c          | 85 ++++++++++++++++++++++++++
 tests/unit/moduleapi/blockedclient.tcl | 18 ++++++
 3 files changed, 135 insertions(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index f1665825c..df30206a3 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -270,6 +270,9 @@ int prepareClientToWrite(client *c) {
  * Low level functions to add more data to output buffers.
  * -------------------------------------------------------------------------- */
 
+/* Attempts to add the reply to the static buffer in the client struct.
+ * Returns C_ERR if the buffer is full, or the reply list is not empty,
+ * in which case the reply must be added to the reply list. */
 int _addReplyToBuffer(client *c, const char *s, size_t len) {
     size_t available = sizeof(c->buf)-c->bufpos;
 
@@ -287,6 +290,8 @@ int _addReplyToBuffer(client *c, const char *s, size_t len) {
     return C_OK;
 }
 
+/* Adds the reply to the reply linked list.
+ * Note: some edits to this function need to be relayed to AddReplyFromClient. */
 void _addReplyProtoToList(client *c, const char *s, size_t len) {
     if (c->flags & CLIENT_CLOSE_AFTER_REPLY) return;
 
@@ -848,14 +853,40 @@ void addReplySubcommandSyntaxError(client *c) {
 /* Append 'src' client output buffers into 'dst' client output buffers. 
  * This function clears the output buffers of 'src' */
 void AddReplyFromClient(client *dst, client *src) {
+    /* If the source client contains a partial response due to client output
+     * buffer limits, propagate that to the dest rather than copy a partial
+     * reply. We don't wanna run the risk of copying partial response in case
+     * for some reason the output limits don't reach the same decision (maybe
+     * they changed) */
+    if (src->flags & CLIENT_CLOSE_ASAP) {
+        sds client = catClientInfoString(sdsempty(),dst);
+        freeClientAsync(dst);
+        serverLog(LL_WARNING,"Client %s scheduled to be closed ASAP for overcoming of output buffer limits.", client);
+        sdsfree(client);
+        return;
+    }
+
+    /* First add the static buffer (either into the static buffer or reply list) */
+    addReplyProto(dst,src->buf, src->bufpos);
+
+    /* We need to check with prepareClientToWrite again (after addReplyProto)
+     * since addReplyProto may have changed something (like CLIENT_CLOSE_ASAP) */
     if (prepareClientToWrite(dst) != C_OK)
         return;
-    addReplyProto(dst,src->buf, src->bufpos);
+
+    /* We're bypassing _addReplyProtoToList, so we need to add the pre/post
+     * checks in it. */
+    if (dst->flags & CLIENT_CLOSE_AFTER_REPLY) return;
+
+    /* Concatenate the reply list into the dest */
     if (listLength(src->reply))
         listJoin(dst->reply,src->reply);
     dst->reply_bytes += src->reply_bytes;
     src->reply_bytes = 0;
     src->bufpos = 0;
+
+    /* Check output buffer limits */
+    asyncCloseClientOnOutputBufferLimitReached(dst);
 }
 
 /* Copy 'src' client output buffers into 'dst' client output buffers.
diff --git a/tests/modules/blockedclient.c b/tests/modules/blockedclient.c
index cfa8319d8..c3b0d5eb4 100644
--- a/tests/modules/blockedclient.c
+++ b/tests/modules/blockedclient.c
@@ -85,6 +85,88 @@ int acquire_gil(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     return REDISMODULE_OK;
 }
 
+typedef struct {
+    RedisModuleString **argv;
+    int argc;
+    RedisModuleBlockedClient *bc;
+} bg_call_data;
+
+void *bg_call_worker(void *arg) {
+    bg_call_data *bg = arg;
+
+    // Get Redis module context
+    RedisModuleCtx *ctx = RedisModule_GetThreadSafeContext(bg->bc);
+
+    // Acquire GIL
+    RedisModule_ThreadSafeContextLock(ctx);
+
+    // Call the command
+    const char* cmd = RedisModule_StringPtrLen(bg->argv[1], NULL);
+    RedisModuleCallReply* rep = RedisModule_Call(ctx, cmd, "v", bg->argv + 2, bg->argc - 2);
+
+    // Release GIL
+    RedisModule_ThreadSafeContextUnlock(ctx);
+
+    // Reply to client
+    if (!rep) {
+        RedisModule_ReplyWithError(ctx, "NULL reply returned");
+    } else {
+        RedisModule_ReplyWithCallReply(ctx, rep);
+        RedisModule_FreeCallReply(rep);
+    }
+
+    // Unblock client
+    RedisModule_UnblockClient(bg->bc, NULL);
+
+    /* Free the arguments */
+    for (int i=0; i<bg->argc; i++)
+        RedisModule_FreeString(ctx, bg->argv[i]);
+    RedisModule_Free(bg->argv);
+    RedisModule_Free(bg);
+
+    // Free the Redis module context
+    RedisModule_FreeThreadSafeContext(ctx);
+
+    return NULL;
+}
+
+int do_bg_rm_call(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    UNUSED(argv);
+    UNUSED(argc);
+
+    /* Make sure we're not trying to block a client when we shouldn't */
+    int flags = RedisModule_GetContextFlags(ctx);
+    int allFlags = RedisModule_GetContextFlagsAll();
+    if ((allFlags & REDISMODULE_CTX_FLAGS_MULTI) &&
+        (flags & REDISMODULE_CTX_FLAGS_MULTI)) {
+        RedisModule_ReplyWithSimpleString(ctx, "Blocked client is not supported inside multi");
+        return REDISMODULE_OK;
+    }
+    if ((allFlags & REDISMODULE_CTX_FLAGS_DENY_BLOCKING) &&
+        (flags & REDISMODULE_CTX_FLAGS_DENY_BLOCKING)) {
+        RedisModule_ReplyWithSimpleString(ctx, "Blocked client is not allowed");
+        return REDISMODULE_OK;
+    }
+
+    /* Make a copy of the arguments and pass them to the thread. */
+    bg_call_data *bg = RedisModule_Alloc(sizeof(bg_call_data));
+    bg->argv = RedisModule_Alloc(sizeof(RedisModuleString*)*argc);
+    bg->argc = argc;
+    for (int i=0; i<argc; i++)
+        bg->argv[i] = RedisModule_HoldString(ctx, argv[i]);
+
+    /* Block the client */
+    bg->bc = RedisModule_BlockClient(ctx, NULL, NULL, NULL, 0);
+
+    /* Start a thread to handle the request */
+    pthread_t tid;
+    int res = pthread_create(&tid, NULL, bg_call_worker, bg);
+    assert(res == 0);
+
+    return REDISMODULE_OK;
+}
+
 int do_rm_call(RedisModuleCtx *ctx, RedisModuleString **argv, int argc){
     UNUSED(argv);
     UNUSED(argc);
@@ -120,5 +202,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     if (RedisModule_CreateCommand(ctx, "do_rm_call", do_rm_call, "", 0, 0, 0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
 
+    if (RedisModule_CreateCommand(ctx, "do_bg_rm_call", do_bg_rm_call, "", 0, 0, 0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
     return REDISMODULE_OK;
 }
diff --git a/tests/unit/moduleapi/blockedclient.tcl b/tests/unit/moduleapi/blockedclient.tcl
index 60039659e..0598a84d6 100644
--- a/tests/unit/moduleapi/blockedclient.tcl
+++ b/tests/unit/moduleapi/blockedclient.tcl
@@ -72,4 +72,22 @@ start_server {tags {"modules"}} {
         } e
         set e
     } {*ERR*DENY BLOCKING*}
+
+    test {RM_Call from blocked client} {
+        r hset hash foo bar
+        r do_bg_rm_call hgetall hash
+    } {foo bar}
+
+    test {blocked client reaches client output buffer limit} {
+        r hset hash big [string repeat x 50000]
+        r hset hash bada [string repeat x 50000]
+        r hset hash boom [string repeat x 50000]
+        r config set client-output-buffer-limit {normal 100000 0 0}
+        r client setname myclient
+        catch {r do_bg_rm_call hgetall hash} e
+        assert_match "*I/O error*" $e
+        reconnect
+        set clients [r client list]
+        assert_no_match "*name=myclient*" $clients
+    }
 }

From 6a3c4ac50a6666feb09de1561b1c483dc2a1891e Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 9 Dec 2020 17:05:05 +0200
Subject: [PATCH 0860/1098] zset full dump sanitization bug (dup score instead
 of field) (#8167)

---
 src/t_zset.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_zset.c b/src/t_zset.c
index d4ffe1318..a9564828a 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -1612,7 +1612,7 @@ static int _zsetZiplistValidateIntegrity(unsigned char *p, void *userdata) {
     } *data = userdata;
 
     /* Even records are field names, add to dict and check that's not a dup */
-    if (((data->count) & 1) == 1) {
+    if (((data->count) & 1) == 0) {
         unsigned char *str;
         unsigned int slen;
         long long vll;

From cf3d79d4c10c5ca4e485ce29c288825d582ca067 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 9 Dec 2020 20:20:55 +0200
Subject: [PATCH 0861/1098] Fix double close on IPv6 setup error. (#8168)

Fixes #8165.
---
 src/anet.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/anet.c b/src/anet.c
index d46ce5398..7a0a1b1ed 100644
--- a/src/anet.c
+++ b/src/anet.c
@@ -470,7 +470,6 @@ static int anetV6Only(char *err, int s) {
     int yes = 1;
     if (setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,&yes,sizeof(yes)) == -1) {
         anetSetError(err, "setsockopt: %s", strerror(errno));
-        close(s);
         return ANET_ERR;
     }
     return ANET_OK;

From 4e064fbab4d310b508593b46ed6ce539aea7aa25 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 9 Dec 2020 20:22:45 +0200
Subject: [PATCH 0862/1098] Add module data-type support for COPY. (#8112)

This adds a copy callback for module data types, in order to make
modules compatible with the new COPY command.

The callback is optional and COPY will fail for keys with data types
that do not implement it.
---
 src/db.c                          |  8 +++----
 src/module.c                      | 30 ++++++++++++++++++++++---
 src/redismodule.h                 |  2 ++
 src/server.h                      |  3 +++
 tests/modules/datatype.c          | 37 ++++++++++++++++++++++++++++---
 tests/unit/moduleapi/datatype.tcl | 14 ++++++++++++
 6 files changed, 84 insertions(+), 10 deletions(-)

diff --git a/src/db.c b/src/db.c
index 12e1c2bbe..41ee53601 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1242,12 +1242,12 @@ void copyCommand(client *c) {
         case OBJ_HASH: newobj = hashTypeDup(o); break;
         case OBJ_STREAM: newobj = streamDup(o); break;
         case OBJ_MODULE:
-            addReplyError(c, "Copying module type object is not supported");
-            return;
-        default: {
+            newobj = moduleTypeDupOrReply(c, key, newkey, o);
+            if (!newobj) return;
+            break;
+        default:
             addReplyError(c, "unknown type object");
             return;
-        };
     }
 
     if (delete) {
diff --git a/src/module.c b/src/module.c
index b5921ed55..be0f939f2 100644
--- a/src/module.c
+++ b/src/module.c
@@ -3639,6 +3639,24 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) {
     }
 }
 
+/* Create a copy of a module type value using the copy callback. If failed
+ * or not supported, produce an error reply and return NULL.
+ */
+robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, robj *value) {
+    moduleValue *mv = value->ptr;
+    moduleType *mt = mv->type;
+    if (!mt->copy) {
+        addReplyError(c, "not supported for this module key");
+        return NULL;
+    }
+    void *newval = mt->copy(fromkey, tokey, mv->value);
+    if (!newval) {
+        addReplyError(c, "module key failed to copy");
+        return NULL;
+    }
+    return createModuleObject(mt, newval);
+}
+
 /* Register a new data type exported by the module. The parameters are the
  * following. Please for in depth documentation check the modules API
  * documentation, especially https://redis.io/topics/modules-native-types.
@@ -3678,6 +3696,7 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) {
  *          .aux_save = myType_AuxRDBSaveCallBack,
  *          .free_effort = myType_FreeEffortCallBack
  *          .unlink = myType_UnlinkCallBack
+ *          .copy = myType_CopyCallback
  *      }
  *
  * * **rdb_load**: A callback function pointer that loads data from RDB files.
@@ -3697,10 +3716,13 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) {
  *   been removed from the DB by redis, and may soon be freed by a background thread. Note that 
  *   it won't be called on FLUSHALL/FLUSHDB (both sync and async), and the module can use the 
  *   RedisModuleEvent_FlushDB to hook into that.
+ * * **copy**: A callback function pointer that is used to make a copy of the specified key.
+ *   The module is expected to perform a deep copy of the specified value and return it.
+ *   In addition, hints about the names of the source and destination keys is provided.
+ *   A NULL return value is considered an error and the copy operation fails.
+ *   Note: if the target key exists and is being overwritten, the copy callback will be
+ *   called first, followed by a free callback to the value that is being replaced.
  * 
- * The **digest** and **mem_usage** methods should currently be omitted since
- * they are not yet implemented inside the Redis modules core.
- *
  * Note: the module name "AAAAAAAAA" is reserved and produces an error, it
  * happens to be pretty lame as well.
  *
@@ -3743,6 +3765,7 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver,
         struct {
             moduleTypeFreeEffortFunc free_effort;
             moduleTypeUnlinkFunc unlink;
+            moduleTypeCopyFunc copy;
         } v3;
     } *tms = (struct typemethods*) typemethods_ptr;
 
@@ -3763,6 +3786,7 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver,
     if (tms->version >= 3) {
         mt->free_effort = tms->v3.free_effort;
         mt->unlink = tms->v3.unlink;
+        mt->copy = tms->v3.copy;
     }
     memcpy(mt->name,name,sizeof(mt->name));
     listAddNodeTail(ctx->module->types,mt);
diff --git a/src/redismodule.h b/src/redismodule.h
index 59f568c6f..e4e8b2b3a 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -494,6 +494,7 @@ typedef void (*RedisModuleTypeDigestFunc)(RedisModuleDigest *digest, void *value
 typedef void (*RedisModuleTypeFreeFunc)(void *value);
 typedef size_t (*RedisModuleTypeFreeEffortFunc)(RedisModuleString *key, const void *value);
 typedef void (*RedisModuleTypeUnlinkFunc)(RedisModuleString *key, const void *value);
+typedef void *(*RedisModuleTypeCopyFunc)(RedisModuleString *fromkey, RedisModuleString *tokey, const void *value);
 typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len);
 typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data);
 typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter);
@@ -516,6 +517,7 @@ typedef struct RedisModuleTypeMethods {
     int aux_save_triggers;
     RedisModuleTypeFreeEffortFunc free_effort;
     RedisModuleTypeUnlinkFunc unlink;
+    RedisModuleTypeCopyFunc copy;
 } RedisModuleTypeMethods;
 
 #define REDISMODULE_GET_API(name) \
diff --git a/src/server.h b/src/server.h
index 42ce6d32a..97604a11b 100644
--- a/src/server.h
+++ b/src/server.h
@@ -529,6 +529,7 @@ typedef size_t (*moduleTypeMemUsageFunc)(const void *value);
 typedef void (*moduleTypeFreeFunc)(void *value);
 typedef size_t (*moduleTypeFreeEffortFunc)(struct redisObject *key, const void *value);
 typedef void (*moduleTypeUnlinkFunc)(struct redisObject *key, void *value);
+typedef void *(*moduleTypeCopyFunc)(struct redisObject *fromkey, struct redisObject *tokey, const void *value);
 
 /* This callback type is called by moduleNotifyUserChanged() every time
  * a user authenticated via the module API is associated with a different
@@ -550,6 +551,7 @@ typedef struct RedisModuleType {
     moduleTypeFreeFunc free;
     moduleTypeFreeEffortFunc free_effort;
     moduleTypeUnlinkFunc unlink;
+    moduleTypeCopyFunc copy;
     moduleTypeAuxLoadFunc aux_load;
     moduleTypeAuxSaveFunc aux_save;
     int aux_save_triggers;
@@ -1694,6 +1696,7 @@ void moduleUnblockClient(client *c);
 int moduleClientIsBlockedOnKeys(client *c);
 void moduleNotifyUserChanged(client *c);
 void moduleNotifyKeyUnlink(robj *key, robj *val);
+robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, robj *value);
 
 /* Utils */
 long long ustime(void);
diff --git a/tests/modules/datatype.c b/tests/modules/datatype.c
index 6596f9368..0c6f95551 100644
--- a/tests/modules/datatype.c
+++ b/tests/modules/datatype.c
@@ -41,6 +41,32 @@ static void datatype_free(void *value) {
     }
 }
 
+static void *datatype_copy(RedisModuleString *fromkey, RedisModuleString *tokey, const void *value) {
+    const DataType *old = value;
+
+    /* Answers to ultimate questions cannot be copied! */
+    if (old->intval == 42)
+        return NULL;
+
+    DataType *new = (DataType *) RedisModule_Alloc(sizeof(DataType));
+
+    new->intval = old->intval;
+    new->strval = RedisModule_CreateStringFromString(NULL, old->strval);
+
+    /* Breaking the rules here! We return a copy that also includes traces
+     * of fromkey/tokey to confirm we get what we expect.
+     */
+    size_t len;
+    const char *str = RedisModule_StringPtrLen(fromkey, &len);
+    RedisModule_StringAppendBuffer(NULL, new->strval, "/", 1);
+    RedisModule_StringAppendBuffer(NULL, new->strval, str, len);
+    RedisModule_StringAppendBuffer(NULL, new->strval, "/", 1);
+    str = RedisModule_StringPtrLen(tokey, &len);
+    RedisModule_StringAppendBuffer(NULL, new->strval, str, len);
+
+    return new;
+}
+
 static int datatype_set(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     if (argc != 4) {
         RedisModule_WrongArity(ctx);
@@ -97,9 +123,13 @@ static int datatype_get(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     DataType *dt = RedisModule_ModuleTypeGetValue(key);
     RedisModule_CloseKey(key);
 
-    RedisModule_ReplyWithArray(ctx, 2);
-    RedisModule_ReplyWithLongLong(ctx, dt->intval);
-    RedisModule_ReplyWithString(ctx, dt->strval);
+    if (!dt) {
+        RedisModule_ReplyWithNullArray(ctx);
+    } else {
+        RedisModule_ReplyWithArray(ctx, 2);
+        RedisModule_ReplyWithLongLong(ctx, dt->intval);
+        RedisModule_ReplyWithString(ctx, dt->strval);
+    }
     return REDISMODULE_OK;
 }
 
@@ -161,6 +191,7 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
         .rdb_load = datatype_load,
         .rdb_save = datatype_save,
         .free = datatype_free,
+        .copy = datatype_copy
     };
 
     datatype = RedisModule_CreateDataType(ctx, "test___dt", 1, &datatype_methods);
diff --git a/tests/unit/moduleapi/datatype.tcl b/tests/unit/moduleapi/datatype.tcl
index e235462ea..cd6ebb32a 100644
--- a/tests/unit/moduleapi/datatype.tcl
+++ b/tests/unit/moduleapi/datatype.tcl
@@ -41,4 +41,18 @@ start_server {tags {"modules"}} {
         catch {r datatype.swap key-a key-b} e
         set e
     } {*ERR*}
+
+    test {DataType: Copy command works for modules} {
+        # Test failed copies
+        r datatype.set answer-to-universe 42 AAA
+        catch {r copy answer-to-universe answer2} e
+        assert_match {*module key failed to copy*} $e
+
+        # Our module's data type copy function copies the int value as-is
+        # but appends /<from-key>/<to-key> to the string value so we can
+        # track passed arguments.
+        r datatype.set sourcekey 1234 AAA
+        r copy sourcekey targetkey
+        r datatype.get targetkey
+    } {1234 AAA/sourcekey/targetkey}
 }

From 8c291b97b95f2e011977b522acf77ead23e26f55 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Fri, 11 Dec 2020 18:31:40 +0200
Subject: [PATCH 0863/1098] TLS: Add different client cert support. (#8076)

This adds a new `tls-client-cert-file` and `tls-client-key-file`
configuration directives which make it possible to use different
certificates for the TLS-server and TLS-client functions of Redis.

This is an optional directive. If it is not specified the `tls-cert-file`
and `tls-key-file` directives are used for TLS client functions as well.

Also, `utils/gen-test-certs.sh` now creates additional server-only and client-only certs and will skip intensive operations if target files already exist.
---
 redis.conf                   |  12 +++
 src/config.c                 |   2 +
 src/sentinel.c               |   3 +-
 src/server.h                 |   6 +-
 src/tls.c                    | 172 +++++++++++++++++++++--------------
 tests/instances.tcl          |  10 +-
 tests/support/benchmark.tcl  |   4 +-
 tests/support/cli.tcl        |   4 +-
 tests/support/redis.tcl      |   4 +-
 tests/support/server.tcl     |  15 ++-
 tests/test_helper.tcl        |   4 +-
 tests/unit/introspection.tcl |   2 +
 tests/unit/tls.tcl           |  21 +++++
 utils/gen-test-certs.sh      |  65 ++++++++++---
 14 files changed, 225 insertions(+), 99 deletions(-)

diff --git a/redis.conf b/redis.conf
index 13766171e..849f171bc 100644
--- a/redis.conf
+++ b/redis.conf
@@ -151,6 +151,18 @@ tcp-keepalive 300
 # tls-cert-file redis.crt 
 # tls-key-file redis.key
 
+# Normally Redis uses the same certificate for both server functions (accepting
+# connections) and client functions (replicating from a master, establishing
+# cluster bus connections, etc.).
+#
+# Sometimes certificates are issued with attributes that designate them as
+# client-only or server-only certificates. In that case it may be desired to use
+# different certificates for incoming (server) and outgoing (client)
+# connections. To do that, use the following directives:
+#
+# tls-client-cert-file client.crt
+# tls-client-key-file client.key
+
 # Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange:
 #
 # tls-dh-params-file redis.dh
diff --git a/src/config.c b/src/config.c
index 5fdde44cd..28b756b5e 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2450,6 +2450,8 @@ standardConfig configs[] = {
     createBoolConfig("tls-session-caching", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.session_caching, 1, NULL, updateTlsCfgBool),
     createStringConfig("tls-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.cert_file, NULL, NULL, updateTlsCfg),
     createStringConfig("tls-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.key_file, NULL, NULL, updateTlsCfg),
+    createStringConfig("tls-client-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_cert_file, NULL, NULL, updateTlsCfg),
+    createStringConfig("tls-client-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_key_file, NULL, NULL, updateTlsCfg),
     createStringConfig("tls-dh-params-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.dh_params_file, NULL, NULL, updateTlsCfg),
     createStringConfig("tls-ca-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_file, NULL, NULL, updateTlsCfg),
     createStringConfig("tls-ca-cert-dir", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_dir, NULL, NULL, updateTlsCfg),
diff --git a/src/sentinel.c b/src/sentinel.c
index e05be4384..95cfa84ad 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -46,6 +46,7 @@ extern char **environ;
 
 #ifdef USE_OPENSSL
 extern SSL_CTX *redis_tls_ctx;
+extern SSL_CTX *redis_tls_client_ctx;
 #endif
 
 #define REDIS_SENTINEL_PORT 26379
@@ -2077,7 +2078,7 @@ static int instanceLinkNegotiateTLS(redisAsyncContext *context) {
     (void) context;
 #else
     if (!redis_tls_ctx) return C_ERR;
-    SSL *ssl = SSL_new(redis_tls_ctx);
+    SSL *ssl = SSL_new(redis_tls_client_ctx ? redis_tls_client_ctx : redis_tls_ctx);
     if (!ssl) return C_ERR;
 
     if (redisInitiateSSL(&context->c, ssl) == REDIS_ERR) return C_ERR;
diff --git a/src/server.h b/src/server.h
index 97604a11b..35b695a42 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1068,8 +1068,10 @@ struct malloc_stats {
  *----------------------------------------------------------------------------*/
 
 typedef struct redisTLSContextConfig {
-    char *cert_file;
-    char *key_file;
+    char *cert_file;                /* Server side and optionally client side cert file name */
+    char *key_file;                 /* Private key filename for cert_file */
+    char *client_cert_file;         /* Certificate to use as a client; if none, use cert_file */
+    char *client_key_file;          /* Private key filename for client_cert_file */
     char *dh_params_file;
     char *ca_cert_file;
     char *ca_cert_dir;
diff --git a/src/tls.c b/src/tls.c
index af16d2a41..caa987965 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -54,7 +54,8 @@
 
 extern ConnectionType CT_Socket;
 
-SSL_CTX *redis_tls_ctx;
+SSL_CTX *redis_tls_ctx = NULL;
+SSL_CTX *redis_tls_client_ctx = NULL;
 
 static int parseProtocolsConfig(const char *str) {
     int i, count = 0;
@@ -163,50 +164,22 @@ void tlsInit(void) {
     pending_list = listCreate();
 }
 
-/* Attempt to configure/reconfigure TLS. This operation is atomic and will
- * leave the SSL_CTX unchanged if fails.
+/* Create a *base* SSL_CTX using the SSL configuration provided. The base context
+ * includes everything that's common for both client-side and server-side connections.
  */
-int tlsConfigure(redisTLSContextConfig *ctx_config) {
+static SSL_CTX *createSSLContext(redisTLSContextConfig *ctx_config, int protocols,
+                                 const char *cert_file, const char *key_file) {
     char errbuf[256];
     SSL_CTX *ctx = NULL;
 
-    if (!ctx_config->cert_file) {
-        serverLog(LL_WARNING, "No tls-cert-file configured!");
-        goto error;
-    }
-
-    if (!ctx_config->key_file) {
-        serverLog(LL_WARNING, "No tls-key-file configured!");
-        goto error;
-    }
-
-    if (((server.tls_auth_clients != TLS_CLIENT_AUTH_NO) || server.tls_cluster || server.tls_replication) &&
-            !ctx_config->ca_cert_file && !ctx_config->ca_cert_dir) {
-        serverLog(LL_WARNING, "Either tls-ca-cert-file or tls-ca-cert-dir must be specified when tls-cluster, tls-replication or tls-auth-clients are enabled!");
-        goto error;
-    }
-
     ctx = SSL_CTX_new(SSLv23_method());
 
     SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3);
-    SSL_CTX_set_options(ctx, SSL_OP_SINGLE_DH_USE);
 
 #ifdef SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
     SSL_CTX_set_options(ctx, SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS);
 #endif
 
-    if (ctx_config->session_caching) {
-        SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_SERVER);
-        SSL_CTX_sess_set_cache_size(ctx, ctx_config->session_cache_size);
-        SSL_CTX_set_timeout(ctx, ctx_config->session_cache_timeout);
-        SSL_CTX_set_session_id_context(ctx, (void *) "redis", 5);
-    } else {
-        SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_OFF);
-    }
-
-    int protocols = parseProtocolsConfig(ctx_config->protocols);
-    if (protocols == -1) goto error;
-
     if (!(protocols & REDIS_TLS_PROTO_TLSv1))
         SSL_CTX_set_options(ctx, SSL_OP_NO_TLSv1);
     if (!(protocols & REDIS_TLS_PROTO_TLSv1_1))
@@ -224,6 +197,87 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
     SSL_CTX_set_options(ctx, SSL_OP_NO_COMPRESSION);
 #endif
 
+    SSL_CTX_set_mode(ctx, SSL_MODE_ENABLE_PARTIAL_WRITE|SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
+    SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);
+
+    if (SSL_CTX_use_certificate_chain_file(ctx, cert_file) <= 0) {
+        ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf));
+        serverLog(LL_WARNING, "Failed to load certificate: %s: %s", cert_file, errbuf);
+        goto error;
+    }
+
+    if (SSL_CTX_use_PrivateKey_file(ctx, key_file, SSL_FILETYPE_PEM) <= 0) {
+        ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf));
+        serverLog(LL_WARNING, "Failed to load private key: %s: %s", key_file, errbuf);
+        goto error;
+    }
+
+    if ((ctx_config->ca_cert_file || ctx_config->ca_cert_dir) &&
+        SSL_CTX_load_verify_locations(ctx, ctx_config->ca_cert_file, ctx_config->ca_cert_dir) <= 0) {
+        ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf));
+        serverLog(LL_WARNING, "Failed to configure CA certificate(s) file/directory: %s", errbuf);
+        goto error;
+    }
+
+    if (ctx_config->ciphers && !SSL_CTX_set_cipher_list(ctx, ctx_config->ciphers)) {
+        serverLog(LL_WARNING, "Failed to configure ciphers: %s", ctx_config->ciphers);
+        goto error;
+    }
+
+#ifdef TLS1_3_VERSION
+    if (ctx_config->ciphersuites && !SSL_CTX_set_ciphersuites(ctx, ctx_config->ciphersuites)) {
+        serverLog(LL_WARNING, "Failed to configure ciphersuites: %s", ctx_config->ciphersuites);
+        goto error;
+    }
+#endif
+
+    return ctx;
+
+error:
+    if (ctx) SSL_CTX_free(ctx);
+    return NULL;
+}
+
+/* Attempt to configure/reconfigure TLS. This operation is atomic and will
+ * leave the SSL_CTX unchanged if fails.
+ */
+int tlsConfigure(redisTLSContextConfig *ctx_config) {
+    char errbuf[256];
+    SSL_CTX *ctx = NULL;
+    SSL_CTX *client_ctx = NULL;
+
+    if (!ctx_config->cert_file) {
+        serverLog(LL_WARNING, "No tls-cert-file configured!");
+        goto error;
+    }
+
+    if (!ctx_config->key_file) {
+        serverLog(LL_WARNING, "No tls-key-file configured!");
+        goto error;
+    }
+
+    if (((server.tls_auth_clients != TLS_CLIENT_AUTH_NO) || server.tls_cluster || server.tls_replication) &&
+            !ctx_config->ca_cert_file && !ctx_config->ca_cert_dir) {
+        serverLog(LL_WARNING, "Either tls-ca-cert-file or tls-ca-cert-dir must be specified when tls-cluster, tls-replication or tls-auth-clients are enabled!");
+        goto error;
+    }
+
+    int protocols = parseProtocolsConfig(ctx_config->protocols);
+    if (protocols == -1) goto error;
+
+    /* Create server side/generla context */
+    ctx = createSSLContext(ctx_config, protocols, ctx_config->cert_file, ctx_config->key_file);
+    if (!ctx) goto error;
+
+    if (ctx_config->session_caching) {
+        SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_SERVER);
+        SSL_CTX_sess_set_cache_size(ctx, ctx_config->session_cache_size);
+        SSL_CTX_set_timeout(ctx, ctx_config->session_cache_timeout);
+        SSL_CTX_set_session_id_context(ctx, (void *) "redis", 5);
+    } else {
+        SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_OFF);
+    }
+
 #ifdef SSL_OP_NO_CLIENT_RENEGOTIATION
     SSL_CTX_set_options(ctx, SSL_OP_NO_CLIENT_RENEGOTIATION);
 #endif
@@ -231,30 +285,10 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
     if (ctx_config->prefer_server_ciphers)
         SSL_CTX_set_options(ctx, SSL_OP_CIPHER_SERVER_PREFERENCE);
 
-    SSL_CTX_set_mode(ctx, SSL_MODE_ENABLE_PARTIAL_WRITE|SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
-    SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);
 #if defined(SSL_CTX_set_ecdh_auto)
     SSL_CTX_set_ecdh_auto(ctx, 1);
 #endif
-
-    if (SSL_CTX_use_certificate_chain_file(ctx, ctx_config->cert_file) <= 0) {
-        ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf));
-        serverLog(LL_WARNING, "Failed to load certificate: %s: %s", ctx_config->cert_file, errbuf);
-        goto error;
-    }
-        
-    if (SSL_CTX_use_PrivateKey_file(ctx, ctx_config->key_file, SSL_FILETYPE_PEM) <= 0) {
-        ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf));
-        serverLog(LL_WARNING, "Failed to load private key: %s: %s", ctx_config->key_file, errbuf);
-        goto error;
-    }
-    
-    if ((ctx_config->ca_cert_file || ctx_config->ca_cert_dir) &&
-        SSL_CTX_load_verify_locations(ctx, ctx_config->ca_cert_file, ctx_config->ca_cert_dir) <= 0) {
-        ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf));
-        serverLog(LL_WARNING, "Failed to configure CA certificate(s) file/directory: %s", errbuf);
-        goto error;
-    }
+    SSL_CTX_set_options(ctx, SSL_OP_SINGLE_DH_USE);
 
     if (ctx_config->dh_params_file) {
         FILE *dhfile = fopen(ctx_config->dh_params_file, "r");
@@ -281,25 +315,22 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
         DH_free(dh);
     }
 
-    if (ctx_config->ciphers && !SSL_CTX_set_cipher_list(ctx, ctx_config->ciphers)) {
-        serverLog(LL_WARNING, "Failed to configure ciphers: %s", ctx_config->ciphers);
-        goto error;
+    /* If a client-side certificate is configured, create an explicit client context */
+    if (ctx_config->client_cert_file && ctx_config->client_key_file) {
+        client_ctx = createSSLContext(ctx_config, protocols, ctx_config->client_cert_file, ctx_config->client_key_file);
+        if (!client_ctx) goto error;
     }
 
-#ifdef TLS1_3_VERSION
-    if (ctx_config->ciphersuites && !SSL_CTX_set_ciphersuites(ctx, ctx_config->ciphersuites)) {
-        serverLog(LL_WARNING, "Failed to configure ciphersuites: %s", ctx_config->ciphersuites);
-        goto error;
-    }
-#endif
-
     SSL_CTX_free(redis_tls_ctx);
+    SSL_CTX_free(redis_tls_client_ctx);
     redis_tls_ctx = ctx;
+    redis_tls_client_ctx = client_ctx;
 
     return C_OK;
 
 error:
     if (ctx) SSL_CTX_free(ctx);
+    if (client_ctx) SSL_CTX_free(client_ctx);
     return C_ERR;
 }
 
@@ -344,14 +375,21 @@ typedef struct tls_connection {
     listNode *pending_list_node;
 } tls_connection;
 
-connection *connCreateTLS(void) {
+static connection *createTLSConnection(bool client_side) {
+    SSL_CTX *ctx = redis_tls_ctx;
+    if (client_side && redis_tls_client_ctx)
+        ctx = redis_tls_client_ctx;
     tls_connection *conn = zcalloc(sizeof(tls_connection));
     conn->c.type = &CT_TLS;
     conn->c.fd = -1;
-    conn->ssl = SSL_new(redis_tls_ctx);
+    conn->ssl = SSL_new(ctx);
     return (connection *) conn;
 }
 
+connection *connCreateTLS(void) {
+    return createTLSConnection(true);
+}
+
 /* Fetch the latest OpenSSL error and store it in the connection */
 static void updateTLSError(tls_connection *conn) {
     conn->c.last_errno = 0;
@@ -370,7 +408,7 @@ static void updateTLSError(tls_connection *conn) {
  * is not in an error state.
  */
 connection *connCreateAcceptedTLS(int fd, int require_auth) {
-    tls_connection *conn = (tls_connection *) connCreateTLS();
+    tls_connection *conn = (tls_connection *) createTLSConnection(false);
     conn->c.fd = fd;
     conn->c.state = CONN_STATE_ACCEPTING;
 
diff --git a/tests/instances.tcl b/tests/instances.tcl
index 156c92706..a9cc01008 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -76,8 +76,10 @@ proc spawn_instance {type base_port count {conf {}}} {
             puts $cfg "tls-replication yes"
             puts $cfg "tls-cluster yes"
             puts $cfg "port 0"
-            puts $cfg [format "tls-cert-file %s/../../tls/redis.crt" [pwd]]
-            puts $cfg [format "tls-key-file %s/../../tls/redis.key" [pwd]]
+            puts $cfg [format "tls-cert-file %s/../../tls/server.crt" [pwd]]
+            puts $cfg [format "tls-key-file %s/../../tls/server.key" [pwd]]
+            puts $cfg [format "tls-client-cert-file %s/../../tls/client.crt" [pwd]]
+            puts $cfg [format "tls-client-key-file %s/../../tls/client.key" [pwd]]
             puts $cfg [format "tls-dh-params-file %s/../../tls/redis.dh" [pwd]]
             puts $cfg [format "tls-ca-cert-file %s/../../tls/ca.crt" [pwd]]
             puts $cfg "loglevel debug"
@@ -234,8 +236,8 @@ proc parse_options {} {
             package require tls 1.6
             ::tls::init \
                 -cafile "$::tlsdir/ca.crt" \
-                -certfile "$::tlsdir/redis.crt" \
-                -keyfile "$::tlsdir/redis.key"
+                -certfile "$::tlsdir/client.crt" \
+                -keyfile "$::tlsdir/client.key"
             set ::tls 1
         } elseif {$opt eq "--help"} {
             puts "--single <pattern>      Only runs tests specified by pattern."
diff --git a/tests/support/benchmark.tcl b/tests/support/benchmark.tcl
index ed75bfeda..3d08b76f5 100644
--- a/tests/support/benchmark.tcl
+++ b/tests/support/benchmark.tcl
@@ -1,7 +1,7 @@
 proc redisbenchmark_tls_config {testsdir} {
     set tlsdir [file join $testsdir tls]
-    set cert [file join $tlsdir redis.crt]
-    set key [file join $tlsdir redis.key]
+    set cert [file join $tlsdir client.crt]
+    set key [file join $tlsdir client.key]
     set cacert [file join $tlsdir ca.crt]
 
     if {$::tls} {
diff --git a/tests/support/cli.tcl b/tests/support/cli.tcl
index d55487931..19e306e24 100644
--- a/tests/support/cli.tcl
+++ b/tests/support/cli.tcl
@@ -1,7 +1,7 @@
 proc rediscli_tls_config {testsdir} {
     set tlsdir [file join $testsdir tls]
-    set cert [file join $tlsdir redis.crt]
-    set key [file join $tlsdir redis.key]
+    set cert [file join $tlsdir client.crt]
+    set key [file join $tlsdir client.key]
     set cacert [file join $tlsdir ca.crt]
 
     if {$::tls} {
diff --git a/tests/support/redis.tcl b/tests/support/redis.tcl
index 26b4510ac..9eb5b94e2 100644
--- a/tests/support/redis.tcl
+++ b/tests/support/redis.tcl
@@ -44,8 +44,8 @@ proc redis {{server 127.0.0.1} {port 6379} {defer 0} {tls 0} {tlsoptions {}}} {
         package require tls
         ::tls::init \
             -cafile "$::tlsdir/ca.crt" \
-            -certfile "$::tlsdir/redis.crt" \
-            -keyfile "$::tlsdir/redis.key" \
+            -certfile "$::tlsdir/client.crt" \
+            -keyfile "$::tlsdir/client.key" \
             {*}$tlsoptions
         set fd [::tls::socket $server $port]
     } else {
diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index e5b167a35..1cddb7068 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -229,6 +229,7 @@ proc start_server {options {code undefined}} {
     # setup defaults
     set baseconfig "default.conf"
     set overrides {}
+    set omit {}
     set tags {}
     set keep_persistence false
 
@@ -241,6 +242,9 @@ proc start_server {options {code undefined}} {
             "overrides" {
                 set overrides $value
             }
+            "omit" {
+                set omit $value
+            }
             "tags" {
                 # If we 'tags' contain multiple tags, quoted and seperated by spaces,
                 # we want to get rid of the quotes in order to have a proper list
@@ -306,8 +310,10 @@ proc start_server {options {code undefined}} {
     set data [split [exec cat "tests/assets/$baseconfig"] "\n"]
     set config {}
     if {$::tls} {
-        dict set config "tls-cert-file" [format "%s/tests/tls/redis.crt" [pwd]]
-        dict set config "tls-key-file" [format "%s/tests/tls/redis.key" [pwd]]
+        dict set config "tls-cert-file" [format "%s/tests/tls/server.crt" [pwd]]
+        dict set config "tls-key-file" [format "%s/tests/tls/server.key" [pwd]]
+        dict set config "tls-client-cert-file" [format "%s/tests/tls/client.crt" [pwd]]
+        dict set config "tls-client-key-file" [format "%s/tests/tls/client.key" [pwd]]
         dict set config "tls-dh-params-file" [format "%s/tests/tls/redis.dh" [pwd]]
         dict set config "tls-ca-cert-file" [format "%s/tests/tls/ca.crt" [pwd]]
         dict set config "loglevel" "debug"
@@ -343,6 +349,11 @@ proc start_server {options {code undefined}} {
         dict set config $directive $arguments
     }
 
+    # remove directives that are marked to be omitted
+    foreach directive $omit {
+        dict unset config $directive
+    }
+
     # write new configuration to temporary file
     set config_file [tmpfile redis.conf]
     create_server_config_file $config_file $config
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 29ebdd7bb..3b8dc16da 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -602,8 +602,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
         set ::tls 1
         ::tls::init \
             -cafile "$::tlsdir/ca.crt" \
-            -certfile "$::tlsdir/redis.crt" \
-            -keyfile "$::tlsdir/redis.key"
+            -certfile "$::tlsdir/client.crt" \
+            -keyfile "$::tlsdir/client.key"
     } elseif {$opt eq {--host}} {
         set ::external 1
         set ::host $arg
diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl
index a250762f2..0a7f7a9c9 100644
--- a/tests/unit/introspection.tcl
+++ b/tests/unit/introspection.tcl
@@ -122,6 +122,8 @@ start_server {tags {"introspection"}} {
                 tls-session-caching
                 tls-cert-file
                 tls-key-file
+                tls-client-cert-file
+                tls-client-key-file
                 tls-dh-params-file
                 tls-ca-cert-file
                 tls-ca-cert-dir
diff --git a/tests/unit/tls.tcl b/tests/unit/tls.tcl
index bb5b6d034..a6c2f3c2c 100644
--- a/tests/unit/tls.tcl
+++ b/tests/unit/tls.tcl
@@ -93,5 +93,26 @@ start_server {tags {"tls"}} {
             r CONFIG SET tls-protocols ""
             r CONFIG SET tls-ciphers "DEFAULT"
         }
+
+        test {TLS: Verify tls-cert-file is also used as a client cert if none specified} {
+            set master [srv 0 client]
+            set master_host [srv 0 host]
+            set master_port [srv 0 port]
+
+            # Use a non-restricted client/server cert for the replica
+            set redis_crt [format "%s/tests/tls/redis.crt" [pwd]]
+            set redis_key [format "%s/tests/tls/redis.key" [pwd]]
+
+            start_server [list overrides [list tls-cert-file $redis_crt tls-key-file $redis_key] \
+                               omit [list tls-client-cert-file tls-client-key-file]] {
+                set replica [srv 0 client]
+                $replica replicaof $master_host $master_port
+                wait_for_condition 30 100 {
+                    [string match {*master_link_status:up*} [$replica info replication]]
+                } else {
+                    fail "Can't authenticate to master using just tls-cert-file!"
+                }
+            }
+        }
     }
 }
diff --git a/utils/gen-test-certs.sh b/utils/gen-test-certs.sh
index a46edc55a..60814483b 100755
--- a/utils/gen-test-certs.sh
+++ b/utils/gen-test-certs.sh
@@ -1,23 +1,58 @@
 #!/bin/bash
+
+# Generate some test certificates which are used by the regression test suite:
+#
+#   tests/tls/ca.{crt,key}          Self signed CA certificate.
+#   tests/tls/redis.{crt,key}       A certificate with no key usage/policy restrictions.
+#   tests/tls/client.{crt,key}      A certificate restricted for SSL client usage.
+#   tests/tls/server.{crt,key}      A certificate restricted fro SSL server usage.
+#   tests/tls/redis.dh              DH Params file.
+
+generate_cert() {
+    local name=$1
+    local cn="$2"
+    local opts="$3"
+
+    local keyfile=tests/tls/${name}.key
+    local certfile=tests/tls/${name}.crt
+
+    [ -f $keyfile ] || openssl genrsa -out $keyfile 2048
+    openssl req \
+        -new -sha256 \
+        -subj "/O=Redis Test/CN=$cn" \
+        -key $keyfile | \
+        openssl x509 \
+            -req -sha256 \
+            -CA tests/tls/ca.crt \
+            -CAkey tests/tls/ca.key \
+            -CAserial tests/tls/ca.txt \
+            -CAcreateserial \
+            -days 365 \
+            $opts \
+            -out $certfile
+}
+
 mkdir -p tests/tls
-openssl genrsa -out tests/tls/ca.key 4096
+[ -f tests/tls/ca.key ] || openssl genrsa -out tests/tls/ca.key 4096
 openssl req \
     -x509 -new -nodes -sha256 \
     -key tests/tls/ca.key \
     -days 3650 \
     -subj '/O=Redis Test/CN=Certificate Authority' \
     -out tests/tls/ca.crt
-openssl genrsa -out tests/tls/redis.key 2048
-openssl req \
-    -new -sha256 \
-    -key tests/tls/redis.key \
-    -subj '/O=Redis Test/CN=Server' | \
-    openssl x509 \
-        -req -sha256 \
-        -CA tests/tls/ca.crt \
-        -CAkey tests/tls/ca.key \
-        -CAserial tests/tls/ca.txt \
-        -CAcreateserial \
-        -days 365 \
-        -out tests/tls/redis.crt
-openssl dhparam -out tests/tls/redis.dh 2048
+
+cat > tests/tls/openssl.cnf <<_END_
+[ server_cert ]
+keyUsage = digitalSignature, keyEncipherment
+nsCertType = server
+
+[ client_cert ]
+keyUsage = digitalSignature, keyEncipherment
+nsCertType = client
+_END_
+
+generate_cert server "Server-only" "-extfile tests/tls/openssl.cnf -extensions server_cert"
+generate_cert client "Client-only" "-extfile tests/tls/openssl.cnf -extensions client_cert"
+generate_cert redis "Generic-cert"
+
+[ -f tests/tls/redis.dh ] || openssl dhparam -out tests/tls/redis.dh 2048

From 4d06d99bf8fc91289c79ae07110c92525ab1c1f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9D=A8=E5=8D=9A=E4=B8=9C?= <bodong.ybd@alibaba-inc.com>
Date: Sat, 12 Dec 2020 08:21:05 +0800
Subject: [PATCH 0864/1098] Add GEOSEARCH / GEOSEARCHSTORE commands (#8094)

Add commands to query geospatial data with bounding box.

Two new commands that replace the existing 4 GEORADIUS* commands.

GEOSEARCH key [FROMMEMBER member] [FROMLOC long lat] [BYRADIUS radius
unit] [BYBOX width height unit] [WITHCORD] [WITHDIST] [WITHASH] [COUNT
count] [ASC|DESC]

GEOSEARCHSTORE dest_key src_key [FROMMEMBER member] [FROMLOC long lat]
[BYRADIUS radius unit] [BYBOX width height unit] [WITHCORD] [WITHDIST]
[WITHASH] [COUNT count] [ASC|DESC] [STOREDIST]

- Add two types of CIRCULAR_TYPE and RECTANGLE_TYPE to achieve different searches
- Judge whether the point is within the rectangle, refer to:
geohashGetDistanceIfInRectangle
---
 src/geo.c            | 211 +++++++++++++++++++++++++++++++------------
 src/geohash.h        |  19 ++++
 src/geohash_helper.c |  71 ++++++++++-----
 src/geohash_helper.h |  12 +--
 src/server.c         |  12 ++-
 src/server.h         |   2 +
 tests/unit/geo.tcl   |  80 ++++++++++++++++
 7 files changed, 317 insertions(+), 90 deletions(-)

diff --git a/src/geo.c b/src/geo.c
index 7594bb2e9..13cbb1c50 100644
--- a/src/geo.c
+++ b/src/geo.c
@@ -144,31 +144,57 @@ double extractUnitOrReply(client *c, robj *unit) {
 
 /* Input Argument Helper.
  * Extract the distance from the specified two arguments starting at 'argv'
- * that should be in the form: <number> <unit>, and return the distance in the
- * specified unit on success. *conversions is populated with the coefficient
- * to use in order to convert meters to the unit.
- *
- * On error a value less than zero is returned. */
-double extractDistanceOrReply(client *c, robj **argv,
-                                     double *conversion) {
+ * that should be in the form: <number> <unit>, and return C_OK or C_ERR means success or failure
+ * *conversions is populated with the coefficient to use in order to convert meters to the unit.*/
+int extractDistanceOrReply(client *c, robj **argv,
+                              double *conversion, double *radius) {
     double distance;
     if (getDoubleFromObjectOrReply(c, argv[0], &distance,
                                    "need numeric radius") != C_OK) {
-        return -1;
+        return C_ERR;
     }
 
     if (distance < 0) {
         addReplyError(c,"radius cannot be negative");
-        return -1;
+        return C_ERR;
     }
+    if (radius) *radius = distance;
 
     double to_meters = extractUnitOrReply(c,argv[1]);
     if (to_meters < 0) {
-        return -1;
+        return C_ERR;
     }
 
     if (conversion) *conversion = to_meters;
-    return distance * to_meters;
+    return C_OK;
+}
+
+/* Input Argument Helper.
+ * Extract height and width from the specified three arguments starting at 'argv'
+ * that should be in the form: <number> <number> <unit>, and return C_OK or C_ERR means success or failure
+ * *conversions is populated with the coefficient to use in order to convert meters to the unit.*/
+int extractBoxOrReply(client *c, robj **argv, double *conversion,
+                         double *height, double *width) {
+    double h, w;
+    if ((getDoubleFromObjectOrReply(c, argv[0], &h, "need numeric height") != C_OK) ||
+        (getDoubleFromObjectOrReply(c, argv[1], &w, "need numeric width") != C_OK)) {
+        return C_ERR;
+    }
+
+    if (h < 0 || w < 0) {
+        addReplyError(c, "height or width cannot be negative");
+        return C_ERR;
+    }
+    if (height) *height = h;
+    if (width) *width = w;
+
+    double to_meters = extractUnitOrReply(c,argv[2]);
+    if (to_meters < 0) {
+        return C_ERR;
+    }
+
+    if (conversion) *conversion = to_meters;
+    return C_OK;
 }
 
 /* The default addReplyDouble has too much accuracy.  We use this
@@ -183,21 +209,22 @@ void addReplyDoubleDistance(client *c, double d) {
 }
 
 /* Helper function for geoGetPointsInRange(): given a sorted set score
- * representing a point, and another point (the center of our search) and
- * a radius, appends this entry as a geoPoint into the specified geoArray
- * only if the point is within the search area.
+ * representing a point, and a GeoShape, appends this entry as a geoPoint
+ * into the specified geoArray only if the point is within the search area.
  *
  * returns C_OK if the point is included, or REIDS_ERR if it is outside. */
-int geoAppendIfWithinRadius(geoArray *ga, double lon, double lat, double radius, double score, sds member) {
-    double distance, xy[2];
+int geoAppendIfWithinShape(geoArray *ga, GeoShape *shape, double score, sds member) {
+    double distance = 0, xy[2];
 
     if (!decodeGeohash(score,xy)) return C_ERR; /* Can't decode. */
     /* Note that geohashGetDistanceIfInRadiusWGS84() takes arguments in
      * reverse order: longitude first, latitude later. */
-    if (!geohashGetDistanceIfInRadiusWGS84(lon,lat, xy[0], xy[1],
-                                           radius, &distance))
-    {
-        return C_ERR;
+    if (shape->type == CIRCULAR_TYPE) {
+        if (!geohashGetDistanceIfInRadiusWGS84(shape->xy[0], shape->xy[1], xy[0], xy[1],
+                                               shape->t.radius*shape->conversion, &distance)) return C_ERR;
+    } else if (shape->type == RECTANGLE_TYPE) {
+        if (!geohashGetDistanceIfInRectangle(shape->bounds, shape->xy[0], shape->xy[1],
+                                             xy[0], xy[1], &distance)) return C_ERR;
     }
 
     /* Append the new element. */
@@ -222,7 +249,7 @@ int geoAppendIfWithinRadius(geoArray *ga, double lon, double lat, double radius,
  * using multiple queries to the sorted set, that we later need to sort
  * via qsort. Similarly we need to be able to reject points outside the search
  * radius area ASAP in order to allocate and process more points than needed. */
-int geoGetPointsInRange(robj *zobj, double min, double max, double lon, double lat, double radius, geoArray *ga) {
+int geoGetPointsInRange(robj *zobj, double min, double max, GeoShape *shape, geoArray *ga) {
     /* minex 0 = include min in range; maxex 1 = exclude max in range */
     /* That's: min <= val < max */
     zrangespec range = { .min = min, .max = max, .minex = 0, .maxex = 1 };
@@ -254,7 +281,7 @@ int geoGetPointsInRange(robj *zobj, double min, double max, double lon, double l
             ziplistGet(eptr, &vstr, &vlen, &vlong);
             member = (vstr == NULL) ? sdsfromlonglong(vlong) :
                                       sdsnewlen(vstr,vlen);
-            if (geoAppendIfWithinRadius(ga,lon,lat,radius,score,member)
+            if (geoAppendIfWithinShape(ga,shape,score,member)
                 == C_ERR) sdsfree(member);
             zzlNext(zl, &eptr, &sptr);
         }
@@ -275,7 +302,7 @@ int geoGetPointsInRange(robj *zobj, double min, double max, double lon, double l
                 break;
 
             ele = sdsdup(ele);
-            if (geoAppendIfWithinRadius(ga,lon,lat,radius,ln->score,ele)
+            if (geoAppendIfWithinShape(ga,shape,ln->score,ele)
                 == C_ERR) sdsfree(ele);
             ln = ln->level[0].forward;
         }
@@ -315,15 +342,15 @@ void scoresOfGeoHashBox(GeoHashBits hash, GeoHashFix52Bits *min, GeoHashFix52Bit
 /* Obtain all members between the min/max of this geohash bounding box.
  * Populate a geoArray of GeoPoints by calling geoGetPointsInRange().
  * Return the number of points added to the array. */
-int membersOfGeoHashBox(robj *zobj, GeoHashBits hash, geoArray *ga, double lon, double lat, double radius) {
+int membersOfGeoHashBox(robj *zobj, GeoHashBits hash, geoArray *ga, GeoShape *shape) {
     GeoHashFix52Bits min, max;
 
     scoresOfGeoHashBox(hash,&min,&max);
-    return geoGetPointsInRange(zobj, min, max, lon, lat, radius, ga);
+    return geoGetPointsInRange(zobj, min, max, shape, ga);
 }
 
 /* Search all eight neighbors + self geohash box */
-int membersOfAllNeighbors(robj *zobj, GeoHashRadius n, double lon, double lat, double radius, geoArray *ga) {
+int membersOfAllNeighbors(robj *zobj, GeoHashRadius n, GeoShape *shape, geoArray *ga) {
     GeoHashBits neighbors[9];
     unsigned int i, count = 0, last_processed = 0;
     int debugmsg = 0;
@@ -374,7 +401,7 @@ int membersOfAllNeighbors(robj *zobj, GeoHashRadius n, double lon, double lat, d
                 D("Skipping processing of %d, same as previous\n",i);
             continue;
         }
-        count += membersOfGeoHashBox(zobj, neighbors[i], ga, lon, lat, radius);
+        count += membersOfGeoHashBox(zobj, neighbors[i], ga, shape);
         last_processed = i;
     }
     return count;
@@ -454,51 +481,60 @@ void geoaddCommand(client *c) {
 
 #define RADIUS_COORDS (1<<0)    /* Search around coordinates. */
 #define RADIUS_MEMBER (1<<1)    /* Search around member. */
-#define RADIUS_NOSTORE (1<<2)   /* Do not acceot STORE/STOREDIST option. */
+#define RADIUS_NOSTORE (1<<2)   /* Do not accept STORE/STOREDIST option. */
+#define GEOSEARCH (1<<3)        /* GEOSEARCH command variant (different arguments supported) */
+#define GEOSEARCHSTORE (1<<4)   /* GEOSEARCHSTORE just accept STOREDIST option */
 
 /* GEORADIUS key x y radius unit [WITHDIST] [WITHHASH] [WITHCOORD] [ASC|DESC]
  *                               [COUNT count] [STORE key] [STOREDIST key]
- * GEORADIUSBYMEMBER key member radius unit ... options ... */
-void georadiusGeneric(client *c, int flags) {
-    robj *key = c->argv[1];
+ * GEORADIUSBYMEMBER key member radius unit ... options ...
+ * GEOSEARCH key [FROMMEMBER member] [FORMLOG long lat] [BYRADIUS radius unit]
+ *               [BYBOX width height unit] [WITHCORD] [WITHDIST] [WITHASH] [COUNT count] [ASC|DESC]
+ * GEOSEARCHSTORE dest_key src_key [FROMMEMBER member] [FORMLOG long lat] [BYRADIUS radius unit]
+ *               [BYBOX width height unit] [WITHCORD] [WITHDIST] [WITHASH] [COUNT count] [ASC|DESC] [STOREDIST]
+ *  */
+void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
     robj *storekey = NULL;
     int storedist = 0; /* 0 for STORE, 1 for STOREDIST. */
 
     /* Look up the requested zset */
     robj *zobj = NULL;
-    if ((zobj = lookupKeyReadOrReply(c, key, shared.emptyarray)) == NULL ||
+    if ((zobj = lookupKeyReadOrReply(c, c->argv[srcKeyIndex], shared.emptyarray)) == NULL ||
         checkType(c, zobj, OBJ_ZSET)) {
         return;
     }
 
-    /* Find long/lat to use for radius search based on inquiry type */
+    /* Find long/lat to use for radius or box search based on inquiry type */
     int base_args;
-    double xy[2] = { 0 };
+    GeoShape shape = {0};
     if (flags & RADIUS_COORDS) {
         base_args = 6;
-        if (extractLongLatOrReply(c, c->argv + 2, xy) == C_ERR)
-            return;
+        shape.type = CIRCULAR_TYPE;
+        if (extractLongLatOrReply(c, c->argv + 2, shape.xy) == C_ERR) return;
+        if (extractDistanceOrReply(c, c->argv+base_args-2, &shape.conversion, &shape.t.radius) != C_OK) return;
     } else if (flags & RADIUS_MEMBER) {
         base_args = 5;
+        shape.type = CIRCULAR_TYPE;
         robj *member = c->argv[2];
-        if (longLatFromMember(zobj, member, xy) == C_ERR) {
+        if (longLatFromMember(zobj, member, shape.xy) == C_ERR) {
             addReplyError(c, "could not decode requested zset member");
             return;
         }
+        if (extractDistanceOrReply(c, c->argv+base_args-2, &shape.conversion, &shape.t.radius) != C_OK) return;
+    } else if (flags & GEOSEARCH) {
+        base_args = 2;
+        if (flags & GEOSEARCHSTORE) {
+            base_args = 3;
+            storekey = c->argv[1];
+        }
     } else {
         addReplyError(c, "Unknown georadius search type");
         return;
     }
 
-    /* Extract radius and units from arguments */
-    double radius_meters = 0, conversion = 1;
-    if ((radius_meters = extractDistanceOrReply(c, c->argv + base_args - 2,
-                                                &conversion)) < 0) {
-        return;
-    }
-
     /* Discover and populate all optional parameters. */
     int withdist = 0, withhash = 0, withcoords = 0;
+    int frommember = 0, fromloc = 0, byradius = 0, bybox = 0;
     int sort = SORT_NONE;
     long long count = 0;
     if (c->argc > base_args) {
@@ -525,18 +561,64 @@ void georadiusGeneric(client *c, int flags) {
                 i++;
             } else if (!strcasecmp(arg, "store") &&
                        (i+1) < remaining &&
-                       !(flags & RADIUS_NOSTORE))
+                       !(flags & RADIUS_NOSTORE) &&
+                       !(flags & GEOSEARCH))
             {
                 storekey = c->argv[base_args+i+1];
                 storedist = 0;
                 i++;
             } else if (!strcasecmp(arg, "storedist") &&
                        (i+1) < remaining &&
-                       !(flags & RADIUS_NOSTORE))
+                       !(flags & RADIUS_NOSTORE) &&
+                       !(flags & GEOSEARCH))
             {
                 storekey = c->argv[base_args+i+1];
                 storedist = 1;
                 i++;
+            } else if (!strcasecmp(arg, "storedist") &&
+                       (flags & GEOSEARCH) &&
+                       (flags & GEOSEARCHSTORE))
+            {
+                storedist = 1;
+            } else if (!strcasecmp(arg, "frommember") &&
+                      (i+1) < remaining &&
+                      flags & GEOSEARCH &&
+                      !fromloc)
+            {
+                if (longLatFromMember(zobj, c->argv[base_args+i+1], shape.xy) == C_ERR) {
+                    addReplyError(c, "could not decode requested zset member");
+                    return;
+                }
+                frommember = 1;
+                i++;
+            } else if (!strcasecmp(arg, "fromloc") &&
+                       (i+2) < remaining &&
+                       flags & GEOSEARCH &&
+                       !frommember)
+            {
+                if (extractLongLatOrReply(c, c->argv+base_args+i+1, shape.xy) == C_ERR) return;
+                fromloc = 1;
+                i += 2;
+            } else if (!strcasecmp(arg, "byradius") &&
+                       (i+2) < remaining &&
+                       flags & GEOSEARCH &&
+                       !bybox)
+            {
+                if (extractDistanceOrReply(c, c->argv+base_args+i+1, &shape.conversion, &shape.t.radius) != C_OK)
+                    return;
+                shape.type = CIRCULAR_TYPE;
+                byradius = 1;
+                i += 2;
+            } else if (!strcasecmp(arg, "bybox") &&
+                       (i+3) < remaining &&
+                       flags & GEOSEARCH &&
+                       !byradius)
+            {
+                if (extractBoxOrReply(c, c->argv+base_args+i+1, &shape.conversion, &shape.t.r.height,
+                        &shape.t.r.width) != C_OK) return;
+                shape.type = RECTANGLE_TYPE;
+                bybox = 1;
+                i += 3;
             } else {
                 addReply(c, shared.syntaxerr);
                 return;
@@ -552,17 +634,26 @@ void georadiusGeneric(client *c, int flags) {
         return;
     }
 
+    if ((flags & GEOSEARCH) && !(frommember || fromloc)) {
+        addReplyErrorFormat(c, "exactly one of FROMMEMBER or FROMLOC can be specified for %s", (char *)c->argv[0]->ptr);
+        return;
+    }
+
+    if ((flags & GEOSEARCH) && !(byradius || bybox)) {
+        addReplyErrorFormat(c, "exactly one of BYRADIUS and BYBOX can be specified for %s", (char *)c->argv[0]->ptr);
+        return;
+    }
+
     /* COUNT without ordering does not make much sense, force ASC
      * ordering if COUNT was specified but no sorting was requested. */
     if (count != 0 && sort == SORT_NONE) sort = SORT_ASC;
 
     /* Get all neighbor geohash boxes for our radius search */
-    GeoHashRadius georadius =
-        geohashGetAreasByRadiusWGS84(xy[0], xy[1], radius_meters);
+    GeoHashRadius georadius = geohashCalculateAreasByShapeWGS84(&shape);
 
     /* Search the zset for all matching points */
     geoArray *ga = geoArrayCreate();
-    membersOfAllNeighbors(zobj, georadius, xy[0], xy[1], radius_meters, ga);
+    membersOfAllNeighbors(zobj, georadius, &shape, ga);
 
     /* If no matching results, the user gets an empty reply. */
     if (ga->used == 0 && storekey == NULL) {
@@ -607,7 +698,7 @@ void georadiusGeneric(client *c, int flags) {
         int i;
         for (i = 0; i < returned_items; i++) {
             geoPoint *gp = ga->array+i;
-            gp->dist /= conversion; /* Fix according to unit. */
+            gp->dist /= shape.conversion; /* Fix according to unit. */
 
             /* If we have options in option_length, return each sub-result
              * as a nested multi-bulk.  Add 1 to account for result value
@@ -645,7 +736,7 @@ void georadiusGeneric(client *c, int flags) {
         for (i = 0; i < returned_items; i++) {
             zskiplistNode *znode;
             geoPoint *gp = ga->array+i;
-            gp->dist /= conversion; /* Fix according to unit. */
+            gp->dist /= shape.conversion; /* Fix according to unit. */
             double score = storedist ? gp->dist : gp->score;
             size_t elelen = sdslen(gp->member);
 
@@ -659,7 +750,7 @@ void georadiusGeneric(client *c, int flags) {
             zsetConvertToZiplistIfNeeded(zobj,maxelelen);
             setKey(c,c->db,storekey,zobj);
             decrRefCount(zobj);
-            notifyKeyspaceEvent(NOTIFY_ZSET,"georadiusstore",storekey,
+            notifyKeyspaceEvent(NOTIFY_ZSET,flags & GEOSEARCH ? "geosearchstore" : "georadiusstore",storekey,
                                 c->db->id);
             server.dirty += returned_items;
         } else if (dbDelete(c->db,storekey)) {
@@ -674,22 +765,30 @@ void georadiusGeneric(client *c, int flags) {
 
 /* GEORADIUS wrapper function. */
 void georadiusCommand(client *c) {
-    georadiusGeneric(c, RADIUS_COORDS);
+    georadiusGeneric(c, 1, RADIUS_COORDS);
 }
 
 /* GEORADIUSBYMEMBER wrapper function. */
 void georadiusbymemberCommand(client *c) {
-    georadiusGeneric(c, RADIUS_MEMBER);
+    georadiusGeneric(c, 1, RADIUS_MEMBER);
 }
 
 /* GEORADIUS_RO wrapper function. */
 void georadiusroCommand(client *c) {
-    georadiusGeneric(c, RADIUS_COORDS|RADIUS_NOSTORE);
+    georadiusGeneric(c, 1, RADIUS_COORDS|RADIUS_NOSTORE);
 }
 
 /* GEORADIUSBYMEMBER_RO wrapper function. */
 void georadiusbymemberroCommand(client *c) {
-    georadiusGeneric(c, RADIUS_MEMBER|RADIUS_NOSTORE);
+    georadiusGeneric(c, 1, RADIUS_MEMBER|RADIUS_NOSTORE);
+}
+
+void geosearchCommand(client *c) {
+    georadiusGeneric(c, 1, GEOSEARCH);
+}
+
+void geosearchstoreCommand(client *c) {
+    georadiusGeneric(c, 2, GEOSEARCH|GEOSEARCHSTORE);
 }
 
 /* GEOHASH key ele1 ele2 ... eleN
diff --git a/src/geohash.h b/src/geohash.h
index ed2ef9336..ebe52ff34 100644
--- a/src/geohash.h
+++ b/src/geohash.h
@@ -90,6 +90,25 @@ typedef struct {
     GeoHashBits south_west;
 } GeoHashNeighbors;
 
+#define CIRCULAR_TYPE 1
+#define RECTANGLE_TYPE 2
+typedef struct {
+    int type; /* search type */
+    double xy[2]; /* search center point, xy[0]: lon, xy[1]: lat */
+    double conversion; /* km: 1000 */
+    double bounds[4]; /* bounds[0]: min_lon, bounds[1]: min_lat
+                       * bounds[2]: max_lon, bounds[3]: max_lat */
+    union {
+        /* CIRCULAR_TYPE */
+        double radius;
+        /* RECTANGLE_TYPE */
+        struct {
+            double height;
+            double width;
+        } r;
+    } t;
+} GeoShape;
+
 /*
  * 0:success
  * -1:failed
diff --git a/src/geohash_helper.c b/src/geohash_helper.c
index 01fb2cb88..a1ddf5d31 100644
--- a/src/geohash_helper.c
+++ b/src/geohash_helper.c
@@ -82,10 +82,9 @@ uint8_t geohashEstimateStepsByRadius(double range_meters, double lat) {
     return step;
 }
 
-/* Return the bounding box of the search area centered at latitude,longitude
- * having a radius of radius_meter. bounds[0] - bounds[2] is the minimum
- * and maximum longitude, while bounds[1] - bounds[3] is the minimum and
- * maximum latitude.
+/* Return the bounding box of the search area by shape (see geohash.h GeoShape)
+ * bounds[0] - bounds[2] is the minimum and maximum longitude
+ * while bounds[1] - bounds[3] is the minimum and maximum latitude.
  *
  * This function does not behave correctly with very large radius values, for
  * instance for the coordinates 81.634948934258375 30.561509253718668 and a
@@ -100,34 +99,51 @@ uint8_t geohashEstimateStepsByRadius(double range_meters, double lat) {
  * Since this function is currently only used as an optimization, the
  * optimization is not used for very big radiuses, however the function
  * should be fixed. */
-int geohashBoundingBox(double longitude, double latitude, double radius_meters,
-                       double *bounds) {
+int geohashBoundingBox(GeoShape *shape, double *bounds) {
     if (!bounds) return 0;
+    double longitude = shape->xy[0];
+    double latitude = shape->xy[1];
+    double height = shape->conversion * (shape->type == CIRCULAR_TYPE ? shape->t.radius : shape->t.r.height/2);
+    double width = shape->conversion * (shape->type == CIRCULAR_TYPE ? shape->t.radius : shape->t.r.width/2);
 
-    bounds[0] = longitude - rad_deg(radius_meters/EARTH_RADIUS_IN_METERS/cos(deg_rad(latitude)));
-    bounds[2] = longitude + rad_deg(radius_meters/EARTH_RADIUS_IN_METERS/cos(deg_rad(latitude)));
-    bounds[1] = latitude - rad_deg(radius_meters/EARTH_RADIUS_IN_METERS);
-    bounds[3] = latitude + rad_deg(radius_meters/EARTH_RADIUS_IN_METERS);
+    const double long_delta = rad_deg(width/EARTH_RADIUS_IN_METERS/cos(deg_rad(latitude)));
+    const double lat_delta = rad_deg(height/EARTH_RADIUS_IN_METERS);
+    bounds[0] = longitude - long_delta;
+    bounds[2] = longitude + long_delta;
+    bounds[1] = latitude - lat_delta;
+    bounds[3] = latitude + lat_delta;
     return 1;
 }
 
-/* Return a set of areas (center + 8) that are able to cover a range query
- * for the specified position and radius. */
-GeoHashRadius geohashGetAreasByRadius(double longitude, double latitude, double radius_meters) {
+/* Calculate a set of areas (center + 8) that are able to cover a range query
+ * for the specified position and shape (see geohash.h GeoShape).
+ * the bounding box saved in shaple.bounds */
+GeoHashRadius geohashCalculateAreasByShapeWGS84(GeoShape *shape) {
     GeoHashRange long_range, lat_range;
     GeoHashRadius radius;
     GeoHashBits hash;
     GeoHashNeighbors neighbors;
     GeoHashArea area;
     double min_lon, max_lon, min_lat, max_lat;
-    double bounds[4];
     int steps;
 
-    geohashBoundingBox(longitude, latitude, radius_meters, bounds);
-    min_lon = bounds[0];
-    min_lat = bounds[1];
-    max_lon = bounds[2];
-    max_lat = bounds[3];
+    geohashBoundingBox(shape, shape->bounds);
+    min_lon = shape->bounds[0];
+    min_lat = shape->bounds[1];
+    max_lon = shape->bounds[2];
+    max_lat = shape->bounds[3];
+
+    double longitude = shape->xy[0];
+    double latitude = shape->xy[1];
+    /* radius_meters is calculated differently in different search types:
+     * 1) CIRCULAR_TYPE, just use radius.
+     * 2) RECTANGLE_TYPE, in order to calculate accurately, we should use
+     * sqrt((width/2)^2 + (height/2)^2), so that the box is bound by a circle,
+     * But the current code a simpler approach resulting in a smaller circle,
+     * which is safe because we search the 8 nearby boxes anyway. */
+    double radius_meters = shape->type == CIRCULAR_TYPE ? shape->t.radius :
+                            shape->t.r.width > shape->t.r.height ? shape->t.r.width/2 : shape->t.r.height/2;
+    radius_meters *= shape->conversion;
 
     steps = geohashEstimateStepsByRadius(radius_meters,latitude);
 
@@ -196,11 +212,6 @@ GeoHashRadius geohashGetAreasByRadius(double longitude, double latitude, double
     return radius;
 }
 
-GeoHashRadius geohashGetAreasByRadiusWGS84(double longitude, double latitude,
-                                           double radius_meters) {
-    return geohashGetAreasByRadius(longitude, latitude, radius_meters);
-}
-
 GeoHashFix52Bits geohashAlign52Bits(const GeoHashBits hash) {
     uint64_t bits = hash.bits;
     bits <<= (52 - hash.step * 2);
@@ -233,3 +244,15 @@ int geohashGetDistanceIfInRadiusWGS84(double x1, double y1, double x2,
                                       double *distance) {
     return geohashGetDistanceIfInRadius(x1, y1, x2, y2, radius, distance);
 }
+
+/* Judge whether a point is in the axis-aligned rectangle.
+ * bounds : see geohash.h GeoShape::bounds
+ * x1, y1 : the center of the box
+ * x2, y2 : the point to be searched
+ */
+int geohashGetDistanceIfInRectangle(double *bounds, double x1, double y1,
+                                    double x2, double y2, double *distance) {
+    if (x2 < bounds[0] || x2 > bounds[2] || y2 < bounds[1] || y2 > bounds[3]) return 0;
+    *distance = geohashGetDistance(x1, y1, x2, y2);
+    return 1;
+}
diff --git a/src/geohash_helper.h b/src/geohash_helper.h
index eb0dda38a..f28896a8d 100644
--- a/src/geohash_helper.h
+++ b/src/geohash_helper.h
@@ -49,14 +49,8 @@ typedef struct {
 
 int GeoHashBitsComparator(const GeoHashBits *a, const GeoHashBits *b);
 uint8_t geohashEstimateStepsByRadius(double range_meters, double lat);
-int geohashBoundingBox(double longitude, double latitude, double radius_meters,
-                        double *bounds);
-GeoHashRadius geohashGetAreasByRadius(double longitude,
-                                      double latitude, double radius_meters);
-GeoHashRadius geohashGetAreasByRadiusWGS84(double longitude, double latitude,
-                                           double radius_meters);
-GeoHashRadius geohashGetAreasByRadiusMercator(double longitude, double latitude,
-                                              double radius_meters);
+int geohashBoundingBox(GeoShape *shape, double *bounds);
+GeoHashRadius geohashCalculateAreasByShapeWGS84(GeoShape *shape);
 GeoHashFix52Bits geohashAlign52Bits(const GeoHashBits hash);
 double geohashGetDistance(double lon1d, double lat1d,
                           double lon2d, double lat2d);
@@ -66,5 +60,7 @@ int geohashGetDistanceIfInRadius(double x1, double y1,
 int geohashGetDistanceIfInRadiusWGS84(double x1, double y1, double x2,
                                       double y2, double radius,
                                       double *distance);
+int geohashGetDistanceIfInRectangle(double *bounds, double x1, double y1,
+                                    double x2, double y2, double *distance);
 
 #endif /* GEOHASH_HELPER_HPP_ */
diff --git a/src/server.c b/src/server.c
index 0c3bbff03..e4d236874 100644
--- a/src/server.c
+++ b/src/server.c
@@ -919,7 +919,7 @@ struct redisCommand redisCommandTable[] = {
 
     /* GEORADIUS has store options that may write. */
     {"georadius",georadiusCommand,-6,
-     "write @geo",
+     "write use-memory @geo",
      0,georadiusGetKeys,1,1,1,0,0,0},
 
     {"georadius_ro",georadiusroCommand,-6,
@@ -927,7 +927,7 @@ struct redisCommand redisCommandTable[] = {
      0,georadiusGetKeys,1,1,1,0,0,0},
 
     {"georadiusbymember",georadiusbymemberCommand,-5,
-     "write @geo",
+     "write use-memory @geo",
      0,georadiusGetKeys,1,1,1,0,0,0},
 
     {"georadiusbymember_ro",georadiusbymemberroCommand,-5,
@@ -946,6 +946,14 @@ struct redisCommand redisCommandTable[] = {
      "read-only @geo",
      0,NULL,1,1,1,0,0,0},
 
+    {"geosearch",geosearchCommand,-7,
+     "read-only @geo",
+      0,NULL,1,1,1,0,0,0},
+
+    {"geosearchstore",geosearchstoreCommand,-8,
+     "write use-memory @geo",
+      0,NULL,1,2,1,0,0,0},
+
     {"pfselftest",pfselftestCommand,1,
      "admin @hyperloglog",
       0,NULL,0,0,0,0,0,0},
diff --git a/src/server.h b/src/server.h
index 35b695a42..4cb5ab855 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2546,6 +2546,8 @@ void geoaddCommand(client *c);
 void geohashCommand(client *c);
 void geoposCommand(client *c);
 void geodistCommand(client *c);
+void geosearchCommand(client *c);
+void geosearchstoreCommand(client *c);
 void pfselftestCommand(client *c);
 void pfaddCommand(client *c);
 void pfcountCommand(client *c);
diff --git a/tests/unit/geo.tcl b/tests/unit/geo.tcl
index 49e421ee9..b3e59d9b2 100644
--- a/tests/unit/geo.tcl
+++ b/tests/unit/geo.tcl
@@ -94,10 +94,43 @@ start_server {tags {"geo"}} {
         r georadius nyc -73.9798091 40.7598464 3 km asc
     } {{central park n/q/r} 4545 {union square}}
 
+    test {GEOSEARCH simple (sorted)} {
+        r geosearch nyc fromloc -73.9798091 40.7598464 bybox 6 6 km asc
+    } {{central park n/q/r} 4545 {union square} {lic market}}
+
+    test {GEOSEARCH FROMLOC and FROMMEMBER cannot exist at the same time} {
+        catch {r geosearch nyc fromloc -73.9798091 40.7598464 frommember xxx bybox 6 6 km asc} e
+        set e
+    } {ERR*syntax*}
+
+    test {GEOSEARCH FROMLOC and FROMMEMBER one must exist} {
+        catch {r geosearch nyc bybox 3 3 km asc desc withhash withdist withcoord} e
+        set e
+    } {ERR*exactly one of FROMMEMBER or FROMLOC*}
+
+    test {GEOSEARCH BYRADIUS and BYBOX cannot exist at the same time} {
+        catch {r geosearch nyc fromloc -73.9798091 40.7598464 byradius 3 km bybox 3 3 km asc} e
+        set e
+    } {ERR*syntax*}
+
+    test {GEOSEARCH BYRADIUS and BYBOX one must exist} {
+        catch {r geosearch nyc fromloc -73.9798091 40.7598464 asc desc withhash withdist withcoord} e
+        set e
+    } {ERR*exactly one of BYRADIUS and BYBOX*}
+
+    test {GEOSEARCH with STOREDIST option} {
+        catch {r geosearch nyc fromloc -73.9798091 40.7598464 bybox 6 6 km asc storedist} e
+        set e
+    } {ERR*syntax*}
+
     test {GEORADIUS withdist (sorted)} {
         r georadius nyc -73.9798091 40.7598464 3 km withdist asc
     } {{{central park n/q/r} 0.7750} {4545 2.3651} {{union square} 2.7697}}
 
+    test {GEOSEARCH withdist (sorted)} {
+        r geosearch nyc fromloc -73.9798091 40.7598464 bybox 6 6 km withdist asc
+    } {{{central park n/q/r} 0.7750} {4545 2.3651} {{union square} 2.7697} {{lic market} 3.1991}}
+
     test {GEORADIUS with COUNT} {
         r georadius nyc -73.9798091 40.7598464 10 km COUNT 3
     } {{central park n/q/r} 4545 {union square}}
@@ -120,6 +153,35 @@ start_server {tags {"geo"}} {
         r georadiusbymember nyc "wtc one" 7 km
     } {{wtc one} {union square} {central park n/q/r} 4545 {lic market}}
 
+    test {GEOSEARCH FROMMEMBER simple (sorted)} {
+        r geosearch nyc frommember "wtc one" bybox 14 14 km
+    } {{wtc one} {union square} {central park n/q/r} 4545 {lic market} q4}
+
+    test {GEOSEARCH vs GEORADIUS} {
+        r del Sicily
+        r geoadd Sicily 13.361389 38.115556 "Palermo" 15.087269 37.502669 "Catania"
+        r geoadd Sicily 12.758489 38.788135 "edge1"   17.241510 38.788135 "eage2"
+        set ret1 [r georadius Sicily 15 37 200 km asc]
+        assert_equal $ret1 {Catania Palermo}
+        set ret2 [r geosearch Sicily fromloc 15 37 bybox 400 400 km asc]
+        assert_equal $ret2 {Catania Palermo eage2 edge1}
+    }
+
+    test {GEOSEARCH non square, long and narrow} {
+        r del Sicily
+        r geoadd Sicily 12.75 37.00 "test1"
+        r geoadd Sicily 12.75 36.50 "test2"
+        r geoadd Sicily 13.00 36.50 "test3"
+        # box height=2km width=400km
+        set ret1 [r geosearch Sicily fromloc 15 37 bybox 2 400 km]
+        assert_equal $ret1 {test1}
+
+        # Add a western Hemisphere point
+        r geoadd Sicily -1 37.00 "test3"
+        set ret2 [r geosearch Sicily fromloc 15 37 bybox 2 3000 km asc]
+        assert_equal $ret2 {test1 test3}
+    }
+
     test {GEORADIUSBYMEMBER withdist (sorted)} {
         r georadiusbymember nyc "wtc one" 7 km withdist
     } {{{wtc one} 0.0000} {{union square} 3.2544} {{central park n/q/r} 6.7000} {4545 6.1975} {{lic market} 6.8969}}
@@ -178,6 +240,11 @@ start_server {tags {"geo"}} {
         set e
     } {*ERR*syntax*}
 
+    test {GEOSEARCHSTORE STORE option: syntax error} {
+        catch {r geosearchstore abc points fromloc 13.361389 38.115556 byradius 50 km store abc} e
+        set e
+    } {*ERR*syntax*}
+
     test {GEORANGE STORE option: incompatible options} {
         r del points
         r geoadd points 13.361389 38.115556 "Palermo" \
@@ -198,6 +265,11 @@ start_server {tags {"geo"}} {
         assert_equal [r zrange points 0 -1] [r zrange points2 0 -1]
     }
 
+    test {GEOSEARCHSTORE STORE option: plain usage} {
+        r geosearchstore points2 points fromloc 13.361389 38.115556 byradius 500 km
+        assert_equal [r zrange points 0 -1] [r zrange points2 0 -1]
+    }
+
     test {GEORANGE STOREDIST option: plain usage} {
         r del points
         r geoadd points 13.361389 38.115556 "Palermo" \
@@ -209,6 +281,14 @@ start_server {tags {"geo"}} {
         assert {[lindex $res 3] < 167}
     }
 
+    test {GEOSEARCHSTORE STOREDIST option: plain usage} {
+        r geosearchstore points2 points fromloc 13.361389 38.115556 byradius 500 km storedist
+        set res [r zrange points2 0 -1 withscores]
+        assert {[lindex $res 1] < 1}
+        assert {[lindex $res 3] > 166}
+        assert {[lindex $res 3] < 167}
+    }
+
     test {GEORANGE STOREDIST option: COUNT ASC and DESC} {
         r del points
         r geoadd points 13.361389 38.115556 "Palermo" \

From feba7cbf4d75f7822f34bd67a56b2ef4a755eff9 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Sat, 12 Dec 2020 17:31:40 +0200
Subject: [PATCH 0865/1098] Adds 'use-memory' to GEORADIUS[BYMEMBER] (#8107)

Partial resolution for #6860, item 7

From ddd43b6bc3ea8560bae1b0eead730235ffe7eafd Mon Sep 17 00:00:00 2001
From: gourav <gourav.bit@gmail.com>
Date: Sat, 12 Dec 2020 17:27:35 -0800
Subject: [PATCH 0866/1098] Randomize the random number generator's seed used
 in redis-benchmark (#8174)

The pid of the benchmark process is used to randomize the random number generator's
seed. This ensures that when multiple benchmark processes are started at the same time
to generate load on a server, they use different seeds. This will ensure randomness in
the keys generated by different benchmark processes.
---
 src/redis-benchmark.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 5567c4af5..4efed4b12 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -1676,7 +1676,7 @@ int main(int argc, const char **argv) {
 
     client c;
 
-    srandom(time(NULL));
+    srandom(time(NULL) ^ getpid());
     signal(SIGHUP, SIG_IGN);
     signal(SIGPIPE, SIG_IGN);
 

From 63c1303cfb60140dec6531ea2965fd709d316094 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 13 Dec 2020 09:56:01 +0200
Subject: [PATCH 0867/1098] Modules: add defrag API support. (#8149)

Add a new set of defrag functions that take a defrag context and allow
defragmenting memory blocks and RedisModuleStrings.

Modules can register a defrag callback which will be invoked when the
defrag process handles globals.

Modules with custom data types can also register a datatype-specific
defrag callback which is invoked for keys that require defragmentation.
The callback and associated functions support both one-step and
multi-step options, depending on the complexity of the key as exposed by
the free_effort callback.
---
 runtest-moduleapi               |   1 +
 src/defrag.c                    |  31 ++++-
 src/module.c                    | 232 ++++++++++++++++++++++++++++++-
 src/redismodule.h               |  17 +++
 src/server.h                    |   8 ++
 tests/modules/Makefile          |   1 +
 tests/modules/defragtest.c      | 234 ++++++++++++++++++++++++++++++++
 tests/support/util.tcl          |  10 +-
 tests/unit/moduleapi/defrag.tcl |  43 ++++++
 9 files changed, 569 insertions(+), 8 deletions(-)
 create mode 100644 tests/modules/defragtest.c
 create mode 100644 tests/unit/moduleapi/defrag.tcl

diff --git a/runtest-moduleapi b/runtest-moduleapi
index fdf60886f..da4c815da 100755
--- a/runtest-moduleapi
+++ b/runtest-moduleapi
@@ -29,4 +29,5 @@ $TCLSH tests/test_helper.tcl \
 --single unit/moduleapi/blockedclient \
 --single unit/moduleapi/getkeys \
 --single unit/moduleapi/test_lazyfree \
+--single unit/moduleapi/defrag \
 "${@}"
diff --git a/src/defrag.c b/src/defrag.c
index 165e55f65..fc327c506 100644
--- a/src/defrag.c
+++ b/src/defrag.c
@@ -794,6 +794,20 @@ long defragStream(redisDb *db, dictEntry *kde) {
     return defragged;
 }
 
+/* Defrag a module key. This is either done immediately or scheduled
+ * for later. Returns then number of pointers defragged.
+ */
+long defragModule(redisDb *db, dictEntry *kde) {
+    robj *obj = dictGetVal(kde);
+    serverAssert(obj->type == OBJ_MODULE);
+    long defragged = 0;
+
+    if (!moduleDefragValue(dictGetKey(kde), obj, &defragged))
+        defragLater(db, kde);
+
+    return defragged;
+}
+
 /* for each key we scan in the main dict, this function will attempt to defrag
  * all the various pointers it has. Returns a stat of how many pointers were
  * moved. */
@@ -865,8 +879,7 @@ long defragKey(redisDb *db, dictEntry *de) {
     } else if (ob->type == OBJ_STREAM) {
         defragged += defragStream(db, de);
     } else if (ob->type == OBJ_MODULE) {
-        /* Currently defragmenting modules private data types
-         * is not supported. */
+        defragged += defragModule(db, de);
     } else {
         serverPanic("Unknown object type");
     }
@@ -928,6 +941,7 @@ long defragOtherGlobals() {
      * that remain static for a long time */
     defragged += activeDefragSdsDict(server.lua_scripts, DEFRAG_SDS_DICT_VAL_IS_STROB);
     defragged += activeDefragSdsListAndDict(server.repl_scriptcache_fifo, server.repl_scriptcache_dict, DEFRAG_SDS_DICT_NO_VAL);
+    defragged += moduleDefragGlobals();
     return defragged;
 }
 
@@ -946,6 +960,8 @@ int defragLaterItem(dictEntry *de, unsigned long *cursor, long long endtime) {
             server.stat_active_defrag_hits += scanLaterHash(ob, cursor);
         } else if (ob->type == OBJ_STREAM) {
             return scanLaterStraemListpacks(ob, cursor, endtime, &server.stat_active_defrag_hits);
+        } else if (ob->type == OBJ_MODULE) {
+            return moduleLateDefrag(dictGetKey(de), ob, cursor, endtime, &server.stat_active_defrag_hits);
         } else {
             *cursor = 0; /* object type may have changed since we schedule it for later */
         }
@@ -1184,4 +1200,15 @@ void activeDefragCycle(void) {
     /* Not implemented yet. */
 }
 
+void *activeDefragAlloc(void *ptr) {
+    UNUSED(ptr);
+    return NULL;
+}
+
+robj *activeDefragStringOb(robj *ob, long *defragged) {
+    UNUSED(ob);
+    UNUSED(defragged);
+    return NULL;
+}
+
 #endif
diff --git a/src/module.c b/src/module.c
index be0f939f2..94bdbf641 100644
--- a/src/module.c
+++ b/src/module.c
@@ -50,6 +50,7 @@ typedef struct RedisModuleInfoCtx {
 } RedisModuleInfoCtx;
 
 typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report);
+typedef void (*RedisModuleDefragFunc)(struct RedisModuleDefragCtx *ctx);
 
 /* This structure represents a module inside the system. */
 struct RedisModule {
@@ -66,6 +67,7 @@ struct RedisModule {
     int options;    /* Module options and capabilities. */
     int blocked_clients;         /* Count of RedisModuleBlockedClient in this module. */
     RedisModuleInfoFunc info_cb; /* Callback for module to add INFO fields. */
+    RedisModuleDefragFunc defrag_cb;    /* Callback for global data defrag. */
 };
 typedef struct RedisModule RedisModule;
 
@@ -875,6 +877,7 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api
     module->in_hook = 0;
     module->options = 0;
     module->info_cb = 0;
+    module->defrag_cb = 0;
     ctx->module = module;
 }
 
@@ -3694,9 +3697,10 @@ robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, robj *value) {
  *          .mem_usage = myType_MemUsageCallBack,
  *          .aux_load = myType_AuxRDBLoadCallBack,
  *          .aux_save = myType_AuxRDBSaveCallBack,
- *          .free_effort = myType_FreeEffortCallBack
- *          .unlink = myType_UnlinkCallBack
- *          .copy = myType_CopyCallback
+ *          .free_effort = myType_FreeEffortCallBack,
+ *          .unlink = myType_UnlinkCallBack,
+ *          .copy = myType_CopyCallback,
+ *          .defrag = myType_DefragCallback
  *      }
  *
  * * **rdb_load**: A callback function pointer that loads data from RDB files.
@@ -3723,6 +3727,21 @@ robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, robj *value) {
  *   Note: if the target key exists and is being overwritten, the copy callback will be
  *   called first, followed by a free callback to the value that is being replaced.
  * 
+ * * **defrag**: A callback function pointer that is used to request the module to defrag
+ *   a key. The module should then iterate pointers and call the relevant RM_Defrag*()
+ *   functions to defragment pointers or complex types. The module should continue
+ *   iterating as long as RM_DefragShouldStop() returns a zero value, and return a
+ *   zero value if finished or non-zero value if more work is left to be done. If more work
+ *   needs to be done, RM_DefragCursorSet() and RM_DefragCursorGet() can be used to track
+ *   this work across different calls.
+ *   Normally, the defrag mechanism invokes the callback without a time limit, so
+ *   RM_DefragShouldStop() always returns zero. The "late defrag" mechanism which has
+ *   a time limit and provides cursor support is used only for keys that are determined
+ *   to have significant internal complexity. To determine this, the defrag mechanism
+ *   uses the free_effort callback and the 'active-defrag-max-scan-fields' config directive.
+ *   NOTE: The value is passed as a void** and the function is expected to update the
+ *   pointer if the top-level value pointer is defragmented and consequentially changes.
+ *
  * Note: the module name "AAAAAAAAA" is reserved and produces an error, it
  * happens to be pretty lame as well.
  *
@@ -3766,6 +3785,7 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver,
             moduleTypeFreeEffortFunc free_effort;
             moduleTypeUnlinkFunc unlink;
             moduleTypeCopyFunc copy;
+            moduleTypeDefragFunc defrag;
         } v3;
     } *tms = (struct typemethods*) typemethods_ptr;
 
@@ -3787,6 +3807,7 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver,
         mt->free_effort = tms->v3.free_effort;
         mt->unlink = tms->v3.unlink;
         mt->copy = tms->v3.copy;
+        mt->defrag = tms->v3.defrag;
     }
     memcpy(mt->name,name,sizeof(mt->name));
     listAddNodeTail(ctx->module->types,mt);
@@ -8165,6 +8186,205 @@ int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc,
     return res;
 }
 
+/* The defrag context, used to manage state during calls to the data type
+ * defrag callback.
+ */
+typedef struct RedisModuleDefragCtx {
+    long defragged;
+    long long int endtime;
+    unsigned long *cursor;
+} RedisModuleDefragCtx;
+
+/* Register a defrag callback for global data, i.e. anything that the module
+ * may allocate that is not tied to a specific data type.
+ */
+int RM_RegisterDefragFunc(RedisModuleCtx *ctx, RedisModuleDefragFunc cb) {
+    ctx->module->defrag_cb = cb;
+    return REDISMODULE_OK;
+}
+
+/* When the data type defrag callback iterates complex structures, this
+ * function should be called periodically. A zero (false) return
+ * indicates the callback may continue its work. A non-zero value (true)
+ * indicates it should stop.
+ *
+ * When stopped, the callback may use RM_DefragCursorSet() to store its
+ * position so it can later use RM_DefragCursorGet() to resume defragging.
+ *
+ * When stopped and more work is left to be done, the callback should
+ * return 1. Otherwise, it should return 0.
+ *
+ * NOTE: Modules should consider the frequency in which this function is called,
+ * so it generally makes sense to do small batches of work in between calls.
+ */
+int RM_DefragShouldStop(RedisModuleDefragCtx *ctx) {
+    return (ctx->endtime != 0 && ctx->endtime < ustime());
+}
+
+/* Store an arbitrary cursor value for future re-use.
+ *
+ * This should only be called if RM_DefragShouldStop() has returned a non-zero
+ * value and the defrag callback is about to exit without fully iterating its
+ * data type.
+ *
+ * This behavior is reserved to cases where late defrag is performed. Late
+ * defrag is selected for keys that implement the free_effort callback and
+ * return a free_effort value that is larger than the defrag
+ * 'active-defrag-max-scan-fields' configuration directive.
+ *
+ * Smaller keys, keys that do not implement free_effort or the global
+ * defrag callback are not called in late-defrag mode. In those cases, a
+ * call to this function will return REDISMODULE_ERR.
+ *
+ * The cursor may be used by the module to represent some progress into the
+ * module's data type. Modules may also store additional cursor-related
+ * information locally and use the cursor as a flag that indicates when
+ * traversal of a new key begins. This is possible because the API makes
+ * a guarantee that concurrent defragmentation of multiple keys will
+ * not be performed.
+ */
+int RM_DefragCursorSet(RedisModuleDefragCtx *ctx, unsigned long cursor) {
+    if (!ctx->cursor)
+        return REDISMODULE_ERR;
+
+    *ctx->cursor = cursor;
+    return REDISMODULE_OK;
+}
+
+/* Fetch a cursor value that has been previously stored using RM_DefragCursorSet().
+ *
+ * If not called for a late defrag operation, REDISMODULE_ERR will be returned and
+ * the cursor should be ignored. See DM_DefragCursorSet() for more details on
+ * defrag cursors.
+ */
+int RM_DefragCursorGet(RedisModuleDefragCtx *ctx, unsigned long *cursor) {
+    if (!ctx->cursor)
+        return REDISMODULE_ERR;
+
+    *cursor = *ctx->cursor;
+    return REDISMODULE_OK;
+}
+
+/* Defrag a memory allocation previously allocated by RM_Alloc, RM_Calloc, etc.
+ * The defragmentation process involves allocating a new memory block and copying
+ * the contents to it, like realloc().
+ *
+ * If defragmentation was not necessary, NULL is returned and the operation has
+ * no other effect.
+ *
+ * If a non-NULL value is returned, the caller should use the new pointer instead
+ * of the old one and update any reference to the old pointer, which must not
+ * be used again.
+ */
+void *RM_DefragAlloc(RedisModuleDefragCtx *ctx, void *ptr) {
+    void *newptr = activeDefragAlloc(ptr);
+    if (newptr)
+        ctx->defragged++;
+
+    return newptr;
+}
+
+/* Defrag a RedisModuleString previously allocated by RM_Alloc, RM_Calloc, etc.
+ * See RM_DefragAlloc() for more information on how the defragmentation process
+ * works.
+ *
+ * NOTE: It is only possible to defrag strings that have a single reference.
+ * Typically this means strings retained with RM_RetainString or RM_HoldString
+ * may not be defragmentable. One exception is command argvs which, if retained
+ * by the module, will end up with a single reference (because the reference
+ * on the Redis side is dropped as soon as the command callback returns).
+ */
+RedisModuleString *RM_DefragRedisModuleString(RedisModuleDefragCtx *ctx, RedisModuleString *str) {
+    return activeDefragStringOb(str, &ctx->defragged);
+}
+
+
+/* Perform a late defrag of a module datatype key.
+ *
+ * Returns a zero value (and initializes the cursor) if no more needs to be done,
+ * or a non-zero value otherwise.
+ */
+int moduleLateDefrag(robj *key, robj *value, unsigned long *cursor, long long endtime, long long *defragged) {
+    moduleValue *mv = value->ptr;
+    moduleType *mt = mv->type;
+
+    RedisModuleDefragCtx defrag_ctx = { 0, endtime, cursor };
+
+    /* Invoke callback. Note that the callback may be missing if the key has been
+     * replaced with a different type since our last visit.
+     */
+    int ret = 0;
+    if (mt->defrag)
+        ret = mt->defrag(&defrag_ctx, key, &mv->value);
+
+    *defragged += defrag_ctx.defragged;
+    if (!ret) {
+        *cursor = 0;    /* No more work to do */
+        return 0;
+    }
+
+    return 1;
+}
+
+/* Attempt to defrag a module data type value. Depending on complexity,
+ * the operation may happen immediately or be scheduled for later.
+ *
+ * Returns 1 if the operation has been completed or 0 if it needs to
+ * be scheduled for late defrag.
+ */
+int moduleDefragValue(robj *key, robj *value, long *defragged) {
+    moduleValue *mv = value->ptr;
+    moduleType *mt = mv->type;
+
+    /* Try to defrag moduleValue itself regardless of whether or not
+     * defrag callbacks are provided.
+     */
+    moduleValue *newmv = activeDefragAlloc(mv);
+    if (newmv) {
+        (*defragged)++;
+        value->ptr = mv = newmv;
+    }
+
+    if (!mt->defrag)
+        return 1;
+
+    /* Use free_effort to determine complexity of module value, and if
+     * necessary schedule it for defragLater instead of quick immediate
+     * defrag.
+     */
+    if (mt->free_effort) {
+        size_t effort = mt->free_effort(key, mv->value);
+        if (!effort)
+            effort = SIZE_MAX;
+        if (effort > server.active_defrag_max_scan_fields) {
+            return 0;  /* Defrag later */
+        }
+    }
+
+    RedisModuleDefragCtx defrag_ctx = { 0, 0, NULL };
+    mt->defrag(&defrag_ctx, key, &mv->value);
+    (*defragged) += defrag_ctx.defragged;
+    return 1;
+}
+
+/* Call registered module API defrag functions */
+long moduleDefragGlobals(void) {
+    dictIterator *di = dictGetIterator(modules);
+    dictEntry *de;
+    long defragged = 0;
+
+    while ((de = dictNext(di)) != NULL) {
+        struct RedisModule *module = dictGetVal(de);
+        if (!module->defrag_cb)
+            continue;
+        RedisModuleDefragCtx defrag_ctx = { 0, 0, NULL };
+        module->defrag_cb(&defrag_ctx);
+        defragged += defrag_ctx.defragged;
+    }
+
+    return defragged;
+}
+
 /* Register all the APIs we export. Keep this function at the end of the
  * file so that's easy to seek it to add new entries. */
 void moduleRegisterCoreAPI(void) {
@@ -8408,4 +8628,10 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(GetClientCertificate);
     REGISTER_API(GetCommandKeys);
     REGISTER_API(GetTypeMethodVersion);
+    REGISTER_API(RegisterDefragFunc);
+    REGISTER_API(DefragAlloc);
+    REGISTER_API(DefragRedisModuleString);
+    REGISTER_API(DefragShouldStop);
+    REGISTER_API(DefragCursorSet);
+    REGISTER_API(DefragCursorGet);
 }
diff --git a/src/redismodule.h b/src/redismodule.h
index e4e8b2b3a..d7d4b7689 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -225,6 +225,7 @@ typedef struct RedisModuleEvent {
 } RedisModuleEvent;
 
 struct RedisModuleCtx;
+struct RedisModuleDefragCtx;
 typedef void (*RedisModuleEventCallback)(struct RedisModuleCtx *ctx, RedisModuleEvent eid, uint64_t subevent, void *data);
 
 static const RedisModuleEvent
@@ -479,6 +480,7 @@ typedef struct RedisModuleCommandFilter RedisModuleCommandFilter;
 typedef struct RedisModuleInfoCtx RedisModuleInfoCtx;
 typedef struct RedisModuleServerInfoData RedisModuleServerInfoData;
 typedef struct RedisModuleScanCursor RedisModuleScanCursor;
+typedef struct RedisModuleDefragCtx RedisModuleDefragCtx;
 typedef struct RedisModuleUser RedisModuleUser;
 
 typedef int (*RedisModuleCmdFunc)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc);
@@ -495,6 +497,7 @@ typedef void (*RedisModuleTypeFreeFunc)(void *value);
 typedef size_t (*RedisModuleTypeFreeEffortFunc)(RedisModuleString *key, const void *value);
 typedef void (*RedisModuleTypeUnlinkFunc)(RedisModuleString *key, const void *value);
 typedef void *(*RedisModuleTypeCopyFunc)(RedisModuleString *fromkey, RedisModuleString *tokey, const void *value);
+typedef int (*RedisModuleTypeDefragFunc)(RedisModuleDefragCtx *ctx, RedisModuleString *key, void **value);
 typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len);
 typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data);
 typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter);
@@ -503,6 +506,7 @@ typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_repor
 typedef void (*RedisModuleScanCB)(RedisModuleCtx *ctx, RedisModuleString *keyname, RedisModuleKey *key, void *privdata);
 typedef void (*RedisModuleScanKeyCB)(RedisModuleKey *key, RedisModuleString *field, RedisModuleString *value, void *privdata);
 typedef void (*RedisModuleUserChangedFunc) (uint64_t client_id, void *privdata);
+typedef int (*RedisModuleDefragFunc)(RedisModuleDefragCtx *ctx);
 
 typedef struct RedisModuleTypeMethods {
     uint64_t version;
@@ -518,6 +522,7 @@ typedef struct RedisModuleTypeMethods {
     RedisModuleTypeFreeEffortFunc free_effort;
     RedisModuleTypeUnlinkFunc unlink;
     RedisModuleTypeCopyFunc copy;
+    RedisModuleTypeDefragFunc defrag;
 } RedisModuleTypeMethods;
 
 #define REDISMODULE_GET_API(name) \
@@ -776,6 +781,12 @@ REDISMODULE_API int (*RedisModule_AuthenticateClientWithUser)(RedisModuleCtx *ct
 REDISMODULE_API int (*RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id) REDISMODULE_ATTR;
 REDISMODULE_API RedisModuleString * (*RedisModule_GetClientCertificate)(RedisModuleCtx *ctx, uint64_t id) REDISMODULE_ATTR;
 REDISMODULE_API int *(*RedisModule_GetCommandKeys)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_RegisterDefragFunc)(RedisModuleCtx *ctx, RedisModuleDefragFunc func) REDISMODULE_ATTR;
+REDISMODULE_API void *(*RedisModule_DefragAlloc)(RedisModuleDefragCtx *ctx, void *ptr) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString *(*RedisModule_DefragRedisModuleString)(RedisModuleDefragCtx *ctx, RedisModuleString *str) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_DefragShouldStop)(RedisModuleDefragCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_DefragCursorSet)(RedisModuleDefragCtx *ctx, unsigned long cursor) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_DefragCursorGet)(RedisModuleDefragCtx *ctx, unsigned long *cursor) REDISMODULE_ATTR;
 #endif
 
 #define RedisModule_IsAOFClient(id) ((id) == CLIENT_ID_AOF)
@@ -1025,6 +1036,12 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(AuthenticateClientWithUser);
     REDISMODULE_GET_API(GetClientCertificate);
     REDISMODULE_GET_API(GetCommandKeys);
+    REDISMODULE_GET_API(RegisterDefragFunc);
+    REDISMODULE_GET_API(DefragAlloc);
+    REDISMODULE_GET_API(DefragRedisModuleString);
+    REDISMODULE_GET_API(DefragShouldStop);
+    REDISMODULE_GET_API(DefragCursorSet);
+    REDISMODULE_GET_API(DefragCursorGet);
 #endif
 
     if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR;
diff --git a/src/server.h b/src/server.h
index 4cb5ab855..89a25a4e0 100644
--- a/src/server.h
+++ b/src/server.h
@@ -514,6 +514,7 @@ struct RedisModuleIO;
 struct RedisModuleDigest;
 struct RedisModuleCtx;
 struct redisObject;
+struct RedisModuleDefragCtx;
 
 /* Each module type implementation should export a set of methods in order
  * to serialize and deserialize the value in the RDB file, rewrite the AOF
@@ -530,6 +531,7 @@ typedef void (*moduleTypeFreeFunc)(void *value);
 typedef size_t (*moduleTypeFreeEffortFunc)(struct redisObject *key, const void *value);
 typedef void (*moduleTypeUnlinkFunc)(struct redisObject *key, void *value);
 typedef void *(*moduleTypeCopyFunc)(struct redisObject *fromkey, struct redisObject *tokey, const void *value);
+typedef int (*moduleTypeDefragFunc)(struct RedisModuleDefragCtx *ctx, struct redisObject *key, void **value);
 
 /* This callback type is called by moduleNotifyUserChanged() every time
  * a user authenticated via the module API is associated with a different
@@ -552,6 +554,7 @@ typedef struct RedisModuleType {
     moduleTypeFreeEffortFunc free_effort;
     moduleTypeUnlinkFunc unlink;
     moduleTypeCopyFunc copy;
+    moduleTypeDefragFunc defrag;
     moduleTypeAuxLoadFunc aux_load;
     moduleTypeAuxSaveFunc aux_save;
     int aux_save_triggers;
@@ -1699,6 +1702,9 @@ int moduleClientIsBlockedOnKeys(client *c);
 void moduleNotifyUserChanged(client *c);
 void moduleNotifyKeyUnlink(robj *key, robj *val);
 robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, robj *value);
+int moduleDefragValue(robj *key, robj *obj, long *defragged);
+int moduleLateDefrag(robj *key, robj *value, unsigned long *cursor, long long endtime, long long *defragged);
+long moduleDefragGlobals(void);
 
 /* Utils */
 long long ustime(void);
@@ -2138,6 +2144,8 @@ void freeMemoryOverheadData(struct redisMemOverhead *mh);
 void checkChildrenDone(void);
 int setOOMScoreAdj(int process_class);
 void rejectCommandFormat(client *c, const char *fmt, ...);
+void *activeDefragAlloc(void *ptr);
+robj *activeDefragStringOb(robj* ob, long *defragged);
 
 #define RESTART_SERVER_NONE 0
 #define RESTART_SERVER_GRACEFULLY (1<<0)     /* Do proper shutdown. */
diff --git a/tests/modules/Makefile b/tests/modules/Makefile
index 36222fc1f..5b2c30c17 100644
--- a/tests/modules/Makefile
+++ b/tests/modules/Makefile
@@ -28,6 +28,7 @@ TEST_MODULES = \
     getkeys.so \
     test_lazyfree.so \
     timer.so \
+    defragtest.so
 
 
 .PHONY: all
diff --git a/tests/modules/defragtest.c b/tests/modules/defragtest.c
new file mode 100644
index 000000000..b63680c63
--- /dev/null
+++ b/tests/modules/defragtest.c
@@ -0,0 +1,234 @@
+/* A module that implements defrag callback mechanisms.
+ */
+
+#define REDISMODULE_EXPERIMENTAL_API
+#include "redismodule.h"
+
+static RedisModuleType *FragType;
+
+struct FragObject {
+    unsigned long len;
+    void **values;
+    int maxstep;
+};
+
+/* Make sure we get the expected cursor */
+unsigned long int last_set_cursor = 0;
+
+unsigned long int datatype_attempts = 0;
+unsigned long int datatype_defragged = 0;
+unsigned long int datatype_resumes = 0;
+unsigned long int datatype_wrong_cursor = 0;
+unsigned long int global_attempts = 0;
+unsigned long int global_defragged = 0;
+
+int global_strings_len = 0;
+RedisModuleString **global_strings = NULL;
+
+static void createGlobalStrings(RedisModuleCtx *ctx, int count)
+{
+    global_strings_len = count;
+    global_strings = RedisModule_Alloc(sizeof(RedisModuleString *) * count);
+
+    for (int i = 0; i < count; i++) {
+        global_strings[i] = RedisModule_CreateStringFromLongLong(ctx, i);
+    }
+}
+
+static int defragGlobalStrings(RedisModuleDefragCtx *ctx)
+{
+    for (int i = 0; i < global_strings_len; i++) {
+        RedisModuleString *new = RedisModule_DefragRedisModuleString(ctx, global_strings[i]);
+        global_attempts++;
+        if (new != NULL) {
+            global_strings[i] = new;
+            global_defragged++;
+        }
+    }
+
+    return 0;
+}
+
+static void FragInfo(RedisModuleInfoCtx *ctx, int for_crash_report) {
+    REDISMODULE_NOT_USED(for_crash_report);
+
+    RedisModule_InfoAddSection(ctx, "stats");
+    RedisModule_InfoAddFieldLongLong(ctx, "datatype_attempts", datatype_attempts);
+    RedisModule_InfoAddFieldLongLong(ctx, "datatype_defragged", datatype_defragged);
+    RedisModule_InfoAddFieldLongLong(ctx, "datatype_resumes", datatype_resumes);
+    RedisModule_InfoAddFieldLongLong(ctx, "datatype_wrong_cursor", datatype_wrong_cursor);
+    RedisModule_InfoAddFieldLongLong(ctx, "global_attempts", global_attempts);
+    RedisModule_InfoAddFieldLongLong(ctx, "global_defragged", global_defragged);
+}
+
+struct FragObject *createFragObject(unsigned long len, unsigned long size, int maxstep) {
+    struct FragObject *o = RedisModule_Alloc(sizeof(*o));
+    o->len = len;
+    o->values = RedisModule_Alloc(sizeof(RedisModuleString*) * len);
+    o->maxstep = maxstep;
+
+    for (unsigned long i = 0; i < len; i++) {
+        o->values[i] = RedisModule_Calloc(1, size);
+    }
+
+    return o;
+}
+
+/* FRAG.RESETSTATS */
+static int fragResetStatsCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+
+    datatype_attempts = 0;
+    datatype_defragged = 0;
+    datatype_resumes = 0;
+    datatype_wrong_cursor = 0;
+    global_attempts = 0;
+    global_defragged = 0;
+
+    RedisModule_ReplyWithSimpleString(ctx, "OK");
+    return REDISMODULE_OK;
+}
+
+/* FRAG.CREATE key len size maxstep */
+static int fragCreateCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 5)
+        return RedisModule_WrongArity(ctx);
+
+    RedisModuleKey *key = RedisModule_OpenKey(ctx,argv[1],
+                                              REDISMODULE_READ|REDISMODULE_WRITE);
+    int type = RedisModule_KeyType(key);
+    if (type != REDISMODULE_KEYTYPE_EMPTY)
+    {
+        return RedisModule_ReplyWithError(ctx, "ERR key exists");
+    }
+
+    long long len;
+    if ((RedisModule_StringToLongLong(argv[2], &len) != REDISMODULE_OK)) {
+        return RedisModule_ReplyWithError(ctx, "ERR invalid len");
+    }
+
+    long long size;
+    if ((RedisModule_StringToLongLong(argv[3], &size) != REDISMODULE_OK)) {
+        return RedisModule_ReplyWithError(ctx, "ERR invalid size");
+    }
+
+    long long maxstep;
+    if ((RedisModule_StringToLongLong(argv[4], &maxstep) != REDISMODULE_OK)) {
+        return RedisModule_ReplyWithError(ctx, "ERR invalid maxstep");
+    }
+
+    struct FragObject *o = createFragObject(len, size, maxstep);
+    RedisModule_ModuleTypeSetValue(key, FragType, o);
+    RedisModule_ReplyWithSimpleString(ctx, "OK");
+    RedisModule_CloseKey(key);
+
+    return REDISMODULE_OK;
+}
+
+void FragFree(void *value) {
+    struct FragObject *o = value;
+
+    for (unsigned long i = 0; i < o->len; i++)
+        RedisModule_Free(o->values[i]);
+    RedisModule_Free(o->values);
+    RedisModule_Free(o);
+}
+
+size_t FragFreeEffort(RedisModuleString *key, const void *value) {
+    REDISMODULE_NOT_USED(key);
+
+    const struct FragObject *o = value;
+    return o->len;
+}
+
+int FragDefrag(RedisModuleDefragCtx *ctx, RedisModuleString *key, void **value) {
+    REDISMODULE_NOT_USED(key);
+    unsigned long i = 0;
+    int steps = 0;
+
+    /* Attempt to get cursor, validate it's what we're exepcting */
+    if (RedisModule_DefragCursorGet(ctx, &i) == REDISMODULE_OK) {
+        if (i > 0) datatype_resumes++;
+
+        /* Validate we're expecting this cursor */
+        if (i != last_set_cursor) datatype_wrong_cursor++;
+    } else {
+        if (last_set_cursor != 0) datatype_wrong_cursor++;
+    }
+
+    /* Attempt to defrag the object itself */
+    datatype_attempts++;
+    struct FragObject *o = RedisModule_DefragAlloc(ctx, *value);
+    if (o == NULL) {
+        /* Not defragged */
+        o = *value;
+    } else {
+        /* Defragged */
+        *value = o;
+        datatype_defragged++;
+    }
+
+    /* Deep defrag now */
+    for (; i < o->len; i++) {
+        datatype_attempts++;
+        void *new = RedisModule_DefragAlloc(ctx, o->values[i]);
+        if (new) {
+            o->values[i] = new;
+            datatype_defragged++;
+        }
+
+        if ((o->maxstep && ++steps > o->maxstep) ||
+            ((i % 64 == 0) && RedisModule_DefragShouldStop(ctx)))
+        {
+            RedisModule_DefragCursorSet(ctx, i);
+            last_set_cursor = i;
+            return 1;
+        }
+    }
+
+    last_set_cursor = 0;
+    return 0;
+}
+
+int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+
+    if (RedisModule_Init(ctx, "defragtest", 1, REDISMODULE_APIVER_1)
+        == REDISMODULE_ERR) return REDISMODULE_ERR;
+
+    if (RedisModule_GetTypeMethodVersion() < REDISMODULE_TYPE_METHOD_VERSION) {
+        return REDISMODULE_ERR;
+    }
+
+    long long glen;
+    if (argc != 1 || RedisModule_StringToLongLong(argv[0], &glen) == REDISMODULE_ERR) {
+        return REDISMODULE_ERR;
+    }
+
+    createGlobalStrings(ctx, glen);
+
+    RedisModuleTypeMethods tm = {
+            .version = REDISMODULE_TYPE_METHOD_VERSION,
+            .free = FragFree,
+            .free_effort = FragFreeEffort,
+            .defrag = FragDefrag
+    };
+
+    FragType = RedisModule_CreateDataType(ctx, "frag_type", 0, &tm);
+    if (FragType == NULL) return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx, "frag.create",
+                                  fragCreateCommand, "write deny-oom", 1, 1, 1) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx, "frag.resetstats",
+                                  fragResetStatsCommand, "write deny-oom", 1, 1, 1) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    RedisModule_RegisterInfoFunc(ctx, FragInfo);
+    RedisModule_RegisterDefragFunc(ctx, defragGlobalStrings);
+
+    return REDISMODULE_OK;
+}
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index 0807ebe45..e4b70ed20 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -46,13 +46,17 @@ proc warnings_from_file {filename} {
     join $result "\n"
 }
 
-# Return value for INFO property
-proc status {r property} {
-    if {[regexp "\r\n$property:(.*?)\r\n" [{*}$r info] _ value]} {
+proc getInfoProperty {infostr property} {
+    if {[regexp "\r\n$property:(.*?)\r\n" $infostr _ value]} {
         set _ $value
     }
 }
 
+# Return value for INFO property
+proc status {r property} {
+    set _ [getInfoProperty [{*}$r info] $property]
+}
+
 proc waitForBgsave r {
     while 1 {
         if {[status r rdb_bgsave_in_progress] eq 1} {
diff --git a/tests/unit/moduleapi/defrag.tcl b/tests/unit/moduleapi/defrag.tcl
new file mode 100644
index 000000000..d4abbac1a
--- /dev/null
+++ b/tests/unit/moduleapi/defrag.tcl
@@ -0,0 +1,43 @@
+set testmodule [file normalize tests/modules/defragtest.so]
+
+start_server {tags {"modules"} overrides {{save ""}}} {
+    r module load $testmodule 10000
+    r config set hz 100
+
+    test {Module defrag: simple key defrag works} {
+        r frag.create key1 1 1000 0
+
+        r config set active-defrag-ignore-bytes 1
+        r config set active-defrag-threshold-lower 0
+        r config set active-defrag-cycle-min 99
+        r config set activedefrag yes
+
+        after 2000
+        set info [r info defragtest_stats]
+        assert {[getInfoProperty $info defragtest_datatype_attempts] > 0}
+        assert_equal 0 [getInfoProperty $info defragtest_datatype_resumes]
+    }
+
+    test {Module defrag: late defrag with cursor works} {
+        r flushdb
+        r frag.resetstats
+
+        # key can only be defragged in no less than 10 iterations
+        # due to maxstep
+        r frag.create key2 10000 100 1000
+
+        after 2000
+        set info [r info defragtest_stats]
+        assert {[getInfoProperty $info defragtest_datatype_resumes] > 10}
+        assert_equal 0 [getInfoProperty $info defragtest_datatype_wrong_cursor]
+    }
+
+    test {Module defrag: global defrag works} {
+        r flushdb
+        r frag.resetstats
+
+        after 2000
+        set info [r info defragtest_stats]
+        assert {[getInfoProperty $info defragtest_global_attempts] > 0}
+    }
+}

From e3ff414513895eff6ebf6564a9864c479ec36d25 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuancode@163.com>
Date: Sun, 13 Dec 2020 16:01:18 +0800
Subject: [PATCH 0868/1098] Add total_forks to INFO STATS (#8155)

---
 src/server.c | 4 ++++
 src/server.h | 1 +
 2 files changed, 5 insertions(+)

diff --git a/src/server.c b/src/server.c
index e4d236874..8753aa0a3 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2925,6 +2925,7 @@ void resetServerStats(void) {
     server.stat_active_defrag_scanned = 0;
     server.stat_fork_time = 0;
     server.stat_fork_rate = 0;
+    server.stat_total_forks = 0;
     server.stat_rejected_conn = 0;
     server.stat_sync_full = 0;
     server.stat_sync_partial_ok = 0;
@@ -4654,6 +4655,7 @@ sds genRedisInfoString(const char *section) {
             "pubsub_channels:%ld\r\n"
             "pubsub_patterns:%lu\r\n"
             "latest_fork_usec:%lld\r\n"
+            "total_forks:%lld\r\n"
             "migrate_cached_sockets:%ld\r\n"
             "slave_expires_tracked_keys:%zu\r\n"
             "active_defrag_hits:%lld\r\n"
@@ -4690,6 +4692,7 @@ sds genRedisInfoString(const char *section) {
             dictSize(server.pubsub_channels),
             listLength(server.pubsub_patterns),
             server.stat_fork_time,
+            server.stat_total_forks,
             dictSize(server.migrate_cached_sockets),
             getSlaveKeyWithExpireCount(),
             server.stat_active_defrag_hits,
@@ -5188,6 +5191,7 @@ int redisFork(int purpose) {
         closeClildUnusedResourceAfterFork();
     } else {
         /* Parent */
+        server.stat_total_forks++;
         server.stat_fork_time = ustime()-start;
         server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */
         latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000);
diff --git a/src/server.h b/src/server.h
index 89a25a4e0..427d7c2f2 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1215,6 +1215,7 @@ struct redisServer {
     size_t stat_peak_memory;        /* Max used memory record */
     long long stat_fork_time;       /* Time needed to perform latest fork() */
     double stat_fork_rate;          /* Fork rate in GB/sec. */
+    long long stat_total_forks;     /* Total count of fork. */
     long long stat_rejected_conn;   /* Clients rejected because of maxclients */
     long long stat_sync_full;       /* Number of full resyncs with slaves. */
     long long stat_sync_partial_ok; /* Number of accepted PSYNC requests. */

From bde33501c41e3cb49174fb282164e5ccb532a4f8 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Sun, 13 Dec 2020 03:04:45 -0500
Subject: [PATCH 0869/1098] redis-cli -e option to exit with error code
 immediately when command fails  (#8136)

without this option, redis-cli returns 0 even if command fails. kept this for backwards compatibility.
---
 src/redis-cli.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index b97eabb73..f5271b2a9 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -248,6 +248,7 @@ static struct config {
     int enable_ldb_on_eval; /* Handle manual SCRIPT DEBUG + EVAL commands. */
     int last_cmd_type;
     int verbose;
+    int set_errcode;
     clusterManagerCommand cluster_manager_command;
     int no_auth_warning;
     int resp3;
@@ -1266,6 +1267,12 @@ static int cliReadReply(int output_raw_strings) {
                 slot, config.hostip, config.hostport);
         config.cluster_reissue_command = 1;
         cliRefreshPrompt();
+    } else if (!config.interactive && config.set_errcode && 
+        reply->type == REDIS_REPLY_ERROR) 
+    {
+        fprintf(stderr,"%s\n",reply->str);
+        exit(1);
+        return REDIS_ERR; /* avoid compiler warning */
     }
 
     if (output) {
@@ -1559,6 +1566,8 @@ static int parseOptions(int argc, char **argv) {
         } else if (!strcmp(argv[i],"-D") && !lastarg) {
             sdsfree(config.cmd_delim);
             config.cmd_delim = sdsnew(argv[++i]);
+        } else if (!strcmp(argv[i],"-e")) {
+            config.set_errcode = 1;
         } else if (!strcmp(argv[i],"--verbose")) {
             config.verbose = 1;
         } else if (!strcmp(argv[i],"--cluster") && !lastarg) {
@@ -1777,6 +1786,7 @@ static void usage(void) {
 "  -d <delimiter>     Delimiter between response bulks for raw formatting (default: \\n).\n"
 "  -D <delimiter>     Delimiter between responses for raw formatting (default: \\n).\n"
 "  -c                 Enable cluster mode (follow -ASK and -MOVED redirections).\n"
+"  -e                 Return exit error code when command execution fails.\n"
 #ifdef USE_OPENSSL
 "  --tls              Establish a secure TLS connection.\n"
 "  --sni <host>       Server name indication for TLS.\n"
@@ -8127,6 +8137,7 @@ int main(int argc, char **argv) {
     config.enable_ldb_on_eval = 0;
     config.last_cmd_type = -1;
     config.verbose = 0;
+    config.set_errcode = 0;
     config.no_auth_warning = 0;
     config.cluster_manager_command.name = NULL;
     config.cluster_manager_command.argc = 0;

From 5abdf9a556e22b7528af3fcc3dcc7964d1ad6d66 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Sun, 13 Dec 2020 01:11:29 -0800
Subject: [PATCH 0870/1098] Removed usage of bool from tls.c (#8175)

---
 src/tls.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/tls.c b/src/tls.c
index caa987965..43c2f5551 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -375,7 +375,7 @@ typedef struct tls_connection {
     listNode *pending_list_node;
 } tls_connection;
 
-static connection *createTLSConnection(bool client_side) {
+static connection *createTLSConnection(int client_side) {
     SSL_CTX *ctx = redis_tls_ctx;
     if (client_side && redis_tls_client_ctx)
         ctx = redis_tls_client_ctx;
@@ -387,7 +387,7 @@ static connection *createTLSConnection(bool client_side) {
 }
 
 connection *connCreateTLS(void) {
-    return createTLSConnection(true);
+    return createTLSConnection(1);
 }
 
 /* Fetch the latest OpenSSL error and store it in the connection */
@@ -408,7 +408,7 @@ static void updateTLSError(tls_connection *conn) {
  * is not in an error state.
  */
 connection *connCreateAcceptedTLS(int fd, int require_auth) {
-    tls_connection *conn = (tls_connection *) createTLSConnection(false);
+    tls_connection *conn = (tls_connection *) createTLSConnection(0);
     conn->c.fd = fd;
     conn->c.state = CONN_STATE_ACCEPTING;
 

From 8a23ca0b28f4b6cfe7db5a77457b635cee1a855a Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Sun, 13 Dec 2020 04:47:24 -0500
Subject: [PATCH 0871/1098] add missing raxStop calls in aof stream rewrite
 (#8162)

---
 src/aof.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/aof.c b/src/aof.c
index 94dc1bdfa..d3cac4514 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1355,6 +1355,9 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) {
                     if (rioWriteStreamEmptyConsumer(r,key,(char*)ri.key,
                                                     ri.key_len,consumer) == 0)
                     {
+                        raxStop(&ri_cons);
+                        raxStop(&ri);
+                        streamIteratorStop(&si);
                         return 0;
                     }
                     continue;

From ab60dcf56488052d5760ec2ed24a29ebb0630d12 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 13 Dec 2020 14:36:06 +0200
Subject: [PATCH 0872/1098] Add module event for repl-diskless-load swapdb
 (#8153)

When a replica uses the diskless-load swapdb approach, it backs up the old database,
then attempts to load a new one, and in case of failure, it restores the backup.

this means that modules with global out of keyspace data, must have an option to
subscribe to events and backup/restore/discard their global data too.
---
 src/db.c                | 12 ++++++++++++
 src/module.c            | 16 +++++++++++++++-
 src/redismodule.h       | 13 +++++++++++--
 tests/modules/testrdb.c | 42 +++++++++++++++++++++++++++++++++++++++++
 4 files changed, 80 insertions(+), 3 deletions(-)

diff --git a/src/db.c b/src/db.c
index 41ee53601..5045935c3 100644
--- a/src/db.c
+++ b/src/db.c
@@ -477,6 +477,10 @@ dbBackup *backupDb(void) {
             sizeof(server.cluster->slots_keys_count));
     }
 
+    moduleFireServerEvent(REDISMODULE_EVENT_REPL_BACKUP,
+                          REDISMODULE_SUBEVENT_REPL_BACKUP_CREATE,
+                          NULL);
+
     return backup;
 }
 
@@ -498,6 +502,10 @@ void discardDbBackup(dbBackup *buckup, int flags, void(callback)(void*)) {
     /* Release buckup. */
     zfree(buckup->dbarray);
     zfree(buckup);
+
+    moduleFireServerEvent(REDISMODULE_EVENT_REPL_BACKUP,
+                          REDISMODULE_SUBEVENT_REPL_BACKUP_DISCARD,
+                          NULL);
 }
 
 /* Restore the previously created backup (discarding what currently resides
@@ -526,6 +534,10 @@ void restoreDbBackup(dbBackup *buckup) {
     /* Release buckup. */
     zfree(buckup->dbarray);
     zfree(buckup);
+
+    moduleFireServerEvent(REDISMODULE_EVENT_REPL_BACKUP,
+                          REDISMODULE_SUBEVENT_REPL_BACKUP_RESTORE,
+                          NULL);
 }
 
 int selectDb(client *c, int id) {
diff --git a/src/module.c b/src/module.c
index 94bdbf641..29e44aa74 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7350,7 +7350,7 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) {
  *
  *      RedisModuleEvent_SwapDB
  *
- *          This event is called when a swap db command has been successfully 
+ *          This event is called when a SWAPDB command has been successfully
  *          Executed. 
  *          For this event call currently there is no subevents available.
  *
@@ -7360,6 +7360,18 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) {
  *             int32_t dbnum_first;    // Swap Db first dbnum 
  *             int32_t dbnum_second;   // Swap Db second dbnum 
  *
+ *      RedisModuleEvent_ReplBackup
+ *
+ *          Called when diskless-repl-load config is set to swapdb,
+ *          And redis needs to backup the the current database for the
+ *          possibility to be restored later. A module with global data and
+ *          maybe with aux_load and aux_save callbacks may need to use this
+ *          notification to backup / restore / discard its globals.
+ *          The following sub events are available:
+ *
+ *              REDISMODULE_SUBEVENT_REPL_BACKUP_CREATE
+ *              REDISMODULE_SUBEVENT_REPL_BACKUP_RESTORE
+ *              REDISMODULE_SUBEVENT_REPL_BACKUP_DISCARD
  *
  *
  * The function returns REDISMODULE_OK if the module was successfully subscribed
@@ -7432,6 +7444,8 @@ int RM_IsSubEventSupported(RedisModuleEvent event, int64_t subevent) {
         return subevent < _REDISMODULE_SUBEVENT_LOADING_PROGRESS_NEXT;
     case REDISMODULE_EVENT_SWAPDB:
         return subevent < _REDISMODULE_SUBEVENT_SWAPDB_NEXT;
+    case REDISMODULE_EVENT_REPL_BACKUP:
+        return subevent < _REDISMODULE_SUBEVENT_REPL_BACKUP_NEXT;
     default:
         break;
     }
diff --git a/src/redismodule.h b/src/redismodule.h
index d7d4b7689..d2afa1f21 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -215,9 +215,10 @@ typedef uint64_t RedisModuleTimerID;
 #define REDISMODULE_EVENT_MODULE_CHANGE 9
 #define REDISMODULE_EVENT_LOADING_PROGRESS 10
 #define REDISMODULE_EVENT_SWAPDB 11
+#define REDISMODULE_EVENT_REPL_BACKUP 12
 
 /* Next event flag, should be updated if a new event added. */
-#define _REDISMODULE_EVENT_NEXT 12
+#define _REDISMODULE_EVENT_NEXT 13
 
 typedef struct RedisModuleEvent {
     uint64_t id;        /* REDISMODULE_EVENT_... defines. */
@@ -276,6 +277,10 @@ static const RedisModuleEvent
     RedisModuleEvent_SwapDB = {
         REDISMODULE_EVENT_SWAPDB,
         1
+    },
+    RedisModuleEvent_ReplBackup = {
+        REDISMODULE_EVENT_REPL_BACKUP,
+        1
     };
 
 /* Those are values that are used for the 'subevent' callback argument. */
@@ -317,11 +322,15 @@ static const RedisModuleEvent
 #define REDISMODULE_SUBEVENT_MODULE_UNLOADED 1
 #define _REDISMODULE_SUBEVENT_MODULE_NEXT 2
 
-
 #define REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB 0
 #define REDISMODULE_SUBEVENT_LOADING_PROGRESS_AOF 1
 #define _REDISMODULE_SUBEVENT_LOADING_PROGRESS_NEXT 2
 
+#define REDISMODULE_SUBEVENT_REPL_BACKUP_CREATE 0
+#define REDISMODULE_SUBEVENT_REPL_BACKUP_RESTORE 1
+#define REDISMODULE_SUBEVENT_REPL_BACKUP_DISCARD 2
+#define _REDISMODULE_SUBEVENT_REPL_BACKUP_NEXT 3
+
 #define _REDISMODULE_SUBEVENT_SHUTDOWN_NEXT 0
 #define _REDISMODULE_SUBEVENT_CRON_LOOP_NEXT 0
 #define _REDISMODULE_SUBEVENT_SWAPDB_NEXT 0
diff --git a/tests/modules/testrdb.c b/tests/modules/testrdb.c
index 3150b88e4..5d9382a3a 100644
--- a/tests/modules/testrdb.c
+++ b/tests/modules/testrdb.c
@@ -13,6 +13,45 @@ RedisModuleType *testrdb_type = NULL;
 RedisModuleString *before_str = NULL;
 RedisModuleString *after_str = NULL;
 
+void replBackupCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data)
+{
+    REDISMODULE_NOT_USED(e);
+    REDISMODULE_NOT_USED(data);
+    static RedisModuleString *before_str_backup = NULL;
+    static RedisModuleString *after_str_backup = NULL;
+
+    switch (sub) {
+    case REDISMODULE_SUBEVENT_REPL_BACKUP_CREATE:
+        assert(before_str_backup == NULL);
+        assert(after_str_backup == NULL);
+        before_str_backup = before_str;
+        after_str_backup = after_str;
+        before_str = NULL;
+        after_str = NULL;
+        break;
+    case REDISMODULE_SUBEVENT_REPL_BACKUP_RESTORE:
+        if (before_str)
+            RedisModule_FreeString(ctx, before_str);
+        if (after_str)
+            RedisModule_FreeString(ctx, after_str);
+        before_str = before_str_backup;
+        after_str = after_str_backup;
+        before_str_backup = NULL;
+        after_str_backup = NULL;
+        break;
+    case REDISMODULE_SUBEVENT_REPL_BACKUP_DISCARD:
+        if (before_str_backup)
+            RedisModule_FreeString(ctx, before_str_backup);
+        if (after_str_backup)
+            RedisModule_FreeString(ctx, after_str_backup);
+        before_str_backup = NULL;
+        after_str_backup = NULL;
+        break;
+    default:
+        assert(0);
+    }
+}
+
 void *testrdb_type_load(RedisModuleIO *rdb, int encver) {
     int count = RedisModule_LoadSigned(rdb);
     RedisModuleString *str = RedisModule_LoadString(rdb);
@@ -247,6 +286,9 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     if (RedisModule_CreateCommand(ctx,"testrdb.get.key", testrdb_get_key,"",1,1,1) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
 
+    RedisModule_SubscribeToServerEvent(ctx,
+        RedisModuleEvent_ReplBackup, replBackupCallback);
+
     return REDISMODULE_OK;
 }
 

From f74c32cad27fcd078f7bb8517c87be178e7339eb Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Sun, 13 Dec 2020 07:40:54 -0500
Subject: [PATCH 0873/1098] redis-cli prompt: show transaction state, and fix
 db number on aborted EXEC (#6728)

---
 src/redis-cli.c | 34 ++++++++++++++++++++++++++++++----
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index f5271b2a9..4f5b0fe03 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -252,6 +252,8 @@ static struct config {
     clusterManagerCommand cluster_manager_command;
     int no_auth_warning;
     int resp3;
+    int in_multi;
+    int pre_multi_dbnum;
 } config;
 
 /* User preferences. */
@@ -307,6 +309,10 @@ static void cliRefreshPrompt(void) {
     if (config.dbnum != 0)
         prompt = sdscatfmt(prompt,"[%i]",config.dbnum);
 
+    /* Add TX if in transaction state*/
+    if (config.in_multi)  
+        prompt = sdscatlen(prompt,"(TX)",4);
+
     /* Copy the prompt in the static buffer. */
     prompt = sdscatlen(prompt,"> ",2);
     snprintf(config.prompt,sizeof(config.prompt),"%s",prompt);
@@ -1396,13 +1402,32 @@ static int cliSendCommand(int argc, char **argv, long repeat) {
             return REDIS_ERR;
         } else {
             /* Store database number when SELECT was successfully executed. */
-            if (!strcasecmp(command,"select") && argc == 2 && config.last_cmd_type != REDIS_REPLY_ERROR) {
+            if (!strcasecmp(command,"select") && argc == 2 && 
+                config.last_cmd_type != REDIS_REPLY_ERROR) 
+            {
                 config.dbnum = atoi(argv[1]);
                 cliRefreshPrompt();
-            } else if (!strcasecmp(command,"auth") && (argc == 2 || argc == 3))
-            {
+            } else if (!strcasecmp(command,"auth") && (argc == 2 || argc == 3)) {
                 cliSelect();
-            }
+            } else if (!strcasecmp(command,"multi") && argc == 1 &&
+                config.last_cmd_type != REDIS_REPLY_ERROR) 
+            {
+                config.in_multi = 1;
+                config.pre_multi_dbnum = config.dbnum;
+                cliRefreshPrompt();
+            } else if (!strcasecmp(command,"exec") && argc == 1 && config.in_multi) {
+                config.in_multi = 0;
+                if (config.last_cmd_type == REDIS_REPLY_ERROR) {
+                    config.dbnum = config.pre_multi_dbnum;
+                }
+                cliRefreshPrompt();
+            } else if (!strcasecmp(command,"discard") && argc == 1 && 
+                config.last_cmd_type != REDIS_REPLY_ERROR) 
+            {
+                config.in_multi = 0;
+                config.dbnum = config.pre_multi_dbnum;
+                cliRefreshPrompt();
+            } 
         }
         if (config.cluster_reissue_command){
             /* If we need to reissue the command, break to prevent a
@@ -8139,6 +8164,7 @@ int main(int argc, char **argv) {
     config.verbose = 0;
     config.set_errcode = 0;
     config.no_auth_warning = 0;
+    config.in_multi = 0;
     config.cluster_manager_command.name = NULL;
     config.cluster_manager_command.argc = 0;
     config.cluster_manager_command.argv = NULL;

From 86e3395c117d9450d52ee3ebbc62c386d11f1b79 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 13 Dec 2020 17:09:54 +0200
Subject: [PATCH 0874/1098] Several (mostly Solaris-related) cleanups (#8171)

* Allow runtest-moduleapi use a different 'make', for systems where GNU Make is 'gmake'.
* Fix issue with builds on Solaris re-building everything from scratch due to CFLAGS/LDFLAGS not stored.
* Fix compile failure on Solaris due to atomicvar and a bunch of warnings.
* Fix garbled log timestamps on Solaris.
---
 runtest-moduleapi      |  3 ++-
 src/Makefile           |  8 +++++---
 src/ae_evport.c        |  8 +++++---
 src/aof.c              |  2 +-
 src/atomicvar.h        |  2 +-
 src/cluster.c          |  2 ++
 src/config.c           |  2 +-
 src/debug.c            |  2 +-
 src/module.c           |  6 +++---
 src/rdb.c              |  6 +++---
 src/util.c             |  3 +--
 src/zmalloc.c          | 11 +++--------
 tests/modules/Makefile |  9 ++++-----
 13 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/runtest-moduleapi b/runtest-moduleapi
index da4c815da..9a48867d2 100755
--- a/runtest-moduleapi
+++ b/runtest-moduleapi
@@ -1,6 +1,7 @@
 #!/bin/sh
 TCL_VERSIONS="8.5 8.6"
 TCLSH=""
+[ -z "$MAKE" ] && MAKE=make
 
 for VERSION in $TCL_VERSIONS; do
 	TCL=`which tclsh$VERSION 2>/dev/null` && TCLSH=$TCL
@@ -12,7 +13,7 @@ then
     exit 1
 fi
 
-make -C tests/modules && \
+$MAKE -C tests/modules && \
 $TCLSH tests/test_helper.tcl \
 --single unit/moduleapi/commandfilter \
 --single unit/moduleapi/fork \
diff --git a/src/Makefile b/src/Makefile
index 653a6e6b0..0329da8c9 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -99,9 +99,11 @@ endif
 
 ifeq ($(uname_S),SunOS)
 	# SunOS
-        ifneq ($(@@),32bit)
-		CFLAGS+= -m64
-		LDFLAGS+= -m64
+	ifeq ($(findstring -m32,$(FINAL_CFLAGS)),)
+		CFLAGS+=-m64
+	endif
+	ifeq ($(findstring -m32,$(FINAL_LDFLAGS)),)
+		LDFLAGS+=-m64
 	endif
 	DEBUG=-g
 	DEBUG_FLAGS=-g
diff --git a/src/ae_evport.c b/src/ae_evport.c
index b79ed9bc7..4e254b602 100644
--- a/src/ae_evport.c
+++ b/src/ae_evport.c
@@ -67,7 +67,7 @@ static int evport_debug = 0;
 
 typedef struct aeApiState {
     int     portfd;                             /* event port */
-    int     npending;                           /* # of pending fds */
+    uint_t  npending;                           /* # of pending fds */
     int     pending_fds[MAX_EVENT_BATCHSZ];     /* pending fds */
     int     pending_masks[MAX_EVENT_BATCHSZ];   /* pending fds' masks */
 } aeApiState;
@@ -95,6 +95,8 @@ static int aeApiCreate(aeEventLoop *eventLoop) {
 }
 
 static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
+    (void) eventLoop;
+    (void) setsize;
     /* Nothing to resize here. */
     return 0;
 }
@@ -107,7 +109,7 @@ static void aeApiFree(aeEventLoop *eventLoop) {
 }
 
 static int aeApiLookupPending(aeApiState *state, int fd) {
-    int i;
+    uint_t i;
 
     for (i = 0; i < state->npending; i++) {
         if (state->pending_fds[i] == fd)
@@ -243,7 +245,7 @@ static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) {
 static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
     aeApiState *state = eventLoop->apidata;
     struct timespec timeout, *tsp;
-    int mask, i;
+    uint_t mask, i;
     uint_t nevents;
     port_event_t event[MAX_EVENT_BATCHSZ];
 
diff --git a/src/aof.c b/src/aof.c
index d3cac4514..79b2f1284 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1728,7 +1728,7 @@ int rewriteAppendOnlyFileBackground(void) {
             return C_ERR;
         }
         serverLog(LL_NOTICE,
-            "Background append only file rewriting started by pid %d",childpid);
+            "Background append only file rewriting started by pid %ld",(long) childpid);
         server.aof_rewrite_scheduled = 0;
         server.aof_rewrite_time_start = time(NULL);
         server.aof_child_pid = childpid;
diff --git a/src/atomicvar.h b/src/atomicvar.h
index 6ac04c605..222b82699 100644
--- a/src/atomicvar.h
+++ b/src/atomicvar.h
@@ -105,7 +105,7 @@
     atomic_store_explicit(&var,value,memory_order_seq_cst)
 #define REDIS_ATOMIC_API "c11-builtin"
 
-#elif !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && !defined(__sun) && \
+#elif !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && \
     (!defined(__clang__) || !defined(__APPLE__) || __apple_build_version__ > 4210057) && \
     defined(__ATOMIC_RELAXED) && defined(__ATOMIC_SEQ_CST)
 /* Implementation using __atomic macros. */
diff --git a/src/cluster.c b/src/cluster.c
index ef73b1093..8651a81d3 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -430,6 +430,8 @@ int clusterLockConfig(char *filename) {
      * (redis-aof-rewrite) is still alive, the fd(lock) will still be held by the
      * child process, and the main process will fail to get lock, means fail to start. */
     server.cluster_config_file_lock_fd = fd;
+#else
+    UNUSED(filename);
 #endif /* __sun */
 
     return C_OK;
diff --git a/src/config.c b/src/config.c
index 28b756b5e..c858df3f3 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1002,7 +1002,7 @@ void configGetCommand(client *c) {
     }
     if (stringmatch(pattern,"unixsocketperm",1)) {
         char buf[32];
-        snprintf(buf,sizeof(buf),"%o",server.unixsocketperm);
+        snprintf(buf,sizeof(buf),"%lo",(unsigned long) server.unixsocketperm);
         addReplyBulkCString(c,"unixsocketperm");
         addReplyBulkCString(c,buf);
         matches++;
diff --git a/src/debug.c b/src/debug.c
index 7cc3c15a3..6ce8b3bdc 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1757,7 +1757,7 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
         "Accessing address: %p", (void*)info->si_addr);
     }
     if (info->si_pid != -1) {
-        serverLog(LL_WARNING, "Killed by PID: %d, UID: %d", info->si_pid, info->si_uid);
+        serverLog(LL_WARNING, "Killed by PID: %ld, UID: %d", (long) info->si_pid, info->si_uid);
     }
 
 #ifdef HAVE_BACKTRACE
diff --git a/src/module.c b/src/module.c
index 29e44aa74..da9ac29e8 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7084,7 +7084,7 @@ int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) {
         moduleForkInfo.done_handler = cb;
         moduleForkInfo.done_handler_user_data = user_data;
         updateDictResizePolicy();
-        serverLog(LL_VERBOSE, "Module fork started pid: %d ", childpid);
+        serverLog(LL_VERBOSE, "Module fork started pid: %ld ", (long) childpid);
     }
     return childpid;
 }
@@ -7134,8 +7134,8 @@ int RM_KillForkChild(int child_pid) {
 
 void ModuleForkDoneHandler(int exitcode, int bysignal) {
     serverLog(LL_NOTICE,
-        "Module fork exited pid: %d, retcode: %d, bysignal: %d",
-        server.module_child_pid, exitcode, bysignal);
+        "Module fork exited pid: %ld, retcode: %d, bysignal: %d",
+        (long) server.module_child_pid, exitcode, bysignal);
     if (moduleForkInfo.done_handler) {
         moduleForkInfo.done_handler(exitcode, bysignal,
             moduleForkInfo.done_handler_user_data);
diff --git a/src/rdb.c b/src/rdb.c
index 0878be831..46f78e75a 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1432,7 +1432,7 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
                 strerror(errno));
             return C_ERR;
         }
-        serverLog(LL_NOTICE,"Background saving started by pid %d",childpid);
+        serverLog(LL_NOTICE,"Background saving started by pid %ld",(long) childpid);
         server.rdb_save_time_start = time(NULL);
         server.rdb_child_pid = childpid;
         server.rdb_child_type = RDB_CHILD_TYPE_DISK;
@@ -2826,8 +2826,8 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
             server.rdb_pipe_numconns_writing = 0;
             closeChildInfoPipe();
         } else {
-            serverLog(LL_NOTICE,"Background RDB transfer started by pid %d",
-                childpid);
+            serverLog(LL_NOTICE,"Background RDB transfer started by pid %ld",
+                (long) childpid);
             server.rdb_save_time_start = time(NULL);
             server.rdb_child_pid = childpid;
             server.rdb_child_type = RDB_CHILD_TYPE_SOCKET;
diff --git a/src/util.c b/src/util.c
index 0d48f5701..eca212e57 100644
--- a/src/util.c
+++ b/src/util.c
@@ -749,9 +749,8 @@ sds getAbsolutePath(char *filename) {
  * Gets the proper timezone in a more portable fashion
  * i.e timezone variables are linux specific.
  */
-
 unsigned long getTimeZone(void) {
-#ifdef __linux__
+#if defined(__linux__) || defined(__sun)
     return timezone;
 #else
     struct timeval tv;
diff --git a/src/zmalloc.c b/src/zmalloc.c
index 2f9b4029d..86b15b0ee 100644
--- a/src/zmalloc.c
+++ b/src/zmalloc.c
@@ -31,6 +31,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdint.h>
+#include <unistd.h>
 
 /* This function provide us access to the original libc free(). This is useful
  * for instance to free results obtained by backtrace_symbols(). We need
@@ -333,7 +334,6 @@ void zmalloc_set_oom_handler(void (*oom_handler)(size_t)) {
  * version of the function. */
 
 #if defined(HAVE_PROC_STAT)
-#include <unistd.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
@@ -346,7 +346,7 @@ size_t zmalloc_get_rss(void) {
     int fd, count;
     char *p, *x;
 
-    snprintf(filename,256,"/proc/%d/stat",getpid());
+    snprintf(filename,256,"/proc/%ld/stat",(long) getpid());
     if ((fd = open(filename,O_RDONLY)) == -1) return 0;
     if (read(fd,buf,4096) <= 0) {
         close(fd);
@@ -370,9 +370,6 @@ size_t zmalloc_get_rss(void) {
     return rss;
 }
 #elif defined(HAVE_TASKINFO)
-#include <unistd.h>
-#include <stdio.h>
-#include <stdlib.h>
 #include <sys/types.h>
 #include <sys/sysctl.h>
 #include <mach/task.h>
@@ -393,7 +390,6 @@ size_t zmalloc_get_rss(void) {
 #include <sys/types.h>
 #include <sys/sysctl.h>
 #include <sys/user.h>
-#include <unistd.h>
 
 size_t zmalloc_get_rss(void) {
     struct kinfo_proc info;
@@ -416,7 +412,6 @@ size_t zmalloc_get_rss(void) {
 #elif defined(__NetBSD__)
 #include <sys/types.h>
 #include <sys/sysctl.h>
-#include <unistd.h>
 
 size_t zmalloc_get_rss(void) {
     struct kinfo_proc2 info;
@@ -443,7 +438,7 @@ size_t zmalloc_get_rss(void) {
     char filename[256];
     int fd;
 
-    snprintf(filename,256,"/proc/%d/psinfo",getpid());
+    snprintf(filename,256,"/proc/%ld/psinfo",(long) getpid());
 
     if ((fd = open(filename,O_RDONLY)) == -1) return 0;
     if (ioctl(fd, PIOCPSINFO, &info) == -1) {
diff --git a/tests/modules/Makefile b/tests/modules/Makefile
index 5b2c30c17..7363c98bc 100644
--- a/tests/modules/Makefile
+++ b/tests/modules/Makefile
@@ -2,13 +2,12 @@
 # find the OS
 uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
 
-# Compile flags for linux / osx
-ifeq ($(uname_S),Linux)
-	SHOBJ_CFLAGS ?= -W -Wall -fno-common -g -ggdb -std=c99 -O2
-	SHOBJ_LDFLAGS ?= -shared
-else
+ifeq ($(uname_S),Darwin)
 	SHOBJ_CFLAGS ?= -W -Wall -dynamic -fno-common -g -ggdb -std=c99 -O2
 	SHOBJ_LDFLAGS ?= -bundle -undefined dynamic_lookup
+else	# Linux, others
+	SHOBJ_CFLAGS ?= -W -Wall -fno-common -g -ggdb -std=c99 -O2
+	SHOBJ_LDFLAGS ?= -shared
 endif
 
 TEST_MODULES = \

From 19d46f8f2f4239bea738a9578e083a64c1467012 Mon Sep 17 00:00:00 2001
From: filipe oliveira <filipecosta.90@gmail.com>
Date: Sun, 13 Dec 2020 17:14:46 +0000
Subject: [PATCH 0875/1098] Expose Redis main thread cpu time, and scrape
 system time via INFO command. (#8132)

Exposes the main thread CPU info via info modules ( linux specific only )
(used_cpu_sys_main_thread and used_cpu_user_main_thread). This is important for:

- distinguish between main thread and io-threads cpu time total cpu time consumed ( check
  what is the first bottleneck on the used config )
- distinguish between main thread and modules threads total cpu time consumed

Apart from it, this commit also exposes the server_time_usec within the Server section so that we can
properly differentiate consecutive collection and calculate for example the CPU% and or / cpu time vs
wall time, etc...
---
 src/server.c | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/server.c b/src/server.c
index 8753aa0a3..ac43c32e7 100644
--- a/src/server.c
+++ b/src/server.c
@@ -56,6 +56,7 @@
 #include <sys/utsname.h>
 #include <locale.h>
 #include <sys/socket.h>
+#include <sys/resource.h>
 
 /* Our shared "common" objects */
 
@@ -4278,7 +4279,6 @@ sds genRedisInfoString(const char *section) {
     sds info = sdsempty();
     time_t uptime = server.unixtime-server.stat_starttime;
     int j;
-    struct rusage self_ru, c_ru;
     int allsections = 0, defsections = 0, everything = 0, modules = 0;
     int sections = 0;
 
@@ -4289,9 +4289,6 @@ sds genRedisInfoString(const char *section) {
     modules = strcasecmp(section,"modules") == 0;
     if (everything) allsections = 1;
 
-    getrusage(RUSAGE_SELF, &self_ru);
-    getrusage(RUSAGE_CHILDREN, &c_ru);
-
     /* Server */
     if (allsections || defsections || !strcasecmp(section,"server")) {
         static int call_uname = 1;
@@ -4337,6 +4334,7 @@ sds genRedisInfoString(const char *section) {
             "process_supervised:%s\r\n"
             "run_id:%s\r\n"
             "tcp_port:%i\r\n"
+            "server_time_usec:%I\r\n"
             "uptime_in_seconds:%I\r\n"
             "uptime_in_days:%I\r\n"
             "hz:%i\r\n"
@@ -4363,6 +4361,7 @@ sds genRedisInfoString(const char *section) {
             supervised,
             server.runid,
             server.port ? server.port : server.tls_port,
+            (int64_t)server.ustime,
             (int64_t)uptime,
             (int64_t)(uptime/(3600*24)),
             server.hz,
@@ -4850,6 +4849,10 @@ sds genRedisInfoString(const char *section) {
     /* CPU */
     if (allsections || defsections || !strcasecmp(section,"cpu")) {
         if (sections++) info = sdscat(info,"\r\n");
+
+        struct rusage self_ru, c_ru;
+        getrusage(RUSAGE_SELF, &self_ru);
+        getrusage(RUSAGE_CHILDREN, &c_ru);
         info = sdscatprintf(info,
         "# CPU\r\n"
         "used_cpu_sys:%ld.%06ld\r\n"
@@ -4860,6 +4863,15 @@ sds genRedisInfoString(const char *section) {
         (long)self_ru.ru_utime.tv_sec, (long)self_ru.ru_utime.tv_usec,
         (long)c_ru.ru_stime.tv_sec, (long)c_ru.ru_stime.tv_usec,
         (long)c_ru.ru_utime.tv_sec, (long)c_ru.ru_utime.tv_usec);
+#ifdef RUSAGE_THREAD
+        struct rusage m_ru;
+        getrusage(RUSAGE_THREAD, &m_ru);
+        info = sdscatprintf(info,
+            "used_cpu_sys_main_thread:%ld.%06ld\r\n"
+            "used_cpu_user_main_thread:%ld.%06ld\r\n",
+            (long)m_ru.ru_stime.tv_sec, (long)m_ru.ru_stime.tv_usec,
+            (long)m_ru.ru_utime.tv_sec, (long)m_ru.ru_utime.tv_usec);
+#endif  /* RUSAGE_THREAD */
     }
 
     /* Modules */

From 1e301ff83e2f0d8e63646a6689aacc021729e181 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 13 Dec 2020 22:11:06 +0200
Subject: [PATCH 0876/1098] Fix redis-cli crash on nil invalidate messages.
 (#8183)

This is a backport of redis/hiredis@b9b9f44.

Co-authored-by: michael-grunder <michael.grunder@gmail.com>
---
 deps/hiredis/hiredis.c |  3 ++-
 deps/hiredis/test.c    | 38 +++++++++++++++++++++++++++++++-------
 2 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c
index 38e7e950a..51f22a665 100644
--- a/deps/hiredis/hiredis.c
+++ b/deps/hiredis/hiredis.c
@@ -257,7 +257,8 @@ static void *createNilObject(const redisReadTask *task) {
         parent = task->parent->obj;
         assert(parent->type == REDIS_REPLY_ARRAY ||
                parent->type == REDIS_REPLY_MAP ||
-               parent->type == REDIS_REPLY_SET);
+               parent->type == REDIS_REPLY_SET ||
+               parent->type == REDIS_REPLY_PUSH);
         parent->element[task->idx] = r;
     }
     return r;
diff --git a/deps/hiredis/test.c b/deps/hiredis/test.c
index 012b6ad61..829536739 100644
--- a/deps/hiredis/test.c
+++ b/deps/hiredis/test.c
@@ -53,6 +53,11 @@ struct privdata {
     int dtor_counter;
 };
 
+struct pushCounters {
+    int nil;
+    int str;
+};
+
 #ifdef HIREDIS_TEST_SSL
 redisSSLContext *_ssl_ctx = NULL;
 #endif
@@ -677,11 +682,25 @@ static void test_blocking_connection_errors(void) {
 #endif
 }
 
-/* Dummy push handler */
-void push_handler(void *privdata, void *reply) {
-    int *counter = privdata;
+/* Test push handler */
+void push_handler(void *privdata, void *r) {
+    struct pushCounters *pcounts = privdata;
+    redisReply *reply = r, *payload;
+
+    assert(reply && reply->type == REDIS_REPLY_PUSH && reply->elements == 2);
+
+    payload = reply->element[1];
+    if (payload->type == REDIS_REPLY_ARRAY) {
+        payload = payload->element[0];
+    }
+
+    if (payload->type == REDIS_REPLY_STRING) {
+        pcounts->str++;
+    } else if (payload->type == REDIS_REPLY_NIL) {
+        pcounts->nil++;
+    }
+
     freeReplyObject(reply);
-    *counter += 1;
 }
 
 /* Dummy function just to test setting a callback with redisOptions */
@@ -691,16 +710,16 @@ void push_handler_async(redisAsyncContext *ac, void *reply) {
 }
 
 static void test_resp3_push_handler(redisContext *c) {
+    struct pushCounters pc = {0};
     redisPushFn *old = NULL;
     redisReply *reply;
     void *privdata;
-    int n = 0;
 
     /* Switch to RESP3 and turn on client tracking */
     send_hello(c, 3);
     send_client_tracking(c, "ON");
     privdata = c->privdata;
-    c->privdata = &n;
+    c->privdata = &pc;
 
     reply = redisCommand(c, "GET key:0");
     assert(reply != NULL);
@@ -717,7 +736,12 @@ static void test_resp3_push_handler(redisContext *c) {
     old = redisSetPushCallback(c, push_handler);
     test("We can set a custom RESP3 PUSH handler: ");
     reply = redisCommand(c, "SET key:0 val:0");
-    test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && n == 1);
+    test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && pc.str == 1);
+    freeReplyObject(reply);
+
+    test("We properly handle a NIL invalidation payload: ");
+    reply = redisCommand(c, "FLUSHDB");
+    test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && pc.nil == 1);
     freeReplyObject(reply);
 
     /* Unset the push callback and generate an invalidate message making

From 7d9b09adaaa92b07a11cb1b40aea6958fab049a6 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 14 Dec 2020 11:13:46 +0200
Subject: [PATCH 0877/1098] Tests: fix new defrag test to be skipped when not
 supported (#8185)

Additionally the older defrag tests are using an obsolete way to check
if the defragger is suuported (the error no longer contains "DISABLED").
this doesn't usually makes a difference since these tests are completely
skipped if the allocator is not jemalloc, but that would fail if the
allocator is a jemalloc that doesn't support defrag.
---
 tests/unit/memefficiency.tcl    |  8 ++---
 tests/unit/moduleapi/defrag.tcl | 59 +++++++++++++++++----------------
 2 files changed, 35 insertions(+), 32 deletions(-)

diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl
index 6dee77bed..357089c8f 100644
--- a/tests/unit/memefficiency.tcl
+++ b/tests/unit/memefficiency.tcl
@@ -63,7 +63,7 @@ start_server {tags {"defrag"} overrides {appendonly yes auto-aof-rewrite-percent
             r config set maxmemory 110mb ;# prevent further eviction (not to fail the digest test)
             set digest [r debug digest]
             catch {r config set activedefrag yes} e
-            if {![string match {DISABLED*} $e]} {
+            if {[r config get activedefrag] eq "activedefrag yes"} {
                 # Wait for the active defrag to start working (decision once a
                 # second).
                 wait_for_condition 50 100 {
@@ -236,7 +236,7 @@ start_server {tags {"defrag"} overrides {appendonly yes auto-aof-rewrite-percent
 
             set digest [r debug digest]
             catch {r config set activedefrag yes} e
-            if {![string match {DISABLED*} $e]} {
+            if {[r config get activedefrag] eq "activedefrag yes"} {
                 # wait for the active defrag to start working (decision once a second)
                 wait_for_condition 50 100 {
                     [s active_defrag_running] ne 0
@@ -332,7 +332,7 @@ start_server {tags {"defrag"} overrides {appendonly yes auto-aof-rewrite-percent
 
             set digest [r debug digest]
             catch {r config set activedefrag yes} e
-            if {![string match {DISABLED*} $e]} {
+            if {[r config get activedefrag] eq "activedefrag yes"} {
                 # wait for the active defrag to start working (decision once a second)
                 wait_for_condition 50 100 {
                     [s active_defrag_running] ne 0
@@ -452,7 +452,7 @@ start_server {tags {"defrag"} overrides {appendonly yes auto-aof-rewrite-percent
 
                 set digest [r debug digest]
                 catch {r config set activedefrag yes} e
-                if {![string match {DISABLED*} $e]} {
+                if {[r config get activedefrag] eq "activedefrag yes"} {
                     # wait for the active defrag to start working (decision once a second)
                     wait_for_condition 50 100 {
                         [s active_defrag_running] ne 0
diff --git a/tests/unit/moduleapi/defrag.tcl b/tests/unit/moduleapi/defrag.tcl
index d4abbac1a..b2e23967e 100644
--- a/tests/unit/moduleapi/defrag.tcl
+++ b/tests/unit/moduleapi/defrag.tcl
@@ -3,41 +3,44 @@ set testmodule [file normalize tests/modules/defragtest.so]
 start_server {tags {"modules"} overrides {{save ""}}} {
     r module load $testmodule 10000
     r config set hz 100
+    r config set active-defrag-ignore-bytes 1
+    r config set active-defrag-threshold-lower 0
+    r config set active-defrag-cycle-min 99
 
-    test {Module defrag: simple key defrag works} {
-        r frag.create key1 1 1000 0
+    # try to enable active defrag, it will fail if redis was compiled without it
+    catch {r config set activedefrag yes} e
+    if {[r config get activedefrag] eq "activedefrag yes"} {
 
-        r config set active-defrag-ignore-bytes 1
-        r config set active-defrag-threshold-lower 0
-        r config set active-defrag-cycle-min 99
-        r config set activedefrag yes
+        test {Module defrag: simple key defrag works} {
+            r frag.create key1 1 1000 0
 
-        after 2000
-        set info [r info defragtest_stats]
-        assert {[getInfoProperty $info defragtest_datatype_attempts] > 0}
-        assert_equal 0 [getInfoProperty $info defragtest_datatype_resumes]
-    }
+            after 2000
+            set info [r info defragtest_stats]
+            assert {[getInfoProperty $info defragtest_datatype_attempts] > 0}
+            assert_equal 0 [getInfoProperty $info defragtest_datatype_resumes]
+        }
 
-    test {Module defrag: late defrag with cursor works} {
-        r flushdb
-        r frag.resetstats
+        test {Module defrag: late defrag with cursor works} {
+            r flushdb
+            r frag.resetstats
 
-        # key can only be defragged in no less than 10 iterations
-        # due to maxstep
-        r frag.create key2 10000 100 1000
+            # key can only be defragged in no less than 10 iterations
+            # due to maxstep
+            r frag.create key2 10000 100 1000
 
-        after 2000
-        set info [r info defragtest_stats]
-        assert {[getInfoProperty $info defragtest_datatype_resumes] > 10}
-        assert_equal 0 [getInfoProperty $info defragtest_datatype_wrong_cursor]
-    }
+            after 2000
+            set info [r info defragtest_stats]
+            assert {[getInfoProperty $info defragtest_datatype_resumes] > 10}
+            assert_equal 0 [getInfoProperty $info defragtest_datatype_wrong_cursor]
+        }
 
-    test {Module defrag: global defrag works} {
-        r flushdb
-        r frag.resetstats
+        test {Module defrag: global defrag works} {
+            r flushdb
+            r frag.resetstats
 
-        after 2000
-        set info [r info defragtest_stats]
-        assert {[getInfoProperty $info defragtest_global_attempts] > 0}
+            after 2000
+            set info [r info defragtest_stats]
+            assert {[getInfoProperty $info defragtest_global_attempts] > 0}
+        }
     }
 }

From cfb449cc8066a0b1f0225b6278241d0d9d20a779 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 14 Dec 2020 17:10:31 +0200
Subject: [PATCH 0878/1098] Sanitize dump payload: excessive free on dup zset
 fields (#8189)

---
 src/rdb.c                          |  2 +-
 tests/integration/corrupt-dump.tcl | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/rdb.c b/src/rdb.c
index 46f78e75a..58e7d2cff 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1659,7 +1659,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
             if (dictAdd(zs->dict,sdsele,&znode->score) != DICT_OK) {
                 rdbReportCorruptRDB("Duplicate zset fields detected");
                 decrRefCount(o);
-                sdsfree(sdsele);
+                /* no need to free 'sdsele', will be released by zslFree together with 'o' */
                 return NULL;
             }
         }
diff --git a/tests/integration/corrupt-dump.tcl b/tests/integration/corrupt-dump.tcl
index 4b09d45b9..cc597bb4d 100644
--- a/tests/integration/corrupt-dump.tcl
+++ b/tests/integration/corrupt-dump.tcl
@@ -497,5 +497,15 @@ test {corrupt payload: fuzzer findings - valgrind negative malloc} {
     }
 }
 
+test {corrupt payload: fuzzer findings - valgrind invalid read} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload yes
+        r debug set-skip-checksum-validation 1
+        catch {r RESTORE _key 0 "\x05\x0A\x02\x5F\x39\x00\x00\x00\x00\x00\x00\x22\x40\xC0\x08\x00\x00\x00\x00\x00\x00\x20\x40\x02\x5F\x37\x00\x00\x00\x00\x00\x00\x1C\x40\xC0\x06\x00\x00\x00\x00\x00\x00\x18\x40\x02\x5F\x33\x00\x00\x00\x00\x00\x00\x14\x40\xC0\x04\x00\x00\x00\x00\x00\x00\x10\x40\x02\x5F\x33\x00\x00\x00\x00\x00\x00\x08\x40\xC0\x02\x00\x00\x00\x00\x00\x00\x00\x40\x02\x5F\x31\x00\x00\x00\x00\x00\x00\xF0\x3F\xC0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x09\x00\x3C\x66\xD7\x14\xA9\xDA\x3C\x69"} err
+        assert_match "*Bad data format*" $err
+        r ping
+    }
+}
+
 } ;# tags
 

From 9acd40d97b00aefeae1ae8f523cd9b0adde80d81 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Mon, 14 Dec 2020 17:15:12 +0200
Subject: [PATCH 0879/1098] GEOSEARCH: change 'FROMLOC' to 'FROMLONLAT' (#8190)

And formats style a tiniee-winiee bit
---
 src/geo.c          | 10 +++++++---
 tests/unit/geo.tcl | 30 +++++++++++++++---------------
 2 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/src/geo.c b/src/geo.c
index 13cbb1c50..2fd1cb21b 100644
--- a/src/geo.c
+++ b/src/geo.c
@@ -591,7 +591,7 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
                 }
                 frommember = 1;
                 i++;
-            } else if (!strcasecmp(arg, "fromloc") &&
+            } else if (!strcasecmp(arg, "fromlonlat") &&
                        (i+2) < remaining &&
                        flags & GEOSEARCH &&
                        !frommember)
@@ -635,12 +635,16 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
     }
 
     if ((flags & GEOSEARCH) && !(frommember || fromloc)) {
-        addReplyErrorFormat(c, "exactly one of FROMMEMBER or FROMLOC can be specified for %s", (char *)c->argv[0]->ptr);
+        addReplyErrorFormat(c,
+            "exactly one of FROMMEMBER or FROMLONLAT can be specified for %s",
+            (char *)c->argv[0]->ptr);
         return;
     }
 
     if ((flags & GEOSEARCH) && !(byradius || bybox)) {
-        addReplyErrorFormat(c, "exactly one of BYRADIUS and BYBOX can be specified for %s", (char *)c->argv[0]->ptr);
+        addReplyErrorFormat(c,
+            "exactly one of BYRADIUS and BYBOX can be specified for %s",
+            (char *)c->argv[0]->ptr);
         return;
     }
 
diff --git a/tests/unit/geo.tcl b/tests/unit/geo.tcl
index b3e59d9b2..119d2bd64 100644
--- a/tests/unit/geo.tcl
+++ b/tests/unit/geo.tcl
@@ -95,31 +95,31 @@ start_server {tags {"geo"}} {
     } {{central park n/q/r} 4545 {union square}}
 
     test {GEOSEARCH simple (sorted)} {
-        r geosearch nyc fromloc -73.9798091 40.7598464 bybox 6 6 km asc
+        r geosearch nyc fromlonlat -73.9798091 40.7598464 bybox 6 6 km asc
     } {{central park n/q/r} 4545 {union square} {lic market}}
 
-    test {GEOSEARCH FROMLOC and FROMMEMBER cannot exist at the same time} {
-        catch {r geosearch nyc fromloc -73.9798091 40.7598464 frommember xxx bybox 6 6 km asc} e
+    test {GEOSEARCH FROMLONLAT and FROMMEMBER cannot exist at the same time} {
+        catch {r geosearch nyc fromlonlat -73.9798091 40.7598464 frommember xxx bybox 6 6 km asc} e
         set e
     } {ERR*syntax*}
 
-    test {GEOSEARCH FROMLOC and FROMMEMBER one must exist} {
+    test {GEOSEARCH FROMLONLAT and FROMMEMBER one must exist} {
         catch {r geosearch nyc bybox 3 3 km asc desc withhash withdist withcoord} e
         set e
-    } {ERR*exactly one of FROMMEMBER or FROMLOC*}
+    } {ERR*exactly one of FROMMEMBER or FROMLONLAT*}
 
     test {GEOSEARCH BYRADIUS and BYBOX cannot exist at the same time} {
-        catch {r geosearch nyc fromloc -73.9798091 40.7598464 byradius 3 km bybox 3 3 km asc} e
+        catch {r geosearch nyc fromlonlat -73.9798091 40.7598464 byradius 3 km bybox 3 3 km asc} e
         set e
     } {ERR*syntax*}
 
     test {GEOSEARCH BYRADIUS and BYBOX one must exist} {
-        catch {r geosearch nyc fromloc -73.9798091 40.7598464 asc desc withhash withdist withcoord} e
+        catch {r geosearch nyc fromlonlat -73.9798091 40.7598464 asc desc withhash withdist withcoord} e
         set e
     } {ERR*exactly one of BYRADIUS and BYBOX*}
 
     test {GEOSEARCH with STOREDIST option} {
-        catch {r geosearch nyc fromloc -73.9798091 40.7598464 bybox 6 6 km asc storedist} e
+        catch {r geosearch nyc fromlonlat -73.9798091 40.7598464 bybox 6 6 km asc storedist} e
         set e
     } {ERR*syntax*}
 
@@ -128,7 +128,7 @@ start_server {tags {"geo"}} {
     } {{{central park n/q/r} 0.7750} {4545 2.3651} {{union square} 2.7697}}
 
     test {GEOSEARCH withdist (sorted)} {
-        r geosearch nyc fromloc -73.9798091 40.7598464 bybox 6 6 km withdist asc
+        r geosearch nyc fromlonlat -73.9798091 40.7598464 bybox 6 6 km withdist asc
     } {{{central park n/q/r} 0.7750} {4545 2.3651} {{union square} 2.7697} {{lic market} 3.1991}}
 
     test {GEORADIUS with COUNT} {
@@ -163,7 +163,7 @@ start_server {tags {"geo"}} {
         r geoadd Sicily 12.758489 38.788135 "edge1"   17.241510 38.788135 "eage2"
         set ret1 [r georadius Sicily 15 37 200 km asc]
         assert_equal $ret1 {Catania Palermo}
-        set ret2 [r geosearch Sicily fromloc 15 37 bybox 400 400 km asc]
+        set ret2 [r geosearch Sicily fromlonlat 15 37 bybox 400 400 km asc]
         assert_equal $ret2 {Catania Palermo eage2 edge1}
     }
 
@@ -173,12 +173,12 @@ start_server {tags {"geo"}} {
         r geoadd Sicily 12.75 36.50 "test2"
         r geoadd Sicily 13.00 36.50 "test3"
         # box height=2km width=400km
-        set ret1 [r geosearch Sicily fromloc 15 37 bybox 2 400 km]
+        set ret1 [r geosearch Sicily fromlonlat 15 37 bybox 2 400 km]
         assert_equal $ret1 {test1}
 
         # Add a western Hemisphere point
         r geoadd Sicily -1 37.00 "test3"
-        set ret2 [r geosearch Sicily fromloc 15 37 bybox 2 3000 km asc]
+        set ret2 [r geosearch Sicily fromlonlat 15 37 bybox 2 3000 km asc]
         assert_equal $ret2 {test1 test3}
     }
 
@@ -241,7 +241,7 @@ start_server {tags {"geo"}} {
     } {*ERR*syntax*}
 
     test {GEOSEARCHSTORE STORE option: syntax error} {
-        catch {r geosearchstore abc points fromloc 13.361389 38.115556 byradius 50 km store abc} e
+        catch {r geosearchstore abc points fromlonlat 13.361389 38.115556 byradius 50 km store abc} e
         set e
     } {*ERR*syntax*}
 
@@ -266,7 +266,7 @@ start_server {tags {"geo"}} {
     }
 
     test {GEOSEARCHSTORE STORE option: plain usage} {
-        r geosearchstore points2 points fromloc 13.361389 38.115556 byradius 500 km
+        r geosearchstore points2 points fromlonlat 13.361389 38.115556 byradius 500 km
         assert_equal [r zrange points 0 -1] [r zrange points2 0 -1]
     }
 
@@ -282,7 +282,7 @@ start_server {tags {"geo"}} {
     }
 
     test {GEOSEARCHSTORE STOREDIST option: plain usage} {
-        r geosearchstore points2 points fromloc 13.361389 38.115556 byradius 500 km storedist
+        r geosearchstore points2 points fromlonlat 13.361389 38.115556 byradius 500 km storedist
         set res [r zrange points2 0 -1 withscores]
         assert {[lindex $res 1] < 1}
         assert {[lindex $res 3] > 166}

From b8c67ce41b51247c02a639929e4fab20189afa1e Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 14 Dec 2020 20:54:10 +0200
Subject: [PATCH 0880/1098] Redis 6.2 RC1.

---
 00-RELEASENOTES | 226 +++++++++++++++++++++++++++++++++++++++++++++---
 src/help.h      |  89 ++++++++++++++++---
 src/version.h   |   4 +-
 3 files changed, 294 insertions(+), 25 deletions(-)

diff --git a/00-RELEASENOTES b/00-RELEASENOTES
index ce472159e..8fdfcf760 100644
--- a/00-RELEASENOTES
+++ b/00-RELEASENOTES
@@ -1,16 +1,220 @@
-Hello! This file is just a placeholder, since this is the "unstable" branch
-of Redis, the place where all the development happens.
+================================================================================
+Redis 6.2 RC1   Released Mon Dec 14 11:50:00 IST 2020
+================================================================================
 
-There is no release notes for this branch, it gets forked into another branch
-every time there is a partial feature freeze in order to eventually create
-a new stable release.
+Upgrade urgency LOW: This is the first RC of Redis 6.2.
 
-Usually "unstable" is stable enough for you to use it in development environments
-however you should never use it in production environments. It is possible
-to download the latest stable release here:
+Introduction to the Redis 6.2 release
+=====================================
 
-    http://download.redis.io/releases/redis-stable.tar.gz
+This release is the first significant Redis release managed by the core team
+under the new project governance model.
 
-More information is available at http://redis.io
+Redis 6.2 includes many new commands and improvements, but no big features. It
+mainly makes Redis more complete and addresses issues that have been requested
+by many users frequently or for a long time.
 
-Happy hacking!
+Many of these changes were not eligible for 6.0.x for several reasons:
+
+1. They are not backward compatible, which is always the case with new or
+   extended commands (that cannot be replicated to an older replica).
+2. They require a longer release-candidate test cycle.
+
+
+Here is a comprehensive list of changes in this release compared to 6.0.9,
+each one includes the PR number that added it, so you can get more details
+at https://github.com/redis/redis/pull/<number>
+
+New commands / args:
+* Add SMISMEMBER command that checks multiple members (#7615)
+* Add ZMSCORE command that returns an array of scores (#7593)
+* Add LMOVE and BLMOVE commands that pop and push arbitrarily (#6929)
+* Add RESET command that resets client connection state(#7982)
+* Add COPY command that copies keys (#7953)
+* Add ZDIFF and ZDIFFSTORE commands (#7961)
+* Add ZINTER and ZUNION commands (#7794)
+* Add GEOSEARCH/GEOSEARCHSTORE commands for bounding box spatial queries (#8094)
+* Add GET parameter to SET command, for more powerful GETSET (#7852)
+* Add exclusive range query to XPENDING (#8130)
+* Add exclusive range query to X[REV]RANGE (#8072)
+* Add GT and LT options to ZADD for conditional score updates (#7818)
+* Add CLIENT INFO and CLIENT LIST for specific ids (#8113)
+* Add IDLE argument to XPENDING command (#7972)
+* Add local address to CLIENT LIST, and a CLIENT KILL filter. (#7913)
+* Add NOMKSTREAM option to XADD command (#7910)
+* Add command introspection to Sentinel (#7940)
+* Add SENTINEL MYID subcommand (#7858)
+
+New features:
+* Dump payload sanitization: prevent corrupt payload causing crashes (#7807)
+  Has flags to enable full O(N) validation (disabled by default).
+* ACL patterns for Pub/Sub channels (#7993)
+* Support ACL for Sentinel mode (#7888)
+* Support getting configuration from both stdin and file at the same time (#7893)
+  Lets you avoid storing secrets on the disk.
+
+New features in CLI tools:
+* redis-cli RESP3 push support (#7609)
+* redis-cli cluster import support source and target that require auth (#7994)
+* redis-cli URIs able to provide user name in addition to password (#8048)
+* redis-cli/redis-benchmark allow specifying the prefered ciphers/ciphersuites (#8005)
+* redis-cli add -e option to exit with code when command execution fails (#8136)
+
+Command behavior changes:
+* EXISTS should not alter LRU (#8016)
+  In Redis 5.0 and 6.0 it would have touched the LRU/LFU of the key.
+* OBJECT should not reveal logically expired keys (#8016)
+  Will now behave the same TYPE or any other non-DEBUG command.
+* Improve db id range check for SELECT and MOVE (#8085)
+  Changes the error message text on a wrong db index.
+* Modify AUTH / HELLO error message (#7648)
+  Changes the error message text when the user isn't found or is disabled.
+* BITOPS length limited to proto_max_bulk_len rather than 512MB (#8096)
+  The limit is now configurable like in SETRANGE, and APPEND.
+* GEORADIUS[BYMEMBER] can fail with -OOM if Redis is over the memory limit (#8107)
+
+Other behavior changes:
+* Optionally (default) fail to start if requested bind address is not available (#7936)
+  If you rely on Redis starting successfully even if one of the bind addresses
+  is not available, you'll need to tune the new config.
+* Limit the main db dictionaries expansion to prevent key eviction (#7954)
+  In the past big dictionary rehashing could result in massive data eviction.
+  Now this rehashing is delayed (up to a limit), which can result in performance
+  loss due to hash collisions.
+* CONFIG REWRITE is atomic and safer, but requires write access to the config file's folder (#7824, #8051)
+  This change was already present in 6.0.9, but was missing from the release
+  notes.
+* A new incremental eviction mechanism that reduces latency on eviction spikes (#7653)
+  In pathological cases this can cause memory to grow uncontrolled and may require
+  specific tuning.
+* Not resetting "save" config when Redis is started with command line arguments. (#7092)
+  In case you provide command line arguments without "save" and count on it
+  being disabled, Now the defaults "save" config will kick in.
+* Update memory metrics for INFO during loading (#7690)
+* When "supervised" config is enabled, it takes precedence over "daemonize". (#8036)
+* Assertion and panic, print crash log without generating SIGSEGV (#7585)
+* Added crash log report on SIGABRT, instead of silently exiting (#8004)
+* Disable THP (Transparent Huge Pages) if enabled (#7381)
+  If you deliberately enabled it, you'll need to config Redis to keep it.
+
+Bug fixes:
+* Handle output buffer limits for module blocked clients (#8141)
+  Could result in a module sending reply to a blocked client to go beyond the
+  limit.
+* Fix setproctitle related crashes. (#8150, #8088)
+  Caused various crashes on startup, mainly on Apple M1 chips or under
+  instrumentation.
+* A module doing RM_Call could cause replicas to get nested MULTI (#8097).
+* Backup/restore cluster mode keys to slots map for repl-diskless-load=swapdb (#8108)
+  In cluster mode with repl-diskless-load, when loading failed, slot map
+  wouldn't have been restored.
+* Fix oom-score-adj-values range, and bug when used in config file (#8046)
+  Enabling setting this in the config file in a line after enabling it, would
+  have been buggy.
+* Reset average ttl when empty databases (#8106)
+  Just causing misleading metric in INFO
+* Disable rehash when Redis has child process (#8007)
+  This could have caused excessive CoW during BGSAVE, replication or AOFRW.
+* Further improved ACL algorithm for picking categories (#7966)
+  Output of ACL GETUSER is now more similar to the one provided by ACL SETUSER.
+* Fix bug with module GIL being released prematurely (#8061)
+  Could in theory (and rarely) cause multi-threaded modules to corrupt memory.
+* Fix cluster redirect for module command with no firstkey. (#7539)
+* Reduce effect of client tracking causing feedback loop in key eviction (#8100)
+* Kill disk-based fork child when all replicas drop and 'save' is not enabled (#7819)
+* Rewritten commands (modified for propagation) are logged as their original command (#8006)
+* Fix cluster access to unaligned memory (SIGBUS on old ARM) #7958
+* If diskless repl child is killed, make sure to reap the child pid (#7742)
+* Broadcast a PONG message when slot's migration is over, may reduce MOVED responses (#7571)
+
+Other improvements:
+* TLS Support in redis-benchmark (#7959)
+* Accelerate diskless master connections, and general re-connections (#6271)
+* Run active defrag while blocked / loading (#7726)
+* Performance and memory reporting improvement - sds take control of its internal fragmentation (#7875)
+* Speedup cluster failover. (#7948)
+
+Platform / toolchain support related improvements:
+* Optionally (not by default) use H/W Monotonic clock for faster time sampling (#7644)
+* Remove the requirements for C11 and _Atomic supporting compiler (#7707)
+  This would allow to more easily build and use Redis on older systems and
+  compilers again.
+* Fix crash log registers output on ARM. (#8020)
+* Raspberry build fix. (#8095)
+* Setting process title support for Haiku. (#8060)
+* DragonFlyBSD RSS memory sampling support. (#8023)
+
+New configuration options:
+* Enable configuring OpenSSL using the standard openssl.cnf (#8143)
+* oom-score-adj-values can config can now take absolute values (besides relative ones) (#8046)
+* TLS: Add different client cert support. (#8076)
+* Note that a few other changes listed above added their config options.
+
+Info fields and introspection changes:
+* Add INFO fields to track diskless and disk-based replication progress (#7981)
+* Add INFO field for main thread cpu time, and scrape system time. (#8132)
+* Add total_forks to INFO STATS (#8155)
+* Add maxclients and cluster_connections to INFO CLIENTS (#7979)
+* Add tracking bcast flag and client redirection in client list (#7995)
+* Fixed INFO client_recent_max_input_buffer includes argv array (#8065, see #7874)
+* Note that a few other changes listed above added their info fields.
+
+Module API changes:
+* Add CTX_FLAGS_DENY_BLOCKING as a unified the way to know if blocking is allowed (#8025)
+* Add data type callbacks for lazy free effort, and unlink (#7912)
+* Add data type callback for COPY command (#8112)
+* Add callbacks for defrag support. (#8149)
+* Add module event for repl-diskless-load swapdb (#8153)
+
+Module related fixes:
+* Moved RMAPI_FUNC_SUPPORTED so that it's usable (#8037)
+* Improve timer accuracy (#7987)
+* Allow '\0' inside of result of RM_CreateStringPrintf (#6260)
+
+
+Thanks to all the users and developers who made this release possible.
+We'll follow up with more RC releases, until the code looks production ready
+and we don't get reports of serious issues for a while.
+
+A special thank you for the amount of work put into this release by:
+- Wang Yuan
+- Felipe Machado
+- Filipe Oliveira
+- Tatsuya Arisawa
+- Yang Bodong
+- Jim Brunner
+- Wen Hui
+- Guy Benoish
+- Chen Yang
+- Nitai Caro
+- Meir Shpilraien
+- Madelyn Olson
+- maohuazhu
+- Valentino Geron
+- Tyson Andre
+- Michael Grunder
+- alexronke-channeladvisor
+- sundb
+- WuYunlong
+- Yoav Steinberg
+- Uri Shachar
+- Nykolas Laurentino de Lima
+- David CARLIER
+
+
+Migrating from 6.0 to 6.2
+=========================
+
+Redis 6.2 is mostly a strict superset of 6.0, you should not have any problem
+upgrading your application from 6.0 to 6.2. However there are some small changes
+of behavior listed above, please make sure you are not badly affected by any of
+them.
+
+Specifically these sections:
+* Command behavior changes
+* Other behavior changes
+
+--------------------------------------------------------------------------------
+
+Cheers,
+The Redis team
diff --git a/src/help.h b/src/help.h
index 8a12458e1..d6754f570 100644
--- a/src/help.h
+++ b/src/help.h
@@ -1,4 +1,4 @@
-/* Automatically generated by generate-command-help.rb, do not edit. */
+/* Automatically generated by ./utils/generate-command-help.rb, do not edit. */
 
 #ifndef __REDIS_HELP_H
 #define __REDIS_HELP_H
@@ -94,7 +94,7 @@ struct commandHelp {
     1,
     "2.0.0" },
     { "AUTH",
-    "password",
+    "[username] password",
     "Authenticate to the server",
     8,
     "1.0.0" },
@@ -128,6 +128,11 @@ struct commandHelp {
     "Find first bit set or clear in a string",
     1,
     "2.8.7" },
+    { "BLMOVE",
+    "source destination LEFT|RIGHT LEFT|RIGHT timeout",
+    "Pop an element from a list, push it to another list and return it; or block until one is available",
+    2,
+    "6.2.0" },
     { "BLPOP",
     "key [key ...] timeout",
     "Remove and get the first element in a list, or block until one is available",
@@ -173,13 +178,18 @@ struct commandHelp {
     "Returns the client ID for the current connection",
     8,
     "5.0.0" },
+    { "CLIENT INFO",
+    "-",
+    "Returns information about the current client connection.",
+    8,
+    "6.2.0" },
     { "CLIENT KILL",
     "[ip:port] [ID client-id] [TYPE normal|master|slave|pubsub] [USER username] [ADDR ip:port] [SKIPME yes/no]",
     "Kill the connection of a client",
     8,
     "2.4.0" },
     { "CLIENT LIST",
-    "[TYPE normal|master|replica|pubsub]",
+    "[TYPE normal|master|replica|pubsub] [ID client-id [client-id ...]]",
     "Get the list of client connections",
     8,
     "2.4.0" },
@@ -358,6 +368,11 @@ struct commandHelp {
     "Set a configuration parameter to the given value",
     9,
     "2.0.0" },
+    { "COPY",
+    "source destination [DB destination-db] [REPLACE]",
+    "Copy a key",
+    0,
+    "6.2.0" },
     { "DBSIZE",
     "-",
     "Return the number of keys in the selected database",
@@ -473,6 +488,16 @@ struct commandHelp {
     "Query a sorted set representing a geospatial index to fetch members matching a given maximum distance from a member",
     13,
     "3.2.0" },
+    { "GEOSEARCH",
+    "key [FROMMEMBER member] [FROMLONLAT longitude latitude] [BYRADIUS radius m|km|ft|mi] [BYBOX width height m|km|ft|mi] [ASC|DESC] [COUNT count] [WITHCOORD] [WITHDIST] [WITHHASH]",
+    "Query a sorted set representing a geospatial index to fetch members inside an area of a box or a circle.",
+    13,
+    "6.2" },
+    { "GEOSEARCHSTORE",
+    "destination source [FROMMEMBER member] [FROMLONLAT longitude latitude] [BYRADIUS radius m|km|ft|mi] [BYBOX width height m|km|ft|mi] [ASC|DESC] [COUNT count] [WITHCOORD] [WITHDIST] [WITHHASH] [STOREDIST]",
+    "Query a sorted set representing a geospatial index to fetch members inside an area of a box or a circle, and store the result in another key.",
+    13,
+    "6.2" },
     { "GET",
     "key",
     "Get the value of a key",
@@ -648,6 +673,11 @@ struct commandHelp {
     "Get the length of a list",
     2,
     "1.0.0" },
+    { "LMOVE",
+    "source destination LEFT|RIGHT LEFT|RIGHT",
+    "Pop an element from a list, push it to another list and return it",
+    2,
+    "6.2.0" },
     { "LOLWUT",
     "[VERSION version]",
     "Display some computer art and the Redis version",
@@ -659,7 +689,7 @@ struct commandHelp {
     2,
     "1.0.0" },
     { "LPOS",
-    "key element [FIRST rank] [COUNT num-matches] [MAXLEN len]",
+    "key element [RANK rank] [COUNT num-matches] [MAXLEN len]",
     "Return the index of matching elements on a list",
     2,
     "6.0.6" },
@@ -729,7 +759,7 @@ struct commandHelp {
     1,
     "1.0.0" },
     { "MIGRATE",
-    "host port key|"" destination-db timeout [COPY] [REPLACE] [AUTH password] [KEYS key]",
+    "host port key|"" destination-db timeout [COPY] [REPLACE] [AUTH password] [AUTH2 username password] [KEYS key]",
     "Atomically transfer a key from a Redis instance to another one.",
     0,
     "2.6.0" },
@@ -883,6 +913,11 @@ struct commandHelp {
     "Make the server a replica of another instance, or promote it as master.",
     9,
     "5.0.0" },
+    { "RESET",
+    "-",
+    "Reset the connection",
+    8,
+    "6.2" },
     { "RESTORE",
     "key ttl serialized-value [REPLACE] [ABSTTL] [IDLETIME seconds] [FREQ frequency]",
     "Create a key using the provided serialized value, previously obtained using DUMP.",
@@ -974,7 +1009,7 @@ struct commandHelp {
     8,
     "1.0.0" },
     { "SET",
-    "key value [EX seconds|PX milliseconds] [NX|XX] [KEEPTTL]",
+    "key value [EX seconds|PX milliseconds|KEEPTTL] [NX|XX] [GET]",
     "Set the string value of a key",
     1,
     "1.0.0" },
@@ -1033,6 +1068,11 @@ struct commandHelp {
     "Get all the members in a set",
     3,
     "1.0.0" },
+    { "SMISMEMBER",
+    "key member [member ...]",
+    "Returns the membership associated with the given elements for a set",
+    3,
+    "6.2.0" },
     { "SMOVE",
     "source destination member",
     "Move a member from one set to another",
@@ -1149,7 +1189,7 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XADD",
-    "key ID field value [field value ...]",
+    "key [MAXLEN [=|~] length] [NOMKSTREAM] *|ID field value [field value ...]",
     "Appends a new entry to a stream",
     14,
     "5.0.0" },
@@ -1164,7 +1204,7 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XGROUP",
-    "[CREATE key groupname id-or-$] [SETID key groupname id-or-$] [DESTROY key groupname] [DELCONSUMER key groupname consumername]",
+    "[CREATE key groupname ID|$ [MKSTREAM]] [SETID key groupname ID|$] [DESTROY key groupname] [CREATECONSUMER key groupname consumername] [DELCONSUMER key groupname consumername]",
     "Create, destroy, and manage consumer groups.",
     14,
     "5.0.0" },
@@ -1179,7 +1219,7 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XPENDING",
-    "key group [start end count] [consumer]",
+    "key group [[IDLE min-idle-time] start end count [consumer]]",
     "Return information and entries from a stream consumer group pending entries list, that are messages fetched but never acknowledged.",
     14,
     "5.0.0" },
@@ -1189,7 +1229,7 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XREAD",
-    "[COUNT count] [BLOCK milliseconds] STREAMS key [key ...] id [id ...]",
+    "[COUNT count] [BLOCK milliseconds] STREAMS key [key ...] ID [ID ...]",
     "Return never seen elements in multiple streams, with IDs greater than the ones reported by the caller for each stream. Can block.",
     14,
     "5.0.0" },
@@ -1204,12 +1244,12 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XTRIM",
-    "key MAXLEN [~] count",
+    "key MAXLEN [=|~] length",
     "Trims the stream to (approximately if '~' is passed) a certain size",
     14,
     "5.0.0" },
     { "ZADD",
-    "key [NX|XX] [CH] [INCR] score member [score member ...]",
+    "key [NX|XX] [GT|LT] [CH] [INCR] score member [score member ...]",
     "Add one or more members to a sorted set, or update its score if it already exists",
     4,
     "1.2.0" },
@@ -1223,11 +1263,26 @@ struct commandHelp {
     "Count the members in a sorted set with scores within the given values",
     4,
     "2.0.0" },
+    { "ZDIFF",
+    "numkeys key [key ...] [WITHSCORES]",
+    "Subtract multiple sorted sets",
+    4,
+    "6.2.0" },
+    { "ZDIFFSTORE",
+    "destination numkeys key [key ...]",
+    "Subtract multiple sorted sets and store the resulting sorted set in a new key",
+    4,
+    "6.2.0" },
     { "ZINCRBY",
     "key increment member",
     "Increment the score of a member in a sorted set",
     4,
     "1.2.0" },
+    { "ZINTER",
+    "numkeys key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX] [WITHSCORES]",
+    "Intersect multiple sorted sets",
+    4,
+    "6.2.0" },
     { "ZINTERSTORE",
     "destination numkeys key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX]",
     "Intersect multiple sorted sets and store the resulting sorted set in a new key",
@@ -1238,6 +1293,11 @@ struct commandHelp {
     "Count the number of members in a sorted set between a given lexicographical range",
     4,
     "2.8.9" },
+    { "ZMSCORE",
+    "key member [member ...]",
+    "Get the score associated with the given members in a sorted set",
+    4,
+    "6.2.0" },
     { "ZPOPMAX",
     "key [count]",
     "Remove and return members with the highest scores in a sorted set",
@@ -1318,6 +1378,11 @@ struct commandHelp {
     "Get the score associated with the given member in a sorted set",
     4,
     "1.2.0" },
+    { "ZUNION",
+    "numkeys key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX] [WITHSCORES]",
+    "Add multiple sorted sets",
+    4,
+    "6.2.0" },
     { "ZUNIONSTORE",
     "destination numkeys key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX]",
     "Add multiple sorted sets and store the resulting sorted set in a new key",
diff --git a/src/version.h b/src/version.h
index 89aef53fc..26505bfcb 100644
--- a/src/version.h
+++ b/src/version.h
@@ -1,2 +1,2 @@
-#define REDIS_VERSION "255.255.255"
-#define REDIS_VERSION_NUM 0x00ffffff
+#define REDIS_VERSION "6.1.240"
+#define REDIS_VERSION_NUM 0x000601f0

From 7993780dda22df01cebba42d16f805213d66e194 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Tue, 15 Dec 2020 15:30:24 +0800
Subject: [PATCH 0881/1098] Fix some wrong server.dirty increments (#8140)

Fix wrong server dirty increment in
* spopWithCountCommand
* hsetCommand
* ltrimCommand
* pfaddCommand

Some didn't increment the amount of fields (just one per command).
Others had excessive increments.
---
 src/db.c          | 3 +++
 src/hyperloglog.c | 2 +-
 src/t_hash.c      | 2 +-
 src/t_list.c      | 2 +-
 src/t_set.c       | 4 +---
 5 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/db.c b/src/db.c
index 5045935c3..4f27534fc 100644
--- a/src/db.c
+++ b/src/db.c
@@ -616,6 +616,9 @@ void flushAllDataAndResetRDB(int flags) {
         rdbSave(server.rdb_filename,rsiptr);
         server.dirty = saved_dirty;
     }
+
+    /* Without that extra dirty++, when db was already empty, FLUSHALL will
+     * not be replicated nor put into the AOF. */
     server.dirty++;
 #if defined(USE_JEMALLOC)
     /* jemalloc 5 doesn't release pages back to the OS when there's no traffic.
diff --git a/src/hyperloglog.c b/src/hyperloglog.c
index d018e975e..648c26a02 100644
--- a/src/hyperloglog.c
+++ b/src/hyperloglog.c
@@ -1211,7 +1211,7 @@ void pfaddCommand(client *c) {
     if (updated) {
         signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_STRING,"pfadd",c->argv[1],c->db->id);
-        server.dirty++;
+        server.dirty += updated;
         HLL_INVALIDATE_CACHE(hdr);
     }
     addReply(c, updated ? shared.cone : shared.czero);
diff --git a/src/t_hash.c b/src/t_hash.c
index ff9ac742e..51c7d6758 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -644,7 +644,7 @@ void hsetCommand(client *c) {
     }
     signalModifiedKey(c,c->db,c->argv[1]);
     notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id);
-    server.dirty++;
+    server.dirty += (c->argc - 2)/2;
 }
 
 void hincrbyCommand(client *c) {
diff --git a/src/t_list.c b/src/t_list.c
index 42b4f92df..106f960f6 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -508,7 +508,7 @@ void ltrimCommand(client *c) {
         notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id);
     }
     signalModifiedKey(c,c->db,c->argv[1]);
-    server.dirty++;
+    server.dirty += (ltrim + rtrim);
     addReply(c,shared.ok);
 }
 
diff --git a/src/t_set.c b/src/t_set.c
index 7c71dfc2f..fd9f4442a 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -476,7 +476,7 @@ void spopWithCountCommand(client *c) {
 
     /* Generate an SPOP keyspace notification */
     notifyKeyspaceEvent(NOTIFY_SET,"spop",c->argv[1],c->db->id);
-    server.dirty += count;
+    server.dirty += (count >= size) ? size : count;
 
     /* CASE 1:
      * The number of requested elements is greater than or equal to
@@ -492,7 +492,6 @@ void spopWithCountCommand(client *c) {
         /* Propagate this command as a DEL operation */
         rewriteClientCommandVector(c,2,shared.del,c->argv[1]);
         signalModifiedKey(c,c->db,c->argv[1]);
-        server.dirty++;
         return;
     }
 
@@ -594,7 +593,6 @@ void spopWithCountCommand(client *c) {
     decrRefCount(propargv[0]);
     preventCommandPropagation(c);
     signalModifiedKey(c,c->db,c->argv[1]);
-    server.dirty++;
 }
 
 void spopCommand(client *c) {

From 1f42bd70572c8e85fa431a66952c7b79eb182a87 Mon Sep 17 00:00:00 2001
From: filipe oliveira <filipecosta.90@gmail.com>
Date: Tue, 15 Dec 2020 20:03:05 +0000
Subject: [PATCH 0882/1098] Included in redis.conf explicit explanation of
 tls-protocol defaults (#8193)

---
 redis.conf | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/redis.conf b/redis.conf
index 849f171bc..af4b4be1f 100644
--- a/redis.conf
+++ b/redis.conf
@@ -196,9 +196,12 @@ tcp-keepalive 300
 #
 # tls-cluster yes
 
-# Explicitly specify TLS versions to support. Allowed values are case insensitive
-# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) or
-# any combination. To enable only TLSv1.2 and TLSv1.3, use:
+# By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended
+# that older formally deprecated versions are kept disabled to reduce the attack surface.
+# You can explicitly specify TLS versions to support.
+# Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2",
+# "TLSv1.3" (OpenSSL >= 1.1.1) or any combination.
+# To enable only TLSv1.2 and TLSv1.3, use:
 #
 # tls-protocols "TLSv1.2 TLSv1.3"
 

From 4f67d0b647caba9fa301a1a604214c0ecc40940a Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Wed, 16 Dec 2020 16:19:12 -0500
Subject: [PATCH 0883/1098] fix wrong comment in cluster.h (#8191)

---
 src/cluster.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cluster.h b/src/cluster.h
index 7e5f79c87..d58f350ce 100644
--- a/src/cluster.h
+++ b/src/cluster.h
@@ -40,7 +40,7 @@ typedef struct clusterLink {
     sds sndbuf;                 /* Packet send buffer */
     char *rcvbuf;               /* Packet reception buffer */
     size_t rcvbuf_len;          /* Used size of rcvbuf */
-    size_t rcvbuf_alloc;        /* Used size of rcvbuf */
+    size_t rcvbuf_alloc;        /* Allocated size of rcvbuf */
     struct clusterNode *node;   /* Node related to this link if any, or NULL */
 } clusterLink;
 

From 6413e5f81a8b6296f4bb57cae11f5b2534f5f663 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuancode@163.com>
Date: Thu, 17 Dec 2020 16:22:13 +0800
Subject: [PATCH 0884/1098] [Redis-benchmark] Use IP from CLUSTER NODE reply
 for first node too (#8154)

If we only has one node in cluster or before 8fdc857, we don't know myself ip, so we should use config.hostip for myself.
However, we should use the IP from the command response to update node->ip if it exists and is different from config.hostip

otherwise, when there's more than one node in cluster, if we use -h with virtual IP or DNS, benchmark doesn't show node real ip and port of myself even though it could get right IP and port by CLUSTER NODES command.
---
 src/redis-benchmark.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 4efed4b12..18e19b0e0 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -1182,8 +1182,8 @@ static int fetchClusterConfiguration() {
         }
         if (myself) {
             node = firstNode;
-            if (node->ip == NULL && ip != NULL) {
-                node->ip = ip;
+            if (ip != NULL && strcmp(node->ip, ip) != 0) {
+                node->ip = sdsnew(ip);
                 node->port = port;
             }
         } else {

From 407da77ea4f6de622d6f242ad16580aa6f4e7b85 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Thu, 17 Dec 2020 17:02:17 +0800
Subject: [PATCH 0885/1098] Fix comment of georadiusGeneric function (#8202)

---
 src/geo.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/geo.c b/src/geo.c
index 2fd1cb21b..925c090ed 100644
--- a/src/geo.c
+++ b/src/geo.c
@@ -488,9 +488,9 @@ void geoaddCommand(client *c) {
 /* GEORADIUS key x y radius unit [WITHDIST] [WITHHASH] [WITHCOORD] [ASC|DESC]
  *                               [COUNT count] [STORE key] [STOREDIST key]
  * GEORADIUSBYMEMBER key member radius unit ... options ...
- * GEOSEARCH key [FROMMEMBER member] [FORMLOG long lat] [BYRADIUS radius unit]
+ * GEOSEARCH key [FROMMEMBER member] [FROMLONLAT long lat] [BYRADIUS radius unit]
  *               [BYBOX width height unit] [WITHCORD] [WITHDIST] [WITHASH] [COUNT count] [ASC|DESC]
- * GEOSEARCHSTORE dest_key src_key [FROMMEMBER member] [FORMLOG long lat] [BYRADIUS radius unit]
+ * GEOSEARCHSTORE dest_key src_key [FROMMEMBER member] [FROMLONLAT long lat] [BYRADIUS radius unit]
  *               [BYBOX width height unit] [WITHCORD] [WITHDIST] [WITHASH] [COUNT count] [ASC|DESC] [STOREDIST]
  *  */
 void georadiusGeneric(client *c, int srcKeyIndex, int flags) {

From a56cbd30366ac7776bcd71e2ac8d9c421ca54cc8 Mon Sep 17 00:00:00 2001
From: Hanif Ariffin <hanif.ariffin.4326@gmail.com>
Date: Thu, 17 Dec 2020 03:00:48 -0800
Subject: [PATCH 0886/1098] More fixes to printf format specifier. (#7909)

mostly signed / unsigned mismatches.

Signed-off-by: Hanif Bin Ariffin <hanif.ariffin.4326@gmail.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/dict.c      | 8 ++++----
 src/rax.c       | 2 +-
 src/redis-cli.c | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/dict.c b/src/dict.c
index 4736dacd5..2c59dd466 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -1135,10 +1135,10 @@ size_t _dictGetStatsHt(char *buf, size_t bufsize, dictht *ht, int tableid) {
     /* Generate human readable stats. */
     l += snprintf(buf+l,bufsize-l,
         "Hash table %d stats (%s):\n"
-        " table size: %ld\n"
-        " number of elements: %ld\n"
-        " different slots: %ld\n"
-        " max chain length: %ld\n"
+        " table size: %lu\n"
+        " number of elements: %lu\n"
+        " different slots: %lu\n"
+        " max chain length: %lu\n"
         " avg chain length (counted): %.02f\n"
         " avg chain length (computed): %.02f\n"
         " Chain length distribution:\n",
diff --git a/src/rax.c b/src/rax.c
index 89345fe26..1ffb42287 100644
--- a/src/rax.c
+++ b/src/rax.c
@@ -1892,7 +1892,7 @@ void raxShow(rax *rax) {
 /* Used by debugnode() macro to show info about a given node. */
 void raxDebugShowNode(const char *msg, raxNode *n) {
     if (raxDebugMsg == 0) return;
-    printf("%s: %p [%.*s] key:%d size:%d children:",
+    printf("%s: %p [%.*s] key:%u size:%u children:",
         msg, (void*)n, (int)n->size, (char*)n->data, n->iskey, n->size);
     int numcld = n->iscompr ? 1 : n->size;
     raxNode **cldptr = raxNodeLastChildPtr(n) - (numcld-1);
diff --git a/src/redis-cli.c b/src/redis-cli.c
index 4f5b0fe03..5310a1b21 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -5578,7 +5578,7 @@ static int clusterManagerCommandCreate(int argc, char **argv) {
         if (last > CLUSTER_MANAGER_SLOTS || i == (masters_count - 1))
             last = CLUSTER_MANAGER_SLOTS - 1;
         if (last < first) last = first;
-        printf("Master[%d] -> Slots %lu - %lu\n", i, first, last);
+        printf("Master[%d] -> Slots %ld - %ld\n", i, first, last);
         master->slots_count = 0;
         for (j = first; j <= last; j++) {
             master->slots[j] = 1;

From 0f3e0cb4adf6bdbb7dbd70679d129ad5a528df9d Mon Sep 17 00:00:00 2001
From: Nick Revin <n@nrvn.cc>
Date: Thu, 17 Dec 2020 16:49:19 +0400
Subject: [PATCH 0887/1098] install redis-check-rdb and redis-check-aof as
 symlinks to redis-server (#5745)

---
 src/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Makefile b/src/Makefile
index 0329da8c9..32cedbac5 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -405,8 +405,8 @@ install: all
 	$(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(INSTALL_BIN)
 	$(REDIS_INSTALL) $(REDIS_BENCHMARK_NAME) $(INSTALL_BIN)
 	$(REDIS_INSTALL) $(REDIS_CLI_NAME) $(INSTALL_BIN)
-	$(REDIS_INSTALL) $(REDIS_CHECK_RDB_NAME) $(INSTALL_BIN)
-	$(REDIS_INSTALL) $(REDIS_CHECK_AOF_NAME) $(INSTALL_BIN)
+	@ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_CHECK_RDB_NAME)
+	@ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_CHECK_AOF_NAME)
 	@ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_SENTINEL_NAME)
 
 uninstall:

From f48afb4710e6e102b043add55a0649b22ef4a4ab Mon Sep 17 00:00:00 2001
From: Qu Chen <QuChen88@users.noreply.github.com>
Date: Thu, 17 Dec 2020 09:26:33 -0800
Subject: [PATCH 0888/1098] Handle binary safe string for REQUIREPASS and
 MASTERAUTH directives (#8200)

* Handle binary safe string for REQUIREPASS and MASTERAUTH directives.
---
 src/config.c        | 102 +++++++++++++++++++++++++++++++++++++-------
 src/replication.c   |  30 ++++++++-----
 src/server.h        |   2 +-
 tests/unit/auth.tcl |  41 ++++++++++++++++++
 4 files changed, 148 insertions(+), 27 deletions(-)

diff --git a/src/config.c b/src/config.c
index c858df3f3..4617e8678 100644
--- a/src/config.c
+++ b/src/config.c
@@ -166,6 +166,15 @@ typedef struct stringConfigData {
                                   be stored as a NULL value. */
 } stringConfigData;
 
+typedef struct sdsConfigData {
+    sds *config; /* Pointer to the server config this value is stored in. */
+    const char *default_value; /* Default value of the config on rewrite. */
+    int (*is_valid_fn)(sds val, char **err); /* Optional function to check validity of new value (generic doc above) */
+    int (*update_fn)(sds val, sds prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */
+    int convert_empty_to_null; /* Boolean indicating if empty SDS strings should
+                                  be stored as a NULL value. */
+} sdsConfigData;
+
 typedef struct enumConfigData {
     int *config; /* The pointer to the server config this value is stored in */
     configEnum *enum_value; /* The underlying enum type this data represents */
@@ -212,6 +221,7 @@ typedef struct numericConfigData {
 typedef union typeData {
     boolConfigData yesno;
     stringConfigData string;
+    sdsConfigData sds;
     enumConfigData enumd;
     numericConfigData numeric;
 } typeData;
@@ -512,7 +522,7 @@ void loadServerConfigFromString(char *config) {
             }
             server.repl_state = REPL_STATE_CONNECT;
         } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) {
-            if (strlen(argv[1]) > CONFIG_AUTHPASS_MAX_LEN) {
+            if (sdslen(argv[1]) > CONFIG_AUTHPASS_MAX_LEN) {
                 err = "Password is longer than CONFIG_AUTHPASS_MAX_LEN";
                 goto loaderr;
             }
@@ -524,10 +534,10 @@ void loadServerConfigFromString(char *config) {
             sdsfree(server.requirepass);
             server.requirepass = NULL;
             if (sdslen(argv[1])) {
-                sds aclop = sdscatprintf(sdsempty(),">%s",argv[1]);
+                sds aclop = sdscatlen(sdsnew(">"), argv[1], sdslen(argv[1]));
                 ACLSetUser(DefaultUser,aclop,sdslen(aclop));
                 sdsfree(aclop);
-                server.requirepass = sdsnew(argv[1]);
+                server.requirepass = sdsdup(argv[1]);
             } else {
                 ACLSetUser(DefaultUser,"nopass",-1);
             }
@@ -751,10 +761,10 @@ void configSetCommand(client *c) {
         sdsfree(server.requirepass);
         server.requirepass = NULL;
         if (sdslen(o->ptr)) {
-            sds aclop = sdscatprintf(sdsempty(),">%s",(char*)o->ptr);
+            sds aclop = sdscatlen(sdsnew(">"), o->ptr, sdslen(o->ptr));
             ACLSetUser(DefaultUser,aclop,sdslen(aclop));
             sdsfree(aclop);
-            server.requirepass = sdsnew(o->ptr);
+            server.requirepass = sdsdup(o->ptr);
         } else {
             ACLSetUser(DefaultUser,"nopass",-1);
         }
@@ -1330,6 +1340,28 @@ void rewriteConfigStringOption(struct rewriteConfigState *state, const char *opt
     rewriteConfigRewriteLine(state,option,line,force);
 }
 
+/* Rewrite a SDS string option. */
+void rewriteConfigSdsOption(struct rewriteConfigState *state, const char *option, sds value, const sds defvalue) {
+    int force = 1;
+    sds line;
+
+    /* If there is no value set, we don't want the SDS option
+     * to be present in the configuration at all. */
+    if (value == NULL) {
+        rewriteConfigMarkAsProcessed(state, option);
+        return;
+    }
+
+    /* Set force to zero if the value is set to its default. */
+    if (defvalue && sdscmp(value, defvalue) == 0) force = 0;
+
+    line = sdsnew(option);
+    line = sdscatlen(line, " ", 1);
+    line = sdscatrepr(line, value, sdslen(value));
+
+    rewriteConfigRewriteLine(state, option, line, force);
+}
+
 /* Rewrite a numerical (long long range) option. */
 void rewriteConfigNumericalOption(struct rewriteConfigState *state, const char *option, long long value, long long defvalue) {
     int force = value != defvalue;
@@ -1802,22 +1834,14 @@ static void boolConfigRewrite(typeData data, const char *name, struct rewriteCon
 
 /* String Configs */
 static void stringConfigInit(typeData data) {
-    if (data.string.convert_empty_to_null) {
-        *data.string.config = data.string.default_value ? zstrdup(data.string.default_value) : NULL;
-    } else {
-        *data.string.config = zstrdup(data.string.default_value);
-    }
+    *data.string.config = (data.string.convert_empty_to_null && !data.string.default_value) ? NULL : zstrdup(data.string.default_value);
 }
 
 static int stringConfigSet(typeData data, sds value, int update, char **err) {
     if (data.string.is_valid_fn && !data.string.is_valid_fn(value, err))
         return 0;
     char *prev = *data.string.config;
-    if (data.string.convert_empty_to_null) {
-        *data.string.config = value[0] ? zstrdup(value) : NULL;
-    } else {
-        *data.string.config = zstrdup(value);
-    }
+    *data.string.config = (data.string.convert_empty_to_null && !value[0]) ? NULL : zstrdup(value);
     if (update && data.string.update_fn && !data.string.update_fn(*data.string.config, prev, err)) {
         zfree(*data.string.config);
         *data.string.config = prev;
@@ -1835,6 +1859,38 @@ static void stringConfigRewrite(typeData data, const char *name, struct rewriteC
     rewriteConfigStringOption(state, name,*(data.string.config), data.string.default_value);
 }
 
+/* SDS Configs */
+static void sdsConfigInit(typeData data) {
+    *data.sds.config = (data.sds.convert_empty_to_null && !data.sds.default_value) ? NULL: sdsnew(data.sds.default_value);
+}
+
+static int sdsConfigSet(typeData data, sds value, int update, char **err) {
+    if (data.sds.is_valid_fn && !data.sds.is_valid_fn(value, err))
+        return 0;
+    sds prev = *data.sds.config;
+    *data.sds.config = (data.sds.convert_empty_to_null && (sdslen(value) == 0)) ? NULL : sdsdup(value);
+    if (update && data.sds.update_fn && !data.sds.update_fn(*data.sds.config, prev, err)) {
+        sdsfree(*data.sds.config);
+        *data.sds.config = prev;
+        return 0;
+    }
+    sdsfree(prev);
+    return 1;
+}
+
+static void sdsConfigGet(client *c, typeData data) {
+    if (*data.sds.config) {
+        addReplyBulkSds(c, sdsdup(*data.sds.config));
+    } else {
+        addReplyBulkCString(c, "");
+    }
+}
+
+static void sdsConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) {
+    rewriteConfigSdsOption(state, name, *(data.sds.config), data.sds.default_value ? sdsnew(data.sds.default_value) : NULL);
+}
+
+
 #define ALLOW_EMPTY_STRING 0
 #define EMPTY_STRING_IS_NULL 1
 
@@ -1850,6 +1906,18 @@ static void stringConfigRewrite(typeData data, const char *name, struct rewriteC
     } \
 }
 
+#define createSDSConfig(name, alias, modifiable, empty_to_null, config_addr, default, is_valid, update) { \
+    embedCommonConfig(name, alias, modifiable) \
+    embedConfigInterface(sdsConfigInit, sdsConfigSet, sdsConfigGet, sdsConfigRewrite) \
+    .data.sds = { \
+        .config = &(config_addr), \
+        .default_value = (default), \
+        .is_valid_fn = (is_valid), \
+        .update_fn = (update), \
+        .convert_empty_to_null = (empty_to_null), \
+    } \
+}
+
 /* Enum configs */
 static void enumConfigInit(typeData data) {
     *data.enumd.config = data.enumd.default_value;
@@ -2349,7 +2417,6 @@ standardConfig configs[] = {
     createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.pidfile, NULL, NULL, NULL),
     createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.slave_announce_ip, NULL, NULL, NULL),
     createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masteruser, NULL, NULL, NULL),
-    createStringConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL, NULL, NULL),
     createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL, NULL, NULL),
     createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.syslog_ident, "redis", NULL, NULL),
     createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.rdb_filename, "dump.rdb", isValidDBfilename, NULL),
@@ -2359,6 +2426,9 @@ standardConfig configs[] = {
     createStringConfig("aof_rewrite_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.aof_rewrite_cpulist, NULL, NULL, NULL),
     createStringConfig("bgsave_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bgsave_cpulist, NULL, NULL, NULL),
 
+    /* SDS Configs */
+    createSDSConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL, NULL, NULL),
+
     /* Enum Configs */
     createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE, NULL, NULL),
     createEnumConfig("syslog-facility", NULL, IMMUTABLE_CONFIG, syslog_facility_enum, server.syslog_facility, LOG_LOCAL0, NULL, NULL),
diff --git a/src/replication.c b/src/replication.c
index 64aa41390..34ff44087 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1835,6 +1835,7 @@ error:
  */
 #define SYNC_CMD_READ (1<<0)
 #define SYNC_CMD_WRITE (1<<1)
+#define SYNC_CMD_WRITE_SDS (1<<2)
 #define SYNC_CMD_FULL (SYNC_CMD_READ|SYNC_CMD_WRITE)
 char *sendSynchronousCommand(int flags, connection *conn, ...) {
 
@@ -1852,8 +1853,13 @@ char *sendSynchronousCommand(int flags, connection *conn, ...) {
         while(1) {
             arg = va_arg(ap, char*);
             if (arg == NULL) break;
-
-            cmdargs = sdscatprintf(cmdargs,"$%zu\r\n%s\r\n",strlen(arg),arg);
+            if (flags & SYNC_CMD_WRITE_SDS) {
+                cmdargs = sdscatprintf(cmdargs,"$%zu\r\n", sdslen((sds)arg));
+                cmdargs = sdscatsds(cmdargs, (sds)arg);
+                cmdargs = sdscat(cmdargs, "\r\n");
+            } else {
+                cmdargs = sdscatprintf(cmdargs,"$%zu\r\n%s\r\n",strlen(arg),arg);
+            }
             argslen++;
         }
 
@@ -2166,14 +2172,18 @@ void syncWithMaster(connection *conn) {
 
     /* AUTH with the master if required. */
     if (server.repl_state == REPL_STATE_SEND_AUTH) {
-        if (server.masteruser && server.masterauth) {
-            err = sendSynchronousCommand(SYNC_CMD_WRITE,conn,"AUTH",
-                                         server.masteruser,server.masterauth,NULL);
-            if (err) goto write_error;
-            server.repl_state = REPL_STATE_RECEIVE_AUTH;
-            return;
-        } else if (server.masterauth) {
-            err = sendSynchronousCommand(SYNC_CMD_WRITE,conn,"AUTH",server.masterauth,NULL);
+        if (server.masterauth) {
+            sds auth = sdsnew("AUTH");
+            if (server.masteruser) {
+                sds masteruser = sdsnew(server.masteruser);
+                err = sendSynchronousCommand(SYNC_CMD_WRITE | SYNC_CMD_WRITE_SDS, conn, auth,
+                                             masteruser, server.masterauth, NULL);
+                sdsfree(masteruser);
+            } else {
+                err = sendSynchronousCommand(SYNC_CMD_WRITE | SYNC_CMD_WRITE_SDS, conn, auth,
+                                             server.masterauth, NULL);
+            }
+            sdsfree(auth);
             if (err) goto write_error;
             server.repl_state = REPL_STATE_RECEIVE_AUTH;
             return;
diff --git a/src/server.h b/src/server.h
index 427d7c2f2..5c75aed08 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1386,7 +1386,7 @@ struct redisServer {
     int repl_diskless_sync_delay;   /* Delay to start a diskless repl BGSAVE. */
     /* Replication (slave) */
     char *masteruser;               /* AUTH with this user and masterauth with master */
-    char *masterauth;               /* AUTH with this password with master */
+    sds masterauth;                 /* AUTH with this password with master */
     char *masterhost;               /* Hostname of master */
     int masterport;                 /* Port of master */
     int repl_timeout;               /* Timeout after N seconds of master idle */
diff --git a/tests/unit/auth.tcl b/tests/unit/auth.tcl
index 9080d4bf7..b63cf0126 100644
--- a/tests/unit/auth.tcl
+++ b/tests/unit/auth.tcl
@@ -25,3 +25,44 @@ start_server {tags {"auth"} overrides {requirepass foobar}} {
         r incr foo
     } {101}
 }
+
+start_server {tags {"auth_binary_password"}} {
+    test {AUTH fails when binary password is wrong} {
+        r config set requirepass "abc\x00def"
+        catch {r auth abc} err
+        set _ $err
+    } {WRONGPASS*}
+
+    test {AUTH succeeds when binary password is correct} {
+        r config set requirepass "abc\x00def"
+        r auth "abc\x00def"
+    } {OK}
+
+    start_server {tags {"masterauth"}} {
+        set master [srv -1 client]
+        set master_host [srv -1 host]
+        set master_port [srv -1 port]
+        set slave [srv 0 client]
+
+        test {MASTERAUTH test with binary password} {
+            $master config set requirepass "abc\x00def"
+
+            # Configure the replica with masterauth
+            set loglines [count_log_lines 0]
+            $slave slaveof $master_host $master_port
+            $slave config set masterauth "abc"
+
+            # Verify replica is not able to sync with master
+            wait_for_log_messages 0 {"*Unable to AUTH to MASTER*"} $loglines 1000 10
+            assert_equal {down} [s 0 master_link_status]
+            
+            # Test replica with the correct masterauth
+            $slave config set masterauth "abc\x00def"
+            wait_for_condition 50 100 {
+                [s 0 master_link_status] eq {up}
+            } else {
+                fail "Can't turn the instance into a replica"
+            }
+        }
+    }
+}

From 11b3325e9999721d35ec64afac7b917664f6291b Mon Sep 17 00:00:00 2001
From: Qu Chen <QuChen88@users.noreply.github.com>
Date: Thu, 17 Dec 2020 11:58:58 -0800
Subject: [PATCH 0889/1098] Not over-allocate client query buffer when reading
 large objects. (#5954)

In response to large client query buffer optimization introduced in 1898e6c. The calculation of the amount of
remaining bytes we need to write to the query buffer was calculated wrong, as a result we are unnecessarily
growing the client query buffer by sdslen(c->querybuf) always. This fix corrects that behavior.

Please note the previous behavior prior to the before-mentioned change was correctly calculating the remaining
additional bytes, and this change makes that calculate to be consistent.

Useful context, the argument of size `ll` starts at qb_pos (which is now the beginning of the sds), but much of it
may have already been read from the socket, so we only need to grow the sds for the remainder of it.
---
 src/networking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index df30206a3..b81060cbb 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1851,7 +1851,7 @@ int processMultibulkBuffer(client *c) {
                     c->qb_pos = 0;
                     /* Hint the sds library about the amount of bytes this string is
                      * going to contain. */
-                    c->querybuf = sdsMakeRoomFor(c->querybuf,ll+2);
+                    c->querybuf = sdsMakeRoomFor(c->querybuf,ll+2-sdslen(c->querybuf));
                 }
             }
             c->bulklen = ll;

From 51eb0da824d2dae45bf5966e83f2089fba8d9c24 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Fri, 18 Dec 2020 20:55:39 +0800
Subject: [PATCH 0890/1098] Fix command reset's arity (#8212)

---
 src/server.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index ac43c32e7..de7e2736d 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1059,7 +1059,7 @@ struct redisCommand redisCommandTable[] = {
      "read-only @string",
      0,lcsGetKeys,0,0,0,0,0,0},
 
-    {"reset",resetCommand,-1,
+    {"reset",resetCommand,1,
      "no-script ok-stale ok-loading fast @connection",
      0,NULL,0,0,0,0,0,0}
 };

From 4c13945c370b98b69886c6bcd1e71d4934344cae Mon Sep 17 00:00:00 2001
From: valentinogeron <valentino@redislabs.com>
Date: Mon, 21 Dec 2020 15:40:20 +0200
Subject: [PATCH 0891/1098] Fix PFDEBUG commands flag (#8222)

- Mark it as a @hyperloglog command (ACL)
- Should not be allowed in OOM
- Add firstkey, lastkey, step
- Add comment that explains the 'write' flag
---
 src/server.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/server.c b/src/server.c
index de7e2736d..279e75f2a 100644
--- a/src/server.c
+++ b/src/server.c
@@ -975,9 +975,12 @@ struct redisCommand redisCommandTable[] = {
      "write use-memory @hyperloglog",
      0,NULL,1,-1,1,0,0,0},
 
+    /* Unlike PFCOUNT that is considered as a read-only command (although
+     * it changes a bit), PFDEBUG may change the entire key when converting
+     * from sparse to dense representation */
     {"pfdebug",pfdebugCommand,-3,
-     "admin write",
-     0,NULL,0,0,0,0,0,0},
+     "admin write use-memory @hyperloglog",
+     0,NULL,2,2,1,0,0,0},
 
     {"xadd",xaddCommand,-5,
      "write use-memory fast random @stream",

From 4bc14da2b3330c7ce8b9366ea3880eba65ba2ff9 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Tue, 22 Dec 2020 14:57:45 +0800
Subject: [PATCH 0892/1098] Fix some redundancy use of semicolon in do-while
 macros (#8221)

* Fix some redundancy use of semicolon in do-while macros
---
 deps/hiredis/async_private.h | 2 +-
 src/config.c                 | 7 +++----
 src/debugmacro.h             | 2 +-
 src/dict.c                   | 2 +-
 src/modules/testmodule.c     | 2 +-
 src/quicklist.c              | 2 +-
 src/server.h                 | 4 ++--
 src/testhelp.h               | 4 ++--
 8 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/deps/hiredis/async_private.h b/deps/hiredis/async_private.h
index b9d23fffd..ea0558d42 100644
--- a/deps/hiredis/async_private.h
+++ b/deps/hiredis/async_private.h
@@ -51,7 +51,7 @@
 #define _EL_CLEANUP(ctx) do { \
         if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \
         ctx->ev.cleanup = NULL; \
-    } while(0);
+    } while(0)
 
 static inline void refreshTimeout(redisAsyncContext *ctx) {
     #define REDIS_TIMER_ISSET(tvp) \
diff --git a/src/config.c b/src/config.c
index 4617e8678..a92b41624 100644
--- a/src/config.c
+++ b/src/config.c
@@ -915,7 +915,7 @@ badfmt: /* Bad format errors */
         addReplyBulkCString(c,_var ? _var : ""); \
         matches++; \
     } \
-} while(0);
+} while(0)
 
 #define config_get_bool_field(_name,_var) do { \
     if (stringmatch(pattern,_name,1)) { \
@@ -923,7 +923,7 @@ badfmt: /* Bad format errors */
         addReplyBulkCString(c,_var ? "yes" : "no"); \
         matches++; \
     } \
-} while(0);
+} while(0)
 
 #define config_get_numerical_field(_name,_var) do { \
     if (stringmatch(pattern,_name,1)) { \
@@ -932,8 +932,7 @@ badfmt: /* Bad format errors */
         addReplyBulkCString(c,buf); \
         matches++; \
     } \
-} while(0);
-
+} while(0)
 
 void configGetCommand(client *c) {
     robj *o = c->argv[2];
diff --git a/src/debugmacro.h b/src/debugmacro.h
index ded2d2667..58e6577e5 100644
--- a/src/debugmacro.h
+++ b/src/debugmacro.h
@@ -38,4 +38,4 @@
         fprintf(fp,__VA_ARGS__);                                             \
         fprintf(fp,"\n");                                                    \
         fclose(fp);                                                          \
-    } while (0);
+    } while (0)
diff --git a/src/dict.c b/src/dict.c
index 2c59dd466..4f0916a27 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -1215,7 +1215,7 @@ dictType BenchmarkDictType = {
 #define end_benchmark(msg) do { \
     elapsed = timeInMilliseconds()-start; \
     printf(msg ": %ld items in %lld ms\n", count, elapsed); \
-} while(0);
+} while(0)
 
 /* dict-benchmark [count] */
 int main(int argc, char **argv) {
diff --git a/src/modules/testmodule.c b/src/modules/testmodule.c
index 5634530dd..078c02c5c 100644
--- a/src/modules/testmodule.c
+++ b/src/modules/testmodule.c
@@ -364,7 +364,7 @@ int TestAssertIntegerReply(RedisModuleCtx *ctx, RedisModuleCallReply *reply, lon
     do { \
         RedisModule_Log(ctx,"warning","Testing %s", name); \
         reply = RedisModule_Call(ctx,name,__VA_ARGS__); \
-    } while (0);
+    } while (0)
 
 /* TEST.IT -- Run all the tests. */
 int TestIt(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
diff --git a/src/quicklist.c b/src/quicklist.c
index 5ed9b6a5d..4dc3576ae 100644
--- a/src/quicklist.c
+++ b/src/quicklist.c
@@ -69,7 +69,7 @@ static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536};
         printf("%s:%s:%d:\t", __FILE__, __FUNCTION__, __LINE__);               \
         printf(__VA_ARGS__);                                                   \
         printf("\n");                                                          \
-    } while (0);
+    } while (0)
 #endif
 
 /* Bookmarks forward declarations */
diff --git a/src/server.h b/src/server.h
index 5c75aed08..8a1f191f3 100644
--- a/src/server.h
+++ b/src/server.h
@@ -605,7 +605,7 @@ typedef struct RedisModuleIO {
     iovar.ver = 0; \
     iovar.key = keyptr; \
     iovar.ctx = NULL; \
-} while(0);
+} while(0)
 
 /* This is a structure used to export DEBUG DIGEST capabilities to Redis
  * modules. We want to capture both the ordered and unordered elements of
@@ -621,7 +621,7 @@ typedef struct RedisModuleDigest {
 #define moduleInitDigestContext(mdvar) do { \
     memset(mdvar.o,0,sizeof(mdvar.o)); \
     memset(mdvar.x,0,sizeof(mdvar.x)); \
-} while(0);
+} while(0)
 
 /* Objects encoding. Some kind of objects like Strings and Hashes can be
  * internally represented in multiple ways. The 'encoding' field of the object
diff --git a/src/testhelp.h b/src/testhelp.h
index 450334046..c6c1b55bf 100644
--- a/src/testhelp.h
+++ b/src/testhelp.h
@@ -44,7 +44,7 @@ int __test_num = 0;
 #define test_cond(descr,_c) do { \
     __test_num++; printf("%d - %s: ", __test_num, descr); \
     if(_c) printf("PASSED\n"); else {printf("FAILED\n"); __failed_tests++;} \
-} while(0);
+} while(0)
 #define test_report() do { \
     printf("%d tests, %d passed, %d failed\n", __test_num, \
                     __test_num-__failed_tests, __failed_tests); \
@@ -52,6 +52,6 @@ int __test_num = 0;
         printf("=== WARNING === We have failed tests here...\n"); \
         exit(1); \
     } \
-} while(0);
+} while(0)
 
 #endif

From 411c18bbce26dacf39e4dd792e4a091a48270349 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 22 Dec 2020 12:03:49 +0200
Subject: [PATCH 0893/1098] Remove read-only flag from non-keyspace cmds,
 different approach for EXEC to propagate MULTI (#8216)

In the distant history there was only the read flag for commands, and whatever
command that didn't have the read flag was a write one.
Then we added the write flag, but some portions of the code still used !read
Also some commands that don't work on the keyspace at all, still have the read
flag.

Changes in this commit:
1. remove the read-only flag from TIME, ECHO, ROLE and LASTSAVE

2. EXEC command used to decides if it should propagate a MULTI by looking at
   the command flags (!read & !admin).
   When i was about to change it to look at the write flag instead, i realized
   that this would cause it not to propagate a MULTI for PUBLISH, EVAL, and
   SCRIPT, all 3 are not marked as either a read command or a write one (as
   they should), but all 3 are calling forceCommandPropagation.

   So instead of introducing a new flag to denote a command that "writes" but
   not into the keyspace, and still needs propagation, i decided to rely on
   the forceCommandPropagation, and just fix the code to propagate MULTI when
   needed rather than depending on the command flags at all.

   The implication of my change then is that now it won't decide to propagate
   MULTI when it sees one of these: SELECT, PING, INFO, COMMAND, TIME and
   other commands which are neither read nor write.

3. Changing getNodeByQuery and clusterRedirectBlockedClientIfNeeded in
   cluster.c to look at !write rather than read flag.
   This should have no implications, since these code paths are only reachable
   for commands which access keys, and these are always marked as either read
   or write.

This commit improve MULTI propagation tests, for modules and a bunch of
other special cases, all of which used to pass already before that commit.
the only one that test change that uncovered a change of behavior is the
one that DELs a non-existing key, it used to propagate an empty
multi-exec block, and no longer does.
---
 src/cluster.c                      |  11 ++-
 src/module.c                       |   2 +-
 src/multi.c                        |  24 ++----
 src/scripting.c                    |   4 +-
 src/server.c                       |  26 ++++---
 src/server.h                       |   4 +-
 tests/modules/propagate.c          |  44 +++++++----
 tests/unit/moduleapi/propagate.tcl | 114 ++++++++++++++++++++++++-----
 tests/unit/multi.tcl               |  87 +++++++++++++++++++++-
 9 files changed, 242 insertions(+), 74 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 8651a81d3..2cd6b2521 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -5763,7 +5763,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
              * cluster is down. */
             if (error_code) *error_code = CLUSTER_REDIR_DOWN_STATE;
             return NULL;
-        } else if (!(cmd->flags & CMD_READONLY) && !(cmd->proc == evalCommand)
+        } else if ((cmd->flags & CMD_WRITE) && !(cmd->proc == evalCommand)
                 && !(cmd->proc == evalShaCommand))
         {
             /* The cluster is configured to allow read only commands
@@ -5812,11 +5812,10 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
     /* Handle the read-only client case reading from a slave: if this
      * node is a slave and the request is about a hash slot our master
      * is serving, we can reply without redirection. */
-    int is_readonly_command = (c->cmd->flags & CMD_READONLY) ||
-                              (c->cmd->proc == execCommand && !(c->mstate.cmd_inv_flags & CMD_READONLY));
+    int is_write_command = (c->cmd->flags & CMD_WRITE) ||
+                           (c->cmd->proc == execCommand && (c->mstate.cmd_flags & CMD_WRITE));
     if (c->flags & CLIENT_READONLY &&
-        (is_readonly_command || cmd->proc == evalCommand ||
-         cmd->proc == evalShaCommand) &&
+        (!is_write_command || cmd->proc == evalCommand || cmd->proc == evalShaCommand) &&
         nodeIsSlave(myself) &&
         myself->slaveof == n)
     {
@@ -5901,7 +5900,7 @@ int clusterRedirectBlockedClientIfNeeded(client *c) {
             /* if the client is read-only and attempting to access key that our
              * replica can handle, allow it. */
             if ((c->flags & CLIENT_READONLY) &&
-                (c->lastcmd->flags & CMD_READONLY) &&
+                !(c->lastcmd->flags & CMD_WRITE) &&
                 nodeIsSlave(myself) && myself->slaveof == node)
             {
                 node = myself;
diff --git a/src/module.c b/src/module.c
index da9ac29e8..11f5f4489 100644
--- a/src/module.c
+++ b/src/module.c
@@ -1629,7 +1629,7 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) {
         ctx->saved_oparray = server.also_propagate;
         redisOpArrayInit(&server.also_propagate);
     }
-    execCommandPropagateMulti(ctx->client);
+    execCommandPropagateMulti(ctx->client->db->id);
 }
 
 /* Replicate the specified command and arguments to slaves and AOF, as effect
diff --git a/src/multi.c b/src/multi.c
index a2f9ecccf..af0b0c612 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -127,15 +127,15 @@ void beforePropagateMultiOrExec(int multi) {
 
 /* Send a MULTI command to all the slaves and AOF file. Check the execCommand
  * implementation for more information. */
-void execCommandPropagateMulti(client *c) {
+void execCommandPropagateMulti(int dbid) {
     beforePropagateMultiOrExec(1);
-    propagate(server.multiCommand,c->db->id,&shared.multi,1,
+    propagate(server.multiCommand,dbid,&shared.multi,1,
               PROPAGATE_AOF|PROPAGATE_REPL);
 }
 
-void execCommandPropagateExec(client *c) {
+void execCommandPropagateExec(int dbid) {
     beforePropagateMultiOrExec(0);
-    propagate(server.execCommand,c->db->id,&shared.exec,1,
+    propagate(server.execCommand,dbid,&shared.exec,1,
               PROPAGATE_AOF|PROPAGATE_REPL);
 }
 
@@ -162,7 +162,6 @@ void execCommand(client *c) {
     robj **orig_argv;
     int orig_argc;
     struct redisCommand *orig_cmd;
-    int must_propagate = 0; /* Need to propagate MULTI/EXEC to AOF / slaves? */
     int was_master = server.masterhost == NULL;
 
     if (!(c->flags & CLIENT_MULTI)) {
@@ -202,19 +201,6 @@ void execCommand(client *c) {
         c->argv = c->mstate.commands[j].argv;
         c->cmd = c->mstate.commands[j].cmd;
 
-        /* Propagate a MULTI request once we encounter the first command which
-         * is not readonly nor an administrative one.
-         * This way we'll deliver the MULTI/..../EXEC block as a whole and
-         * both the AOF and the replication link will have the same consistency
-         * and atomicity guarantees. */
-        if (!must_propagate &&
-            !server.loading &&
-            !(c->cmd->flags & (CMD_READONLY|CMD_ADMIN)))
-        {
-            execCommandPropagateMulti(c);
-            must_propagate = 1;
-        }
-
         /* ACL permissions are also checked at the time of execution in case
          * they were changed after the commands were ququed. */
         int acl_errpos;
@@ -265,7 +251,7 @@ void execCommand(client *c) {
 
     /* Make sure the EXEC command will be propagated as well if MULTI
      * was already propagated. */
-    if (must_propagate) {
+    if (server.propagate_in_transaction) {
         int is_master = server.masterhost == NULL;
         server.dirty++;
         beforePropagateMultiOrExec(0);
diff --git a/src/scripting.c b/src/scripting.c
index 8dca84478..bc137de25 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -721,7 +721,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) {
         server.lua_write_dirty &&
         server.lua_repl != PROPAGATE_NONE)
     {
-        execCommandPropagateMulti(server.lua_caller);
+        execCommandPropagateMulti(server.lua_caller->db->id);
         server.lua_multi_emitted = 1;
         /* Now we are in the MULTI context, the lua_client should be
          * flag as CLIENT_MULTI. */
@@ -1638,7 +1638,7 @@ void evalGenericCommand(client *c, int evalsha) {
     if (server.lua_replicate_commands) {
         preventCommandPropagation(c);
         if (server.lua_multi_emitted) {
-            execCommandPropagateExec(c);
+            execCommandPropagateExec(c->db->id);
         }
     }
 
diff --git a/src/server.c b/src/server.c
index 279e75f2a..29c810928 100644
--- a/src/server.c
+++ b/src/server.c
@@ -115,9 +115,9 @@ struct redisServer server; /* Server global state */
  *
  * write:       Write command (may modify the key space).
  *
- * read-only:   All the non special commands just reading from keys without
- *              changing the content, or returning other information like
- *              the TIME command. Special commands such administrative commands
+ * read-only:   Commands just reading from keys without changing the content.
+ *              Note that commands that don't read from the keyspace such as
+ *              TIME, SELECT, INFO, administrative commands, and connection
  *              or transaction related commands (multi, exec, discard, ...)
  *              are not flagged as read-only commands, since they affect the
  *              server or the connection in other ways.
@@ -685,7 +685,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"echo",echoCommand,2,
-     "read-only fast @connection",
+     "fast @connection",
      0,NULL,0,0,0,0,0,0},
 
     {"save",saveCommand,1,
@@ -705,7 +705,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"lastsave",lastsaveCommand,1,
-     "read-only random fast ok-loading ok-stale @admin @dangerous",
+     "random fast ok-loading ok-stale @admin @dangerous",
      0,NULL,0,0,0,0,0,0},
 
     {"type",typeCommand,2,
@@ -781,7 +781,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"role",roleCommand,1,
-     "ok-loading ok-stale no-script fast read-only @dangerous",
+     "ok-loading ok-stale no-script fast @dangerous",
      0,NULL,0,0,0,0,0,0},
 
     {"debug",debugCommand,-2,
@@ -891,7 +891,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"time",timeCommand,1,
-     "read-only random fast ok-loading ok-stale",
+     "random fast ok-loading ok-stale",
      0,NULL,0,0,0,0,0,0},
 
     {"bitop",bitopCommand,-4,
@@ -3377,6 +3377,14 @@ struct redisCommand *lookupCommandOrOriginal(sds name) {
 void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc,
                int flags)
 {
+    /* Propagate a MULTI request once we encounter the first command which
+     * is a write command.
+     * This way we'll deliver the MULTI/..../EXEC block as a whole and
+     * both the AOF and the replication link will have the same consistency
+     * and atomicity guarantees. */
+    if (server.in_exec && !server.propagate_in_transaction)
+        execCommandPropagateMulti(dbid);
+
     if (server.aof_state != AOF_OFF && flags & PROPAGATE_AOF)
         feedAppendOnlyFile(cmd,dbid,argv,argc);
     if (flags & PROPAGATE_REPL)
@@ -3607,7 +3615,7 @@ void call(client *c, int flags) {
                 !(c->flags & CLIENT_MULTI) &&
                 !(flags & CMD_CALL_NOWRAP))
             {
-                execCommandPropagateMulti(c);
+                execCommandPropagateMulti(c->db->id);
                 multi_emitted = 1;
             }
 
@@ -3622,7 +3630,7 @@ void call(client *c, int flags) {
             }
 
             if (multi_emitted) {
-                execCommandPropagateExec(c);
+                execCommandPropagateExec(c->db->id);
             }
         }
         redisOpArrayFree(&server.also_propagate);
diff --git a/src/server.h b/src/server.h
index 8a1f191f3..04fd709a8 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1861,8 +1861,8 @@ void touchWatchedKeysOnFlush(int dbid);
 void discardTransaction(client *c);
 void flagTransaction(client *c);
 void execCommandAbort(client *c, sds error);
-void execCommandPropagateMulti(client *c);
-void execCommandPropagateExec(client *c);
+void execCommandPropagateMulti(int dbid);
+void execCommandPropagateExec(int dbid);
 void beforePropagateMultiOrExec(int multi);
 
 /* Redis object implementation */
diff --git a/tests/modules/propagate.c b/tests/modules/propagate.c
index 13277b19d..70cddacbd 100644
--- a/tests/modules/propagate.c
+++ b/tests/modules/propagate.c
@@ -51,18 +51,31 @@ void timerHandler(RedisModuleCtx *ctx, void *data) {
     RedisModule_Replicate(ctx,"INCR","c","timer");
     times++;
 
-    if (times < 10)
+    if (times < 3)
         RedisModule_CreateTimer(ctx,100,timerHandler,NULL);
     else
         times = 0;
 }
 
+int propagateTestTimerCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+{
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+
+    RedisModuleTimerID timer_id =
+        RedisModule_CreateTimer(ctx,100,timerHandler,NULL);
+    REDISMODULE_NOT_USED(timer_id);
+
+    RedisModule_ReplyWithSimpleString(ctx,"OK");
+    return REDISMODULE_OK;
+}
+
 /* The thread entry point. */
 void *threadMain(void *arg) {
     REDISMODULE_NOT_USED(arg);
     RedisModuleCtx *ctx = RedisModule_GetThreadSafeContext(NULL);
     RedisModule_SelectDb(ctx,9); /* Tests ran in database number 9. */
-    for (int i = 0; i < 10; i++) {
+    for (int i = 0; i < 3; i++) {
         RedisModule_ThreadSafeContextLock(ctx);
         RedisModule_Replicate(ctx,"INCR","c","a-from-thread");
         RedisModule_Replicate(ctx,"INCR","c","b-from-thread");
@@ -72,15 +85,11 @@ void *threadMain(void *arg) {
     return NULL;
 }
 
-int propagateTestCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+int propagateTestThreadCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
 {
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
 
-    RedisModuleTimerID timer_id =
-        RedisModule_CreateTimer(ctx,100,timerHandler,NULL);
-    REDISMODULE_NOT_USED(timer_id);
-
     pthread_t tid;
     if (pthread_create(&tid,NULL,threadMain,NULL) != 0)
         return RedisModule_ReplyWithError(ctx,"-ERR Can't start thread");
@@ -90,7 +99,7 @@ int propagateTestCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc
     return REDISMODULE_OK;
 }
 
-int propagateTest2Command(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+int propagateTestSimpleCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
 {
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
@@ -102,7 +111,7 @@ int propagateTest2Command(RedisModuleCtx *ctx, RedisModuleString **argv, int arg
     return REDISMODULE_OK;
 }
 
-int propagateTest3Command(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
+int propagateTestMixedCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
 {
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
@@ -129,18 +138,23 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     if (RedisModule_Init(ctx,"propagate-test",1,REDISMODULE_APIVER_1)
             == REDISMODULE_ERR) return REDISMODULE_ERR;
 
-    if (RedisModule_CreateCommand(ctx,"propagate-test",
-                propagateTestCommand,
+    if (RedisModule_CreateCommand(ctx,"propagate-test.timer",
+                propagateTestTimerCommand,
                 "",1,1,1) == REDISMODULE_ERR)
             return REDISMODULE_ERR;
 
-    if (RedisModule_CreateCommand(ctx,"propagate-test-2",
-                propagateTest2Command,
+    if (RedisModule_CreateCommand(ctx,"propagate-test.thread",
+                propagateTestThreadCommand,
                 "",1,1,1) == REDISMODULE_ERR)
             return REDISMODULE_ERR;
 
-    if (RedisModule_CreateCommand(ctx,"propagate-test-3",
-                propagateTest3Command,
+    if (RedisModule_CreateCommand(ctx,"propagate-test.simple",
+                propagateTestSimpleCommand,
+                "",1,1,1) == REDISMODULE_ERR)
+            return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx,"propagate-test.mixed",
+                propagateTestMixedCommand,
                 "",1,1,1) == REDISMODULE_ERR)
             return REDISMODULE_ERR;
 
diff --git a/tests/unit/moduleapi/propagate.tcl b/tests/unit/moduleapi/propagate.tcl
index aa0f55e5e..adebd37a6 100644
--- a/tests/unit/moduleapi/propagate.tcl
+++ b/tests/unit/moduleapi/propagate.tcl
@@ -14,25 +14,103 @@ tags "modules" {
                 # Start the replication process...
                 $replica replicaof $master_host $master_port
                 wait_for_sync $replica
-
                 after 1000
-                $master propagate-test
 
-                wait_for_condition 5000 10 {
-                    ([$replica get timer] eq "10") && \
-                    ([$replica get a-from-thread] eq "10")
-                } else {
-                    fail "The two counters don't match the expected value."
+                test {module propagates from timer} {
+                    set repl [attach_to_replication_stream]
+
+                    $master propagate-test.timer
+
+                    wait_for_condition 5000 10 {
+                        [$replica get timer] eq "3"
+                    } else {
+                        fail "The two counters don't match the expected value."
+                    }
+
+                    assert_replication_stream $repl {
+                        {select *}
+                        {multi}
+                        {incr timer}
+                        {exec}
+                        {multi}
+                        {incr timer}
+                        {exec}
+                        {multi}
+                        {incr timer}
+                        {exec}
+                    }
+                    close_replication_stream $repl
                 }
 
-                $master propagate-test-2
-                $master propagate-test-3
-                $master multi
-                $master propagate-test-2
-                $master propagate-test-3
-                $master exec
-                wait_for_ofs_sync $master $replica
+                test {module propagates from thread} {
+                    set repl [attach_to_replication_stream]
 
+                    $master propagate-test.thread
+
+                    wait_for_condition 5000 10 {
+                        [$replica get a-from-thread] eq "3"
+                    } else {
+                        fail "The two counters don't match the expected value."
+                    }
+
+                    assert_replication_stream $repl {
+                        {select *}
+                        {incr a-from-thread}
+                        {incr b-from-thread}
+                        {incr a-from-thread}
+                        {incr b-from-thread}
+                        {incr a-from-thread}
+                        {incr b-from-thread}
+                    }
+                    close_replication_stream $repl
+                }
+
+                test {module propagates from from command} {
+                    set repl [attach_to_replication_stream]
+
+                    $master propagate-test.simple
+                    $master propagate-test.mixed
+
+                    # Note the 'after-call' propagation below is out of order (known limitation)
+                    assert_replication_stream $repl {
+                        {select *}
+                        {multi}
+                        {incr counter-1}
+                        {incr counter-2}
+                        {exec}
+                        {multi}
+                        {incr using-call}
+                        {incr after-call}
+                        {incr counter-1}
+                        {incr counter-2}
+                        {exec}
+                    }
+                    close_replication_stream $repl
+                }
+
+                test {module propagates from from multi-exec} {
+                    set repl [attach_to_replication_stream]
+
+                    $master multi
+                    $master propagate-test.simple
+                    $master propagate-test.mixed
+                    $master exec
+                    wait_for_ofs_sync $master $replica
+
+                    # Note the 'after-call' propagation below is out of order (known limitation)
+                    assert_replication_stream $repl {
+                        {select *}
+                        {multi}
+                        {incr counter-1}
+                        {incr counter-2}
+                        {incr using-call}
+                        {incr after-call}
+                        {incr counter-1}
+                        {incr counter-2}
+                        {exec}
+                    }
+                    close_replication_stream $repl
+                }
                 assert_equal [s -1 unexpected_error_replies] 0
             }
         }
@@ -47,11 +125,11 @@ tags "modules aof" {
             r config set auto-aof-rewrite-percentage 0 ; # Disable auto-rewrite.
             waitForBgrewriteaof r
 
-            r propagate-test-2
-            r propagate-test-3
+            r propagate-test.simple
+            r propagate-test.mixed
             r multi
-            r propagate-test-2
-            r propagate-test-3
+            r propagate-test.simple
+            r propagate-test.mixed
             r exec
 
             # Load the AOF
diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl
index 43259b1c0..25e417055 100644
--- a/tests/unit/multi.tcl
+++ b/tests/unit/multi.tcl
@@ -299,10 +299,14 @@ start_server {tags {"multi"}} {
         r multi
         r del foo
         r exec
+
+        # add another command so that when we see it we know multi-exec wasn't
+        # propagated
+        r incr foo
+
         assert_replication_stream $repl {
             {select *}
-            {multi}
-            {exec}
+            {incr foo}
         }
         close_replication_stream $repl
     }
@@ -521,4 +525,83 @@ start_server {tags {"multi"}} {
 
         list $m $res
     } {OK {{} {} {} {} {} {} {} {}}}
+
+    test {MULTI propagation of PUBLISH} {
+        set repl [attach_to_replication_stream]
+
+        # make sure that PUBLISH inside MULTI is propagated in a transaction
+        r multi
+        r publish bla bla
+        r exec
+
+        assert_replication_stream $repl {
+            {select *}
+            {multi}
+            {publish bla bla}
+            {exec}
+        }
+        close_replication_stream $repl
+    }
+
+    test {MULTI propagation of SCRIPT LOAD} {
+        set repl [attach_to_replication_stream]
+
+        # make sure that SCRIPT LOAD inside MULTI is propagated in a transaction
+        r multi
+        r script load {redis.call('set', KEYS[1], 'foo')}
+        set res [r exec]
+        set sha [lindex $res 0]
+
+        assert_replication_stream $repl {
+            {select *}
+            {multi}
+            {script load *}
+            {exec}
+        }
+        close_replication_stream $repl
+    }
+
+    test {MULTI propagation of SCRIPT LOAD} {
+        set repl [attach_to_replication_stream]
+
+        # make sure that EVAL inside MULTI is propagated in a transaction
+        r config set lua-replicate-commands no
+        r multi
+        r eval {redis.call('set', KEYS[1], 'bar')} 1 bar
+        r exec
+
+        assert_replication_stream $repl {
+            {select *}
+            {multi}
+            {eval *}
+            {exec}
+        }
+        close_replication_stream $repl
+    }
+
+    tags {"stream"} {
+        test {MULTI propagation of XREADGROUP} {
+            # stream is a special case because it calls propagate() directly for XREADGROUP
+            set repl [attach_to_replication_stream]
+
+            r XADD mystream * foo bar
+            r XGROUP CREATE mystream mygroup 0
+
+            # make sure the XCALIM (propagated by XREADGROUP) is indeed inside MULTI/EXEC
+            r multi
+            r XREADGROUP GROUP mygroup consumer1 STREAMS mystream ">"
+            r exec
+
+            assert_replication_stream $repl {
+                {select *}
+                {xadd *}
+                {xgroup CREATE *}
+                {multi}
+                {xclaim *}
+                {exec}
+            }
+            close_replication_stream $repl
+        }
+    }
+
 }

From e7047ec2fcc20e150c0c8cf29addf582638d7e80 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 22 Dec 2020 12:24:20 +0200
Subject: [PATCH 0894/1098] Fix crashes with io-threads-do-reads enabled.
 (#8230)

Normally IO threads should simply read data from the socket into the
buffer and attempt to parse it.

If a protocol error is detected, a reply is generated which may result
with installing a write handler which is not thread safe. This fix
delays that until the client is processed back in the main thread.

Fixes #8220
---
 src/networking.c | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index b81060cbb..45c504e46 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -259,8 +259,14 @@ int prepareClientToWrite(client *c) {
     if (!c->conn) return C_ERR; /* Fake client for AOF loading. */
 
     /* Schedule the client to write the output buffers to the socket, unless
-     * it should already be setup to do so (it has already pending data). */
-    if (!clientHasPendingReplies(c)) clientInstallWriteHandler(c);
+     * it should already be setup to do so (it has already pending data).
+     *
+     * If CLIENT_PENDING_READ is set, we're in an IO thread and should
+     * not install a write handler. Instead, it will be done by
+     * handleClientsWithPendingReadsUsingThreads() upon return.
+     */
+    if (!clientHasPendingReplies(c) && !(c->flags & CLIENT_PENDING_READ))
+            clientInstallWriteHandler(c);
 
     /* Authorize the caller to queue in the output buffer of this client. */
     return C_OK;
@@ -3509,6 +3515,12 @@ int handleClientsWithPendingReadsUsingThreads(void) {
             }
         }
         processInputBuffer(c);
+
+        /* We may have pending replies if a thread readQueryFromClient() produced
+         * replies and did not install a write handler (it can't).
+         */
+        if (!(c->flags & CLIENT_PENDING_WRITE) && clientHasPendingReplies(c))
+            clientInstallWriteHandler(c);
     }
 
     /* Update processed count on server */

From 92a483bca2df734aff5caada6c23409ed6256773 Mon Sep 17 00:00:00 2001
From: "Meir Shpilraien (Spielrein)" <meir@redislabs.com>
Date: Tue, 22 Dec 2020 15:17:39 +0200
Subject: [PATCH 0895/1098] Fix issue where fork process deletes the parent
 pidfile (#8231)

Turns out that when the fork child crashes, the crash log was deleting
the pidfile from the disk (although the parent is still running.

Now we set the pidfile of the fork process to NULL so the fork process
will never deletes it.
---
 src/debug.c  | 2 +-
 src/server.c | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/debug.c b/src/debug.c
index 6ce8b3bdc..26c1a91fc 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1808,7 +1808,7 @@ void bugReportEnd(int killViaSignal, int sig) {
 );
 
     /* free(messages); Don't call free() with possibly corrupted memory. */
-    if (server.daemonize && server.supervised == 0) unlink(server.pidfile);
+    if (server.daemonize && server.supervised == 0 && server.pidfile) unlink(server.pidfile);
 
     if (!killViaSignal) {
         if (server.use_exit_on_panic)
diff --git a/src/server.c b/src/server.c
index 29c810928..0be8fb003 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5200,6 +5200,10 @@ void closeClildUnusedResourceAfterFork() {
     closeListeningSockets(0);
     if (server.cluster_enabled && server.cluster_config_file_lock_fd != -1)
         close(server.cluster_config_file_lock_fd);  /* don't care if this fails */
+
+    /* Clear server.pidfile, this is the parent pidfile which should not
+     * be touched (or deleted) by the child (on exit / crash) */
+    server.pidfile = NULL;
 }
 
 /* purpose is one of CHILD_TYPE_ types */

From 781d7b0d9b51d1d80097c1f6ddec6270c1532ede Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Tue, 22 Dec 2020 09:14:15 -0500
Subject: [PATCH 0896/1098] Sentinel: add missing calls for sentinelflushconfig
 when config master at runtime (#8229)

---
 src/sentinel.c | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/sentinel.c b/src/sentinel.c
index 95cfa84ad..7693ac5ae 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -3579,14 +3579,13 @@ void sentinelSetCommand(client *c) {
                     "Reconfiguration of scripts path is denied for "
                     "security reasons. Check the deny-scripts-reconfig "
                     "configuration directive in your Sentinel configuration");
-                return;
+                goto seterr;
             }
 
             if (strlen(value) && access(value,X_OK) == -1) {
                 addReplyError(c,
                     "Notification script seems non existing or non executable");
-                if (changes) sentinelFlushConfig();
-                return;
+                goto seterr;
             }
             sdsfree(ri->notification_script);
             ri->notification_script = strlen(value) ? sdsnew(value) : NULL;
@@ -3599,15 +3598,14 @@ void sentinelSetCommand(client *c) {
                     "Reconfiguration of scripts path is denied for "
                     "security reasons. Check the deny-scripts-reconfig "
                     "configuration directive in your Sentinel configuration");
-                return;
+                goto seterr;
             }
 
             if (strlen(value) && access(value,X_OK) == -1) {
                 addReplyError(c,
                     "Client reconfiguration script seems non existing or "
                     "non executable");
-                if (changes) sentinelFlushConfig();
-                return;
+                goto seterr;
             }
             sdsfree(ri->client_reconfig_script);
             ri->client_reconfig_script = strlen(value) ? sdsnew(value) : NULL;
@@ -3657,8 +3655,7 @@ void sentinelSetCommand(client *c) {
         } else {
             addReplyErrorFormat(c,"Unknown option or number of arguments for "
                                   "SENTINEL SET '%s'", option);
-            if (changes) sentinelFlushConfig();
-            return;
+            goto seterr;
         }
 
         /* Log the event. */
@@ -3684,9 +3681,11 @@ void sentinelSetCommand(client *c) {
     return;
 
 badfmt: /* Bad format errors */
-    if (changes) sentinelFlushConfig();
     addReplyErrorFormat(c,"Invalid argument '%s' for SENTINEL SET '%s'",
         (char*)c->argv[badarg]->ptr,option);
+seterr:
+    if (changes) sentinelFlushConfig();
+    return;
 }
 
 /* Our fake PUBLISH command: it is actually useful only to receive hello messages

From b51f5da31446a56f6e83e021f4f44fb472999c7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20B=C3=BCnemann?= <Felix.Buenemann@gmail.com>
Date: Wed, 23 Dec 2020 08:46:23 +0100
Subject: [PATCH 0897/1098] Fix TLS build on macOS arm64 systems (#8197)

Homebrew for darwin-arm64 uses /opt/homebrew instead of /usr/local as
the prefix, so that it can coexist with darwin-x86_64 using Rosetta 2.
---
 src/Makefile | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/Makefile b/src/Makefile
index 32cedbac5..068dbc9da 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -115,8 +115,18 @@ else
 ifeq ($(uname_S),Darwin)
 	# Darwin
 	FINAL_LIBS+= -ldl
+	# Homebrew's OpenSSL is not linked to /usr/local to avoid
+	# conflicts with the system's LibreSSL installation so it
+	# must be referenced explicitly during build.
+ifeq ($(uname_M),arm64)
+	# Homebrew arm64 uses /opt/homebrew as HOMEBREW_PREFIX
+	OPENSSL_CFLAGS=-I/opt/homebrew/opt/openssl/include
+	OPENSSL_LDFLAGS=-L/opt/homebrew/opt/openssl/lib
+else
+	# Homebrew x86/ppc uses /usr/local as HOMEBREW_PREFIX
 	OPENSSL_CFLAGS=-I/usr/local/opt/openssl/include
 	OPENSSL_LDFLAGS=-L/usr/local/opt/openssl/lib
+endif
 else
 ifeq ($(uname_S),AIX)
         # AIX

From 2426aaa099e5dfee29cce17af39298d0ce14cc2a Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 23 Dec 2020 12:55:05 +0200
Subject: [PATCH 0898/1098] fix valgrind warning created by recent pidfile fix
 (#8235)

This isn't a leak, just an warning due to unreachable
allocation on the fork child.
Problem created by 92a483b
---
 src/server.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/server.c b/src/server.c
index 0be8fb003..4e9bb88eb 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5203,6 +5203,7 @@ void closeClildUnusedResourceAfterFork() {
 
     /* Clear server.pidfile, this is the parent pidfile which should not
      * be touched (or deleted) by the child (on exit / crash) */
+    zfree(server.pidfile);
     server.pidfile = NULL;
 }
 

From 266949c7fcfab9d10f81314fd7480a00638ced80 Mon Sep 17 00:00:00 2001
From: Greg Femec <gfemec@users.noreply.github.com>
Date: Wed, 23 Dec 2020 05:52:07 -0800
Subject: [PATCH 0899/1098] Fix random element selection for large hash tables.
 (#8133)

When a database on a 64 bit build grows past 2^31 keys, the underlying hash table expands to 2^32 buckets. After this point, the algorithms for selecting random elements only return elements from half of the available buckets because they use random() which has a range of 0 to 2^31 - 1. This causes problems for eviction policies which use dictGetSomeKeys or dictGetRandomKey. Over time they cause the hash table to become unbalanced because, while new keys are spread out evenly across all buckets, evictions come from only half of the available buckets. Eventually this half of the table starts to run out of keys and it takes longer and longer to find candidates for eviction. This continues until no more evictions can happen.

This solution addresses this by using a 64 bit PRNG instead of libc random().

Co-authored-by: Greg Femec <gfemec@google.com>
---
 src/Makefile          |   8 +-
 src/dict.c            |  15 +++-
 src/dict.h            |  14 +++-
 src/mt19937-64.c      | 187 ++++++++++++++++++++++++++++++++++++++++++
 src/mt19937-64.h      |  87 ++++++++++++++++++++
 src/redis-benchmark.c |   2 +
 src/redis-check-rdb.c |   9 ++
 src/redis-cli.c       |   5 ++
 src/server.c          |   2 +
 9 files changed, 319 insertions(+), 10 deletions(-)
 create mode 100644 src/mt19937-64.c
 create mode 100644 src/mt19937-64.h

diff --git a/src/Makefile b/src/Makefile
index 068dbc9da..3bc9f11c0 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -270,11 +270,11 @@ endif
 
 REDIS_SERVER_NAME=redis-server$(PROG_SUFFIX)
 REDIS_SENTINEL_NAME=redis-sentinel$(PROG_SUFFIX)
-REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o
+REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o
 REDIS_CLI_NAME=redis-cli$(PROG_SUFFIX)
-REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o
+REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o
 REDIS_BENCHMARK_NAME=redis-benchmark$(PROG_SUFFIX)
-REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o release.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o
+REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o release.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o
 REDIS_CHECK_RDB_NAME=redis-check-rdb$(PROG_SUFFIX)
 REDIS_CHECK_AOF_NAME=redis-check-aof$(PROG_SUFFIX)
 
@@ -346,7 +346,7 @@ $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ)
 $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ)
 	$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/hdr_histogram/hdr_histogram.o $(FINAL_LIBS)
 
-dict-benchmark: dict.c zmalloc.c sds.c siphash.c
+dict-benchmark: dict.c zmalloc.c sds.c siphash.c mt19937-64.c
 	$(REDIS_CC) $(FINAL_CFLAGS) $^ -D DICT_BENCHMARK_MAIN -o $@ $(FINAL_LIBS)
 
 DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d)
diff --git a/src/dict.c b/src/dict.c
index 4f0916a27..6c203b850 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -646,13 +646,13 @@ dictEntry *dictGetRandomKey(dict *d)
         do {
             /* We are sure there are no elements in indexes from 0
              * to rehashidx-1 */
-            h = d->rehashidx + (random() % (dictSlots(d) - d->rehashidx));
+            h = d->rehashidx + (randomULong() % (dictSlots(d) - d->rehashidx));
             he = (h >= d->ht[0].size) ? d->ht[1].table[h - d->ht[0].size] :
                                       d->ht[0].table[h];
         } while(he == NULL);
     } else {
         do {
-            h = random() & d->ht[0].sizemask;
+            h = randomULong() & d->ht[0].sizemask;
             he = d->ht[0].table[h];
         } while(he == NULL);
     }
@@ -718,7 +718,7 @@ unsigned int dictGetSomeKeys(dict *d, dictEntry **des, unsigned int count) {
         maxsizemask = d->ht[1].sizemask;
 
     /* Pick a random point inside the larger table. */
-    unsigned long i = random() & maxsizemask;
+    unsigned long i = randomULong() & maxsizemask;
     unsigned long emptylen = 0; /* Continuous empty entries so far. */
     while(stored < count && maxsteps--) {
         for (j = 0; j < tables; j++) {
@@ -743,7 +743,7 @@ unsigned int dictGetSomeKeys(dict *d, dictEntry **des, unsigned int count) {
             if (he == NULL) {
                 emptylen++;
                 if (emptylen >= 5 && emptylen > count) {
-                    i = random() & maxsizemask;
+                    i = randomULong() & maxsizemask;
                     emptylen = 0;
                 }
             } else {
@@ -1270,6 +1270,13 @@ int main(int argc, char **argv) {
     }
     end_benchmark("Random access of existing elements");
 
+    start_benchmark();
+    for (j = 0; j < count; j++) {
+        dictEntry *de = dictGetRandomKey(dict);
+        assert(de != NULL);
+    }
+    end_benchmark("Accessing random keys");
+
     start_benchmark();
     for (j = 0; j < count; j++) {
         sds key = sdsfromlonglong(rand() % count);
diff --git a/src/dict.h b/src/dict.h
index f7515e905..d96c3148f 100644
--- a/src/dict.h
+++ b/src/dict.h
@@ -33,11 +33,14 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include <stdint.h>
-
 #ifndef __DICT_H
 #define __DICT_H
 
+#include "mt19937-64.h"
+#include <limits.h>
+#include <stdint.h>
+#include <stdlib.h>
+
 #define DICT_OK 0
 #define DICT_ERR 1
 
@@ -148,6 +151,13 @@ typedef void (dictScanBucketFunction)(void *privdata, dictEntry **bucketref);
 #define dictSize(d) ((d)->ht[0].used+(d)->ht[1].used)
 #define dictIsRehashing(d) ((d)->rehashidx != -1)
 
+/* If our unsigned long type can store a 64 bit number, use a 64 bit PRNG. */
+#if ULONG_MAX >= 0xffffffffffffffff
+#define randomULong() ((unsigned long) genrand64_int64())
+#else
+#define randomULong() random()
+#endif
+
 /* API */
 dict *dictCreate(dictType *type, void *privDataPtr);
 int dictExpand(dict *d, unsigned long size);
diff --git a/src/mt19937-64.c b/src/mt19937-64.c
new file mode 100644
index 000000000..a0c897ff6
--- /dev/null
+++ b/src/mt19937-64.c
@@ -0,0 +1,187 @@
+/*
+   A C-program for MT19937-64 (2004/9/29 version).
+   Coded by Takuji Nishimura and Makoto Matsumoto.
+
+   This is a 64-bit version of Mersenne Twister pseudorandom number
+   generator.
+
+   Before using, initialize the state by using init_genrand64(seed)
+   or init_by_array64(init_key, key_length).
+
+   Copyright (C) 2004, Makoto Matsumoto and Takuji Nishimura,
+   All rights reserved.
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+     1. Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+
+     2. Redistributions in binary form must reproduce the above copyright
+        notice, this list of conditions and the following disclaimer in the
+        documentation and/or other materials provided with the distribution.
+
+     3. The names of its contributors may not be used to endorse or promote
+        products derived from this software without specific prior written
+        permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   References:
+   T. Nishimura, ``Tables of 64-bit Mersenne Twisters''
+     ACM Transactions on Modeling and
+     Computer Simulation 10. (2000) 348--357.
+   M. Matsumoto and T. Nishimura,
+     ``Mersenne Twister: a 623-dimensionally equidistributed
+       uniform pseudorandom number generator''
+     ACM Transactions on Modeling and
+     Computer Simulation 8. (Jan. 1998) 3--30.
+
+   Any feedback is very welcome.
+   http://www.math.hiroshima-u.ac.jp/~m-mat/MT/emt.html
+   email: m-mat @ math.sci.hiroshima-u.ac.jp (remove spaces)
+*/
+
+
+#include "mt19937-64.h"
+#include <stdio.h>
+
+#define NN 312
+#define MM 156
+#define MATRIX_A 0xB5026F5AA96619E9ULL
+#define UM 0xFFFFFFFF80000000ULL /* Most significant 33 bits */
+#define LM 0x7FFFFFFFULL /* Least significant 31 bits */
+
+
+/* The array for the state vector */
+static unsigned long long mt[NN];
+/* mti==NN+1 means mt[NN] is not initialized */
+static int mti=NN+1;
+
+/* initializes mt[NN] with a seed */
+void init_genrand64(unsigned long long seed)
+{
+    mt[0] = seed;
+    for (mti=1; mti<NN; mti++)
+        mt[mti] =  (6364136223846793005ULL * (mt[mti-1] ^ (mt[mti-1] >> 62)) + mti);
+}
+
+/* initialize by an array with array-length */
+/* init_key is the array for initializing keys */
+/* key_length is its length */
+void init_by_array64(unsigned long long init_key[],
+                     unsigned long long key_length)
+{
+    unsigned long long i, j, k;
+    init_genrand64(19650218ULL);
+    i=1; j=0;
+    k = (NN>key_length ? NN : key_length);
+    for (; k; k--) {
+        mt[i] = (mt[i] ^ ((mt[i-1] ^ (mt[i-1] >> 62)) * 3935559000370003845ULL))
+          + init_key[j] + j; /* non linear */
+        i++; j++;
+        if (i>=NN) { mt[0] = mt[NN-1]; i=1; }
+        if (j>=key_length) j=0;
+    }
+    for (k=NN-1; k; k--) {
+        mt[i] = (mt[i] ^ ((mt[i-1] ^ (mt[i-1] >> 62)) * 2862933555777941757ULL))
+          - i; /* non linear */
+        i++;
+        if (i>=NN) { mt[0] = mt[NN-1]; i=1; }
+    }
+
+    mt[0] = 1ULL << 63; /* MSB is 1; assuring non-zero initial array */
+}
+
+/* generates a random number on [0, 2^64-1]-interval */
+unsigned long long genrand64_int64(void)
+{
+    int i;
+    unsigned long long x;
+    static unsigned long long mag01[2]={0ULL, MATRIX_A};
+
+    if (mti >= NN) { /* generate NN words at one time */
+
+        /* if init_genrand64() has not been called, */
+        /* a default initial seed is used     */
+        if (mti == NN+1)
+            init_genrand64(5489ULL);
+
+        for (i=0;i<NN-MM;i++) {
+            x = (mt[i]&UM)|(mt[i+1]&LM);
+            mt[i] = mt[i+MM] ^ (x>>1) ^ mag01[(int)(x&1ULL)];
+        }
+        for (;i<NN-1;i++) {
+            x = (mt[i]&UM)|(mt[i+1]&LM);
+            mt[i] = mt[i+(MM-NN)] ^ (x>>1) ^ mag01[(int)(x&1ULL)];
+        }
+        x = (mt[NN-1]&UM)|(mt[0]&LM);
+        mt[NN-1] = mt[MM-1] ^ (x>>1) ^ mag01[(int)(x&1ULL)];
+
+        mti = 0;
+    }
+
+    x = mt[mti++];
+
+    x ^= (x >> 29) & 0x5555555555555555ULL;
+    x ^= (x << 17) & 0x71D67FFFEDA60000ULL;
+    x ^= (x << 37) & 0xFFF7EEE000000000ULL;
+    x ^= (x >> 43);
+
+    return x;
+}
+
+/* generates a random number on [0, 2^63-1]-interval */
+long long genrand64_int63(void)
+{
+    return (long long)(genrand64_int64() >> 1);
+}
+
+/* generates a random number on [0,1]-real-interval */
+double genrand64_real1(void)
+{
+    return (genrand64_int64() >> 11) * (1.0/9007199254740991.0);
+}
+
+/* generates a random number on [0,1)-real-interval */
+double genrand64_real2(void)
+{
+    return (genrand64_int64() >> 11) * (1.0/9007199254740992.0);
+}
+
+/* generates a random number on (0,1)-real-interval */
+double genrand64_real3(void)
+{
+    return ((genrand64_int64() >> 12) + 0.5) * (1.0/4503599627370496.0);
+}
+
+#ifdef MT19937_64_MAIN
+int main(void)
+{
+    int i;
+    unsigned long long init[4]={0x12345ULL, 0x23456ULL, 0x34567ULL, 0x45678ULL}, length=4;
+    init_by_array64(init, length);
+    printf("1000 outputs of genrand64_int64()\n");
+    for (i=0; i<1000; i++) {
+      printf("%20llu ", genrand64_int64());
+      if (i%5==4) printf("\n");
+    }
+    printf("\n1000 outputs of genrand64_real2()\n");
+    for (i=0; i<1000; i++) {
+      printf("%10.8f ", genrand64_real2());
+      if (i%5==4) printf("\n");
+    }
+    return 0;
+}
+#endif
diff --git a/src/mt19937-64.h b/src/mt19937-64.h
new file mode 100644
index 000000000..b98348fd4
--- /dev/null
+++ b/src/mt19937-64.h
@@ -0,0 +1,87 @@
+/*
+   A C-program for MT19937-64 (2004/9/29 version).
+   Coded by Takuji Nishimura and Makoto Matsumoto.
+
+   This is a 64-bit version of Mersenne Twister pseudorandom number
+   generator.
+
+   Before using, initialize the state by using init_genrand64(seed)
+   or init_by_array64(init_key, key_length).
+
+   Copyright (C) 2004, Makoto Matsumoto and Takuji Nishimura,
+   All rights reserved.
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+     1. Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+
+     2. Redistributions in binary form must reproduce the above copyright
+        notice, this list of conditions and the following disclaimer in the
+        documentation and/or other materials provided with the distribution.
+
+     3. The names of its contributors may not be used to endorse or promote
+        products derived from this software without specific prior written
+        permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   References:
+   T. Nishimura, ``Tables of 64-bit Mersenne Twisters''
+     ACM Transactions on Modeling and
+     Computer Simulation 10. (2000) 348--357.
+   M. Matsumoto and T. Nishimura,
+     ``Mersenne Twister: a 623-dimensionally equidistributed
+       uniform pseudorandom number generator''
+     ACM Transactions on Modeling and
+     Computer Simulation 8. (Jan. 1998) 3--30.
+
+   Any feedback is very welcome.
+   http://www.math.hiroshima-u.ac.jp/~m-mat/MT/emt.html
+   email: m-mat @ math.sci.hiroshima-u.ac.jp (remove spaces)
+*/
+
+#ifndef __MT19937_64_H
+#define __MT19937_64_H
+
+/* initializes mt[NN] with a seed */
+void init_genrand64(unsigned long long seed);
+
+/* initialize by an array with array-length */
+/* init_key is the array for initializing keys */
+/* key_length is its length */
+void init_by_array64(unsigned long long init_key[],
+                     unsigned long long key_length);
+
+/* generates a random number on [0, 2^64-1]-interval */
+unsigned long long genrand64_int64(void);
+
+
+/* generates a random number on [0, 2^63-1]-interval */
+long long genrand64_int63(void);
+
+/* generates a random number on [0,1]-real-interval */
+double genrand64_real1(void);
+
+/* generates a random number on [0,1)-real-interval */
+double genrand64_real2(void);
+
+/* generates a random number on (0,1)-real-interval */
+double genrand64_real3(void);
+
+/* generates a random number on (0,1]-real-interval */
+double genrand64_real4(void);
+
+#endif
diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 18e19b0e0..a955c0d4c 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -59,6 +59,7 @@
 #include "crc16_slottable.h"
 #include "hdr_histogram.h"
 #include "cli_common.h"
+#include "mt19937-64.h"
 
 #define UNUSED(V) ((void) V)
 #define RANDPTR_INITIAL_SIZE 8
@@ -1677,6 +1678,7 @@ int main(int argc, const char **argv) {
     client c;
 
     srandom(time(NULL) ^ getpid());
+    init_genrand64(ustime() ^ getpid());
     signal(SIGHUP, SIG_IGN);
     signal(SIGPIPE, SIG_IGN);
 
diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c
index 79dbf3229..335e35189 100644
--- a/src/redis-check-rdb.c
+++ b/src/redis-check-rdb.c
@@ -27,10 +27,13 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "mt19937-64.h"
 #include "server.h"
 #include "rdb.h"
 
 #include <stdarg.h>
+#include <sys/time.h>
+#include <unistd.h>
 
 void createSharedObjects(void);
 void rdbLoadProgressCallback(rio *r, const void *buf, size_t len);
@@ -362,10 +365,16 @@ err:
  * Otherwise if called with a non NULL fp, the function returns C_OK or
  * C_ERR depending on the success or failure. */
 int redis_check_rdb_main(int argc, char **argv, FILE *fp) {
+    struct timeval tv;
+
     if (argc != 2 && fp == NULL) {
         fprintf(stderr, "Usage: %s <rdb-file-name>\n", argv[0]);
         exit(1);
     }
+
+    gettimeofday(&tv, NULL);
+    init_genrand64(((long long) tv.tv_sec * 1000000 + tv.tv_usec) ^ getpid());
+
     /* In order to call the loading functions we need to create the shared
      * integer objects, however since this function may be called from
      * an already initialized Redis instance, check if we really need to. */
diff --git a/src/redis-cli.c b/src/redis-cli.c
index 5310a1b21..ff29de748 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -62,6 +62,7 @@
 #include "anet.h"
 #include "ae.h"
 #include "cli_common.h"
+#include "mt19937-64.h"
 
 #define UNUSED(V) ((void) V)
 
@@ -8123,6 +8124,7 @@ static sds askPassword(const char *msg) {
 
 int main(int argc, char **argv) {
     int firstarg;
+    struct timeval tv;
 
     config.hostip = sdsnew("127.0.0.1");
     config.hostport = 6379;
@@ -8219,6 +8221,9 @@ int main(int argc, char **argv) {
     }
 #endif
 
+    gettimeofday(&tv, NULL);
+    init_genrand64(((long long) tv.tv_sec * 1000000 + tv.tv_usec) ^ getpid());
+
     /* Cluster Manager mode */
     if (CLUSTER_MANAGER_MODE()) {
         clusterManagerCommandProc *proc = validateClusterManagerCommand();
diff --git a/src/server.c b/src/server.c
index 4e9bb88eb..611935a6c 100644
--- a/src/server.c
+++ b/src/server.c
@@ -34,6 +34,7 @@
 #include "bio.h"
 #include "latency.h"
 #include "atomicvar.h"
+#include "mt19937-64.h"
 
 #include <time.h>
 #include <signal.h>
@@ -5452,6 +5453,7 @@ int main(int argc, char **argv) {
     srand(time(NULL)^getpid());
     srandom(time(NULL)^getpid());
     gettimeofday(&tv,NULL);
+    init_genrand64(((long long) tv.tv_sec * 1000000 + tv.tv_usec) ^ getpid());
     crc64_init();
 
     uint8_t hashseed[16];

From ee59dc1b5cdcc631438aba3904d6d3b520354c80 Mon Sep 17 00:00:00 2001
From: Yang Bodong <bodong.ybd@alibaba-inc.com>
Date: Wed, 23 Dec 2020 22:28:17 +0800
Subject: [PATCH 0900/1098] Tests: fix the problem that Darwin memory leak
 detection may fail (#8213)

Apparently the "leaks" took reports a different error string about process
that's not found in each version of MacOS.
This cause the test suite to fail on some OS versions, since some tests terminate
the process before looking for leaks.
Instead of looking at the error string, we now look at the (documented) exit code.
---
 tests/support/server.tcl | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index 1cddb7068..77ba31d84 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -50,11 +50,17 @@ proc kill_server config {
                 tags {"leaks"} {
                     test "Check for memory leaks (pid $pid)" {
                         set output {0 leaks}
-                        catch {exec leaks $pid} output
-                        if {[string match {*process does not exist*} $output] ||
-                            [string match {*cannot examine*} $output]} {
-                            # In a few tests we kill the server process.
-                            set output "0 leaks"
+                        catch {exec leaks $pid} output option
+                        # In a few tests we kill the server process, so leaks will not find it.
+                        # It'll exits with exit code >1 on error, so we ignore these.
+                        if {[dict exists $option -errorcode]} {
+                            set details [dict get $option -errorcode]
+                            if {[lindex $details 0] eq "CHILDSTATUS"} {
+                                  set status [lindex $details 2]
+                                  if {$status > 1} {
+                                      set output "0 leaks"
+                                  }
+                            }
                         }
                         set output
                     } {*0 leaks*}

From 58e9c261152819bfa5fe8d37ea943340c3575421 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Thu, 24 Dec 2020 00:37:33 +0800
Subject: [PATCH 0901/1098] Fix redundancy incrRefCount in lmoveGenericCommand
 (#8218)

---
 src/t_list.c | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/t_list.c b/src/t_list.c
index 106f960f6..1b4062c38 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -729,10 +729,6 @@ void lmoveGenericCommand(client *c, int wherefrom, int whereto) {
         if (checkType(c,dobj,OBJ_LIST)) return;
         value = listTypePop(sobj,wherefrom);
         serverAssert(value); /* assertion for valgrind (avoid NPD) */
-        /* We saved touched key, and protect it, since lmoveHandlePush
-         * may change the client command argument vector (it does not
-         * currently). */
-        incrRefCount(touchedkey);
         lmoveHandlePush(c,c->argv[2],dobj,value,whereto);
 
         /* listTypePop returns an object with its refcount incremented */
@@ -749,7 +745,6 @@ void lmoveGenericCommand(client *c, int wherefrom, int whereto) {
                                 touchedkey,c->db->id);
         }
         signalModifiedKey(c,c->db,touchedkey);
-        decrRefCount(touchedkey);
         server.dirty++;
         if (c->cmd->proc == blmoveCommand) {
             rewriteClientCommandVector(c,5,shared.lmove,

From 55abd1c6d078e0d30ee01a819f069fa7737c3dc2 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuan21@baidu.com>
Date: Thu, 24 Dec 2020 01:16:49 +0800
Subject: [PATCH 0902/1098] Add semicolon to calls of test_cond() (#8238)

---
 src/sds.c | 48 ++++++++++++++++++++++++------------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/src/sds.c b/src/sds.c
index 6a85eb4aa..5efb36de6 100644
--- a/src/sds.c
+++ b/src/sds.c
@@ -1159,8 +1159,8 @@ void sds_free(void *ptr) { s_free(ptr); }
 
 #ifdef REDIS_TEST
 #include <stdio.h>
+#include <limits.h>
 #include "testhelp.h"
-#include "limits.h"
 
 #define UNUSED(x) (void)(x)
 int sdsTest(int argc, char **argv) {
@@ -1171,12 +1171,12 @@ int sdsTest(int argc, char **argv) {
         sds x = sdsnew("foo"), y;
 
         test_cond("Create a string and obtain the length",
-            sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0)
+            sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0);
 
         sdsfree(x);
         x = sdsnewlen("foo",2);
         test_cond("Create a string with specified length",
-            sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0)
+            sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0);
 
         x = sdscat(x,"bar");
         test_cond("Strings concatenation",
@@ -1184,22 +1184,22 @@ int sdsTest(int argc, char **argv) {
 
         x = sdscpy(x,"a");
         test_cond("sdscpy() against an originally longer string",
-            sdslen(x) == 1 && memcmp(x,"a\0",2) == 0)
+            sdslen(x) == 1 && memcmp(x,"a\0",2) == 0);
 
         x = sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk");
         test_cond("sdscpy() against an originally shorter string",
             sdslen(x) == 33 &&
-            memcmp(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk\0",33) == 0)
+            memcmp(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk\0",33) == 0);
 
         sdsfree(x);
         x = sdscatprintf(sdsempty(),"%d",123);
         test_cond("sdscatprintf() seems working in the base case",
-            sdslen(x) == 3 && memcmp(x,"123\0",4) == 0)
+            sdslen(x) == 3 && memcmp(x,"123\0",4) == 0);
 
         sdsfree(x);
         x = sdscatprintf(sdsempty(),"a%cb",0);
         test_cond("sdscatprintf() seems working with \\0 inside of result",
-            sdslen(x) == 3 && memcmp(x,"a\0""b\0",4) == 0)
+            sdslen(x) == 3 && memcmp(x,"a\0""b\0",4) == 0);
 
         {
             sdsfree(x);
@@ -1209,7 +1209,7 @@ int sdsTest(int argc, char **argv) {
             }
             x = sdscatprintf(sdsempty(),"%0*d",(int)sizeof(etalon),0);
             test_cond("sdscatprintf() can print 1MB",
-                sdslen(x) == sizeof(etalon) && memcmp(x,etalon,sizeof(etalon)) == 0)
+                sdslen(x) == sizeof(etalon) && memcmp(x,etalon,sizeof(etalon)) == 0);
         }
 
         sdsfree(x);
@@ -1218,7 +1218,7 @@ int sdsTest(int argc, char **argv) {
         test_cond("sdscatfmt() seems working in the base case",
             sdslen(x) == 60 &&
             memcmp(x,"--Hello Hi! World -9223372036854775808,"
-                     "9223372036854775807--",60) == 0)
+                     "9223372036854775807--",60) == 0);
         printf("[%s]\n",x);
 
         sdsfree(x);
@@ -1226,85 +1226,85 @@ int sdsTest(int argc, char **argv) {
         x = sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX);
         test_cond("sdscatfmt() seems working with unsigned numbers",
             sdslen(x) == 35 &&
-            memcmp(x,"--4294967295,18446744073709551615--",35) == 0)
+            memcmp(x,"--4294967295,18446744073709551615--",35) == 0);
 
         sdsfree(x);
         x = sdsnew(" x ");
         sdstrim(x," x");
         test_cond("sdstrim() works when all chars match",
-            sdslen(x) == 0)
+            sdslen(x) == 0);
 
         sdsfree(x);
         x = sdsnew(" x ");
         sdstrim(x," ");
         test_cond("sdstrim() works when a single char remains",
-            sdslen(x) == 1 && x[0] == 'x')
+            sdslen(x) == 1 && x[0] == 'x');
 
         sdsfree(x);
         x = sdsnew("xxciaoyyy");
         sdstrim(x,"xy");
         test_cond("sdstrim() correctly trims characters",
-            sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0)
+            sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0);
 
         y = sdsdup(x);
         sdsrange(y,1,1);
         test_cond("sdsrange(...,1,1)",
-            sdslen(y) == 1 && memcmp(y,"i\0",2) == 0)
+            sdslen(y) == 1 && memcmp(y,"i\0",2) == 0);
 
         sdsfree(y);
         y = sdsdup(x);
         sdsrange(y,1,-1);
         test_cond("sdsrange(...,1,-1)",
-            sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
+            sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0);
 
         sdsfree(y);
         y = sdsdup(x);
         sdsrange(y,-2,-1);
         test_cond("sdsrange(...,-2,-1)",
-            sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0)
+            sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0);
 
         sdsfree(y);
         y = sdsdup(x);
         sdsrange(y,2,1);
         test_cond("sdsrange(...,2,1)",
-            sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
+            sdslen(y) == 0 && memcmp(y,"\0",1) == 0);
 
         sdsfree(y);
         y = sdsdup(x);
         sdsrange(y,1,100);
         test_cond("sdsrange(...,1,100)",
-            sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
+            sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0);
 
         sdsfree(y);
         y = sdsdup(x);
         sdsrange(y,100,100);
         test_cond("sdsrange(...,100,100)",
-            sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
+            sdslen(y) == 0 && memcmp(y,"\0",1) == 0);
 
         sdsfree(y);
         sdsfree(x);
         x = sdsnew("foo");
         y = sdsnew("foa");
-        test_cond("sdscmp(foo,foa)", sdscmp(x,y) > 0)
+        test_cond("sdscmp(foo,foa)", sdscmp(x,y) > 0);
 
         sdsfree(y);
         sdsfree(x);
         x = sdsnew("bar");
         y = sdsnew("bar");
-        test_cond("sdscmp(bar,bar)", sdscmp(x,y) == 0)
+        test_cond("sdscmp(bar,bar)", sdscmp(x,y) == 0);
 
         sdsfree(y);
         sdsfree(x);
         x = sdsnew("aar");
         y = sdsnew("bar");
-        test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0)
+        test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0);
 
         sdsfree(y);
         sdsfree(x);
         x = sdsnewlen("\a\n\0foo\r",7);
         y = sdscatrepr(sdsempty(),x,sdslen(x));
         test_cond("sdscatrepr(...data...)",
-            memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0)
+            memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0);
 
         {
             unsigned int oldfree;
@@ -1343,7 +1343,7 @@ int sdsTest(int argc, char **argv) {
             sdsfree(x);
         }
     }
-    test_report()
+    test_report();
     return 0;
 }
 #endif

From efaf09ee4b6437c69c467acdb0c62a510207e993 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Wed, 23 Dec 2020 19:06:25 -0800
Subject: [PATCH 0903/1098] Flow through the error handling path for most
 errors (#8226)

Properly throw errors for invalid replication stream and support https://github.com/redis/redis/pull/8217
---
 src/acl.c         |  2 +-
 src/bitops.c      |  8 ++++----
 src/blocked.c     |  4 ++--
 src/cluster.c     | 31 +++++++++++++++----------------
 src/db.c          | 14 +++++++-------
 src/debug.c       |  8 ++++----
 src/geo.c         |  6 +++---
 src/hyperloglog.c | 21 ++++++++++-----------
 src/networking.c  | 16 ++++++++--------
 src/object.c      |  6 +++---
 src/rdb.c         |  6 +++---
 src/replication.c |  4 ++--
 src/scripting.c   |  8 ++++----
 src/sentinel.c    |  2 +-
 src/server.c      | 13 ++++++++-----
 src/sort.c        |  4 ++--
 src/t_list.c      |  8 ++++----
 src/t_set.c       |  4 ++--
 src/t_stream.c    | 15 +++++++--------
 src/t_string.c    |  6 +++---
 src/t_zset.c      | 20 ++++++++++----------
 21 files changed, 103 insertions(+), 103 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index a1a7c4237..606b61cb7 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -2224,7 +2224,7 @@ void addReplyCommandCategories(client *c, struct redisCommand *cmd) {
 void authCommand(client *c) {
     /* Only two or three argument forms are allowed. */
     if (c->argc > 3) {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
 
diff --git a/src/bitops.c b/src/bitops.c
index 5e996679b..afd79ad88 100644
--- a/src/bitops.c
+++ b/src/bitops.c
@@ -611,7 +611,7 @@ void bitopCommand(client *c) {
     else if((opname[0] == 'n' || opname[0] == 'N') && !strcasecmp(opname,"not"))
         op = BITOP_NOT;
     else {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
 
@@ -813,7 +813,7 @@ void bitcountCommand(client *c) {
         end = strlen-1;
     } else {
         /* Syntax error. */
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
 
@@ -878,7 +878,7 @@ void bitposCommand(client *c) {
         end = strlen-1;
     } else {
         /* Syntax error. */
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
 
@@ -970,7 +970,7 @@ void bitfieldGeneric(client *c, int flags) {
             }
             continue;
         } else {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             zfree(ops);
             return;
         }
diff --git a/src/blocked.c b/src/blocked.c
index d85723458..e3f7b74d6 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -200,9 +200,9 @@ void disconnectAllBlockedClients(void) {
         client *c = listNodeValue(ln);
 
         if (c->flags & CLIENT_BLOCKED) {
-            addReplySds(c,sdsnew(
+            addReplyError(c,
                 "-UNBLOCKED force unblock from blocking operation, "
-                "instance state changed (master -> replica?)\r\n"));
+                "instance state changed (master -> replica?)");
             unblockClient(c);
             c->flags |= CLIENT_CLOSE_AFTER_REPLY;
         }
diff --git a/src/cluster.c b/src/cluster.c
index 2cd6b2521..29bdb2109 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4820,7 +4820,7 @@ NULL
                 takeover = 1;
                 force = 1; /* Takeover also implies force. */
             } else {
-                addReply(c,shared.syntaxerr);
+                addReplyErrorObject(c,shared.syntaxerr);
                 return;
             }
         }
@@ -4911,7 +4911,7 @@ NULL
             } else if (!strcasecmp(c->argv[2]->ptr,"soft")) {
                 hard = 0;
             } else {
-                addReply(c,shared.syntaxerr);
+                addReplyErrorObject(c,shared.syntaxerr);
                 return;
             }
         }
@@ -5049,7 +5049,7 @@ void restoreCommand(client *c) {
             }
             j++; /* Consume additional arg. */
         } else {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             return;
         }
     }
@@ -5057,7 +5057,7 @@ void restoreCommand(client *c) {
     /* Make sure this key does not already exist here... */
     robj *key = c->argv[1];
     if (!replace && lookupKeyWrite(c->db,key) != NULL) {
-        addReply(c,shared.busykeyerr);
+        addReplyErrorObject(c,shared.busykeyerr);
         return;
     }
 
@@ -5170,8 +5170,7 @@ migrateCachedSocket* migrateGetSocket(client *c, robj *host, robj *port, long ti
     conn = server.tls_cluster ? connCreateTLS() : connCreateSocket();
     if (connBlockingConnect(conn, c->argv[1]->ptr, atoi(c->argv[2]->ptr), timeout)
             != C_OK) {
-        addReplySds(c,
-            sdsnew("-IOERR error or timeout connecting to the client\r\n"));
+        addReplyError(c,"-IOERR error or timeout connecting to the client");
         connClose(conn);
         sdsfree(name);
         return NULL;
@@ -5259,14 +5258,14 @@ void migrateCommand(client *c) {
             replace = 1;
         } else if (!strcasecmp(c->argv[j]->ptr,"auth")) {
             if (!moreargs) {
-                addReply(c,shared.syntaxerr);
+                addReplyErrorObject(c,shared.syntaxerr);
                 return;
             }
             j++;
             password = c->argv[j]->ptr;
         } else if (!strcasecmp(c->argv[j]->ptr,"auth2")) {
             if (moreargs < 2) {
-                addReply(c,shared.syntaxerr);
+                addReplyErrorObject(c,shared.syntaxerr);
                 return;
             }
             username = c->argv[++j]->ptr;
@@ -5282,7 +5281,7 @@ void migrateCommand(client *c) {
             num_keys = c->argc - j - 1;
             break; /* All the remaining args are keys. */
         } else {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             return;
         }
     }
@@ -5837,23 +5836,23 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
  * be set to the hash slot that caused the redirection. */
 void clusterRedirectClient(client *c, clusterNode *n, int hashslot, int error_code) {
     if (error_code == CLUSTER_REDIR_CROSS_SLOT) {
-        addReplySds(c,sdsnew("-CROSSSLOT Keys in request don't hash to the same slot\r\n"));
+        addReplyError(c,"-CROSSSLOT Keys in request don't hash to the same slot");
     } else if (error_code == CLUSTER_REDIR_UNSTABLE) {
         /* The request spawns multiple keys in the same slot,
          * but the slot is not "stable" currently as there is
          * a migration or import in progress. */
-        addReplySds(c,sdsnew("-TRYAGAIN Multiple keys request during rehashing of slot\r\n"));
+        addReplyError(c,"-TRYAGAIN Multiple keys request during rehashing of slot");
     } else if (error_code == CLUSTER_REDIR_DOWN_STATE) {
-        addReplySds(c,sdsnew("-CLUSTERDOWN The cluster is down\r\n"));
+        addReplyError(c,"-CLUSTERDOWN The cluster is down");
     } else if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) {
-        addReplySds(c,sdsnew("-CLUSTERDOWN The cluster is down and only accepts read commands\r\n"));
+        addReplyError(c,"-CLUSTERDOWN The cluster is down and only accepts read commands");
     } else if (error_code == CLUSTER_REDIR_DOWN_UNBOUND) {
-        addReplySds(c,sdsnew("-CLUSTERDOWN Hash slot not served\r\n"));
+        addReplyError(c,"-CLUSTERDOWN Hash slot not served");
     } else if (error_code == CLUSTER_REDIR_MOVED ||
                error_code == CLUSTER_REDIR_ASK)
     {
-        addReplySds(c,sdscatprintf(sdsempty(),
-            "-%s %d %s:%d\r\n",
+        addReplyErrorSds(c,sdscatprintf(sdsempty(),
+            "-%s %d %s:%d",
             (error_code == CLUSTER_REDIR_ASK) ? "ASK" : "MOVED",
             hashslot,n->ip,n->port));
     } else {
diff --git a/src/db.c b/src/db.c
index 4f27534fc..d5e8b07d3 100644
--- a/src/db.c
+++ b/src/db.c
@@ -593,7 +593,7 @@ int getFlushCommandFlags(client *c, int *flags) {
     /* Parse the optional ASYNC option. */
     if (c->argc > 1) {
         if (c->argc > 2 || strcasecmp(c->argv[1]->ptr,"async")) {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             return C_ERR;
         }
         *flags = EMPTYDB_ASYNC;
@@ -842,7 +842,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) {
             }
 
             if (count < 1) {
-                addReply(c,shared.syntaxerr);
+                addReplyErrorObject(c,shared.syntaxerr);
                 goto cleanup;
             }
 
@@ -861,7 +861,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) {
             typename = c->argv[i+1]->ptr;
             i+= 2;
         } else {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             goto cleanup;
         }
     }
@@ -1050,7 +1050,7 @@ void shutdownCommand(client *c) {
     int flags = 0;
 
     if (c->argc > 2) {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     } else if (c->argc == 2) {
         if (!strcasecmp(c->argv[1]->ptr,"nosave")) {
@@ -1058,7 +1058,7 @@ void shutdownCommand(client *c) {
         } else if (!strcasecmp(c->argv[1]->ptr,"save")) {
             flags |= SHUTDOWN_SAVE;
         } else {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             return;
         }
     }
@@ -1144,7 +1144,7 @@ void moveCommand(client *c) {
     /* If the user is moving using as target the same
      * DB as the source DB it is probably an error. */
     if (src == dst) {
-        addReply(c,shared.sameobjecterr);
+        addReplyErrorObject(c,shared.sameobjecterr);
         return;
     }
 
@@ -1224,7 +1224,7 @@ void copyCommand(client *c) {
     robj *key = c->argv[1];
     robj *newkey = c->argv[2];
     if (src == dst && (sdscmp(key->ptr, newkey->ptr) == 0)) {
-        addReply(c,shared.sameobjecterr);
+        addReplyErrorObject(c,shared.sameobjecterr);
         return;
     }
 
diff --git a/src/debug.c b/src/debug.c
index 26c1a91fc..f95aaba0d 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -474,7 +474,7 @@ NULL
             rdbSaveInfo rsi, *rsiptr;
             rsiptr = rdbPopulateSaveInfo(&rsi);
             if (rdbSave(server.rdb_filename,rsiptr) != C_OK) {
-                addReply(c,shared.err);
+                addReplyErrorObject(c,shared.err);
                 return;
             }
         }
@@ -500,7 +500,7 @@ NULL
         int ret = loadAppendOnlyFile(server.aof_filename);
         unprotectClient(c);
         if (ret != C_OK) {
-            addReply(c,shared.err);
+            addReplyErrorObject(c,shared.err);
             return;
         }
         server.dirty = 0; /* Prevent AOF / replication */
@@ -512,7 +512,7 @@ NULL
         char *strenc;
 
         if ((de = dictFind(c->db->dict,c->argv[2]->ptr)) == NULL) {
-            addReply(c,shared.nokeyerr);
+            addReplyErrorObject(c,shared.nokeyerr);
             return;
         }
         val = dictGetVal(de);
@@ -564,7 +564,7 @@ NULL
         sds key;
 
         if ((de = dictFind(c->db->dict,c->argv[2]->ptr)) == NULL) {
-            addReply(c,shared.nokeyerr);
+            addReplyErrorObject(c,shared.nokeyerr);
             return;
         }
         val = dictGetVal(de);
diff --git a/src/geo.c b/src/geo.c
index 925c090ed..75a88398e 100644
--- a/src/geo.c
+++ b/src/geo.c
@@ -100,8 +100,8 @@ int extractLongLatOrReply(client *c, robj **argv, double *xy) {
     }
     if (xy[0] < GEO_LONG_MIN || xy[0] > GEO_LONG_MAX ||
         xy[1] < GEO_LAT_MIN  || xy[1] > GEO_LAT_MAX) {
-        addReplySds(c, sdscatprintf(sdsempty(),
-            "-ERR invalid longitude,latitude pair %f,%f\r\n",xy[0],xy[1]));
+        addReplyErrorFormat(c,
+            "-ERR invalid longitude,latitude pair %f,%f\r\n",xy[0],xy[1]);
         return C_ERR;
     }
     return C_OK;
@@ -902,7 +902,7 @@ void geodistCommand(client *c) {
         to_meter = extractUnitOrReply(c,c->argv[4]);
         if (to_meter < 0) return;
     } else if (c->argc > 5) {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
 
diff --git a/src/hyperloglog.c b/src/hyperloglog.c
index 648c26a02..75a04227c 100644
--- a/src/hyperloglog.c
+++ b/src/hyperloglog.c
@@ -205,7 +205,7 @@ struct hllhdr {
 #define HLL_RAW 255 /* Only used internally, never exposed. */
 #define HLL_MAX_ENCODING 1
 
-static char *invalid_hll_err = "-INVALIDOBJ Corrupted HLL object detected\r\n";
+static char *invalid_hll_err = "-INVALIDOBJ Corrupted HLL object detected";
 
 /* =========================== Low level bit macros ========================= */
 
@@ -1171,9 +1171,8 @@ int isHLLObjectOrReply(client *c, robj *o) {
     return C_OK;
 
 invalid:
-    addReplySds(c,
-        sdsnew("-WRONGTYPE Key is not a valid "
-               "HyperLogLog string value.\r\n"));
+    addReplyError(c,"-WRONGTYPE Key is not a valid "
+               "HyperLogLog string value.");
     return C_ERR;
 }
 
@@ -1203,7 +1202,7 @@ void pfaddCommand(client *c) {
             updated++;
             break;
         case -1:
-            addReplySds(c,sdsnew(invalid_hll_err));
+            addReplyError(c,invalid_hll_err);
             return;
         }
     }
@@ -1245,7 +1244,7 @@ void pfcountCommand(client *c) {
             /* Merge with this HLL with our 'max' HLL by setting max[i]
              * to MAX(max[i],hll[i]). */
             if (hllMerge(registers,o) == C_ERR) {
-                addReplySds(c,sdsnew(invalid_hll_err));
+                addReplyError(c,invalid_hll_err);
                 return;
             }
         }
@@ -1285,7 +1284,7 @@ void pfcountCommand(client *c) {
             /* Recompute it and update the cached value. */
             card = hllCount(hdr,&invalid);
             if (invalid) {
-                addReplySds(c,sdsnew(invalid_hll_err));
+                addReplyError(c,invalid_hll_err);
                 return;
             }
             hdr->card[0] = card & 0xff;
@@ -1332,7 +1331,7 @@ void pfmergeCommand(client *c) {
         /* Merge with this HLL with our 'max' HLL by setting max[i]
          * to MAX(max[i],hll[i]). */
         if (hllMerge(max,o) == C_ERR) {
-            addReplySds(c,sdsnew(invalid_hll_err));
+            addReplyError(c,invalid_hll_err);
             return;
         }
     }
@@ -1355,7 +1354,7 @@ void pfmergeCommand(client *c) {
     /* Convert the destination object to dense representation if at least
      * one of the inputs was dense. */
     if (use_dense && hllSparseToDense(o) == C_ERR) {
-        addReplySds(c,sdsnew(invalid_hll_err));
+        addReplyError(c,invalid_hll_err);
         return;
     }
 
@@ -1512,7 +1511,7 @@ void pfdebugCommand(client *c) {
 
         if (hdr->encoding == HLL_SPARSE) {
             if (hllSparseToDense(o) == C_ERR) {
-                addReplySds(c,sdsnew(invalid_hll_err));
+                addReplyError(c,invalid_hll_err);
                 return;
             }
             server.dirty++; /* Force propagation on encoding change. */
@@ -1577,7 +1576,7 @@ void pfdebugCommand(client *c) {
 
         if (hdr->encoding == HLL_SPARSE) {
             if (hllSparseToDense(o) == C_ERR) {
-                addReplySds(c,sdsnew(invalid_hll_err));
+                addReplyError(c,invalid_hll_err);
                 return;
             }
             conv = 1;
diff --git a/src/networking.c b/src/networking.c
index 45c504e46..8fee298c6 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2427,7 +2427,7 @@ NULL
                 }
             }
         } else if (c->argc != 2) {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             return;
         }
 
@@ -2446,7 +2446,7 @@ NULL
             if (!(c->flags & CLIENT_REPLY_OFF))
                 c->flags |= CLIENT_REPLY_SKIP_NEXT;
         } else {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             return;
         }
     } else if (!strcasecmp(c->argv[1]->ptr,"kill")) {
@@ -2502,17 +2502,17 @@ NULL
                     } else if (!strcasecmp(c->argv[i+1]->ptr,"no")) {
                         skipme = 0;
                     } else {
-                        addReply(c,shared.syntaxerr);
+                        addReplyErrorObject(c,shared.syntaxerr);
                         return;
                     }
                 } else {
-                    addReply(c,shared.syntaxerr);
+                    addReplyErrorObject(c,shared.syntaxerr);
                     return;
                 }
                 i += 2;
             }
         } else {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             return;
         }
 
@@ -2649,7 +2649,7 @@ NULL
                 prefix[numprefix++] = c->argv[j];
             } else {
                 zfree(prefix);
-                addReply(c,shared.syntaxerr);
+                addReplyErrorObject(c,shared.syntaxerr);
                 return;
             }
         }
@@ -2711,7 +2711,7 @@ NULL
             disableTracking(c);
         } else {
             zfree(prefix);
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             return;
         }
         zfree(prefix);
@@ -2740,7 +2740,7 @@ NULL
                 return;
             }
         } else {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             return;
         }
 
diff --git a/src/object.c b/src/object.c
index 71eceb6d6..583d13737 100644
--- a/src/object.c
+++ b/src/object.c
@@ -404,7 +404,7 @@ robj *resetRefCount(robj *obj) {
 int checkType(client *c, robj *o, int type) {
     /* A NULL is considered an empty key */
     if (o && o->type != type) {
-        addReply(c,shared.wrongtypeerr);
+        addReplyErrorObject(c,shared.wrongtypeerr);
         return 1;
     }
     return 0;
@@ -1321,13 +1321,13 @@ NULL
                 if (getLongLongFromObjectOrReply(c,c->argv[j+1],&samples,NULL)
                      == C_ERR) return;
                 if (samples < 0) {
-                    addReply(c,shared.syntaxerr);
+                    addReplyErrorObject(c,shared.syntaxerr);
                     return;
                 }
                 if (samples == 0) samples = LLONG_MAX;
                 j++; /* skip option argument. */
             } else {
-                addReply(c,shared.syntaxerr);
+                addReplyErrorObject(c,shared.syntaxerr);
                 return;
             }
         }
diff --git a/src/rdb.c b/src/rdb.c
index 58e7d2cff..abcfeadd7 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -2852,7 +2852,7 @@ void saveCommand(client *c) {
     if (rdbSave(server.rdb_filename,rsiptr) == C_OK) {
         addReply(c,shared.ok);
     } else {
-        addReply(c,shared.err);
+        addReplyErrorObject(c,shared.err);
     }
 }
 
@@ -2866,7 +2866,7 @@ void bgsaveCommand(client *c) {
         if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"schedule")) {
             schedule = 1;
         } else {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             return;
         }
     }
@@ -2889,7 +2889,7 @@ void bgsaveCommand(client *c) {
     } else if (rdbSaveBackground(server.rdb_filename,rsiptr) == C_OK) {
         addReplyStatus(c,"Background saving started");
     } else {
-        addReply(c,shared.err);
+        addReplyErrorObject(c,shared.err);
     }
 }
 
diff --git a/src/replication.c b/src/replication.c
index 34ff44087..92d7294f5 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -715,7 +715,7 @@ void syncCommand(client *c) {
     /* Refuse SYNC requests if we are a slave but the link with our master
      * is not ok... */
     if (server.masterhost && server.repl_state != REPL_STATE_CONNECTED) {
-        addReplySds(c,sdsnew("-NOMASTERLINK Can't SYNC while not connected with my master\r\n"));
+        addReplyError(c,"-NOMASTERLINK Can't SYNC while not connected with my master");
         return;
     }
 
@@ -866,7 +866,7 @@ void replconfCommand(client *c) {
     if ((c->argc % 2) == 0) {
         /* Number of arguments must be odd to make sure that every
          * option has a corresponding value. */
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
 
diff --git a/src/scripting.c b/src/scripting.c
index bc137de25..2ac8268ed 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -368,7 +368,7 @@ void luaReplyToRedisReply(client *c, lua_State *lua) {
         if (t == LUA_TSTRING) {
             sds err = sdsnew(lua_tostring(lua,-1));
             sdsmapchars(err,"\r\n","  ",2);
-            addReplySds(c,sdscatprintf(sdsempty(),"-%s\r\n",err));
+            addReplyErrorSds(c,sdscatprintf(sdsempty(),"-%s",err));
             sdsfree(err);
             lua_pop(lua,2);
             return;
@@ -1740,11 +1740,11 @@ NULL
         forceCommandPropagation(c,PROPAGATE_REPL|PROPAGATE_AOF);
     } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"kill")) {
         if (server.lua_caller == NULL) {
-            addReplySds(c,sdsnew("-NOTBUSY No scripts in execution right now.\r\n"));
+            addReplyError(c,"-NOTBUSY No scripts in execution right now.");
         } else if (server.lua_caller->flags & CLIENT_MASTER) {
-            addReplySds(c,sdsnew("-UNKILLABLE The busy script was sent by a master instance in the context of replication and cannot be killed.\r\n"));
+            addReplyError(c,"-UNKILLABLE The busy script was sent by a master instance in the context of replication and cannot be killed.");
         } else if (server.lua_write_dirty) {
-            addReplySds(c,sdsnew("-UNKILLABLE Sorry the script already executed write commands against the dataset. You can either wait the script termination or kill the server in a hard way using the SHUTDOWN NOSAVE command.\r\n"));
+            addReplyError(c,"-UNKILLABLE Sorry the script already executed write commands against the dataset. You can either wait the script termination or kill the server in a hard way using the SHUTDOWN NOSAVE command.");
         } else {
             server.lua_kill = 1;
             addReply(c,shared.ok);
diff --git a/src/sentinel.c b/src/sentinel.c
index 7693ac5ae..9764f9004 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -3446,7 +3446,7 @@ numargserr:
 /* SENTINEL INFO [section] */
 void sentinelInfoCommand(client *c) {
     if (c->argc > 2) {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
 
diff --git a/src/server.c b/src/server.c
index 611935a6c..f28c6e114 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2380,9 +2380,9 @@ void afterSleep(struct aeEventLoop *eventLoop) {
 void createSharedObjects(void) {
     int j;
 
+    /* Shared command responses */
     shared.crlf = createObject(OBJ_STRING,sdsnew("\r\n"));
     shared.ok = createObject(OBJ_STRING,sdsnew("+OK\r\n"));
-    shared.err = createObject(OBJ_STRING,sdsnew("-ERR\r\n"));
     shared.emptybulk = createObject(OBJ_STRING,sdsnew("$0\r\n\r\n"));
     shared.czero = createObject(OBJ_STRING,sdsnew(":0\r\n"));
     shared.cone = createObject(OBJ_STRING,sdsnew(":1\r\n"));
@@ -2390,8 +2390,14 @@ void createSharedObjects(void) {
     shared.pong = createObject(OBJ_STRING,sdsnew("+PONG\r\n"));
     shared.queued = createObject(OBJ_STRING,sdsnew("+QUEUED\r\n"));
     shared.emptyscan = createObject(OBJ_STRING,sdsnew("*2\r\n$1\r\n0\r\n*0\r\n"));
+    shared.space = createObject(OBJ_STRING,sdsnew(" "));
+    shared.colon = createObject(OBJ_STRING,sdsnew(":"));
+    shared.plus = createObject(OBJ_STRING,sdsnew("+"));
+
+    /* Shared command error responses */
     shared.wrongtypeerr = createObject(OBJ_STRING,sdsnew(
         "-WRONGTYPE Operation against a key holding the wrong kind of value\r\n"));
+    shared.err = createObject(OBJ_STRING,sdsnew("-ERR\r\n"));
     shared.nokeyerr = createObject(OBJ_STRING,sdsnew(
         "-ERR no such key\r\n"));
     shared.syntaxerr = createObject(OBJ_STRING,sdsnew(
@@ -2422,9 +2428,6 @@ void createSharedObjects(void) {
         "-NOREPLICAS Not enough good replicas to write.\r\n"));
     shared.busykeyerr = createObject(OBJ_STRING,sdsnew(
         "-BUSYKEY Target key name already exists.\r\n"));
-    shared.space = createObject(OBJ_STRING,sdsnew(" "));
-    shared.colon = createObject(OBJ_STRING,sdsnew(":"));
-    shared.plus = createObject(OBJ_STRING,sdsnew("+"));
 
     /* The shared NULL depends on the protocol version. */
     shared.null[0] = NULL;
@@ -4956,7 +4959,7 @@ void infoCommand(client *c) {
     char *section = c->argc == 2 ? c->argv[1]->ptr : "default";
 
     if (c->argc > 2) {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
     sds info = genRedisInfoString(section);
diff --git a/src/sort.c b/src/sort.c
index aeef53e6a..44637720b 100644
--- a/src/sort.c
+++ b/src/sort.c
@@ -256,7 +256,7 @@ void sortCommand(client *c) {
             getop++;
             j++;
         } else {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             syntax_error++;
             break;
         }
@@ -279,7 +279,7 @@ void sortCommand(client *c) {
                    sortval->type != OBJ_ZSET)
     {
         listRelease(operations);
-        addReply(c,shared.wrongtypeerr);
+        addReplyErrorObject(c,shared.wrongtypeerr);
         return;
     }
 
diff --git a/src/t_list.c b/src/t_list.c
index 1b4062c38..c8323c612 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -294,7 +294,7 @@ void linsertCommand(client *c) {
     } else if (strcasecmp(c->argv[2]->ptr,"before") == 0) {
         where = LIST_HEAD;
     } else {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
 
@@ -373,7 +373,7 @@ void lsetCommand(client *c) {
         int replaced = quicklistReplaceAtIndex(ql, index,
                                                value->ptr, sdslen(value->ptr));
         if (!replaced) {
-            addReply(c,shared.outofrangeerr);
+            addReplyErrorObject(c,shared.outofrangeerr);
         } else {
             addReply(c,shared.ok);
             signalModifiedKey(c,c->db,c->argv[1]);
@@ -566,7 +566,7 @@ void lposCommand(client *c) {
                 return;
             }
         } else {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             return;
         }
     }
@@ -698,7 +698,7 @@ int getListPositionFromObjectOrReply(client *c, robj *arg, int *position) {
     } else if (strcasecmp(arg->ptr,"left") == 0) {
         *position = LIST_HEAD;
     } else {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return C_ERR;
     }
     return C_OK;
diff --git a/src/t_set.c b/src/t_set.c
index fd9f4442a..64bbbd3a0 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -605,7 +605,7 @@ void spopCommand(client *c) {
         spopWithCountCommand(c);
         return;
     } else if (c->argc > 3) {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
 
@@ -801,7 +801,7 @@ void srandmemberCommand(client *c) {
         srandmemberWithCountCommand(c);
         return;
     } else if (c->argc > 3) {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
 
diff --git a/src/t_stream.c b/src/t_stream.c
index d61fb3eab..3ed9618fc 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1534,7 +1534,7 @@ void xrangeGenericCommand(client *c, int rev) {
                 if (count < 0) count = 0;
                 j++; /* Consume additional arg. */
             } else {
-                addReply(c,shared.syntaxerr);
+                addReplyErrorObject(c,shared.syntaxerr);
                 return;
             }
         }
@@ -1643,14 +1643,14 @@ void xreadCommand(client *c) {
             }
             noack = 1;
         } else {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             return;
         }
     }
 
     /* STREAMS option is mandatory. */
     if (streams_arg == 0) {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
 
@@ -2076,8 +2076,7 @@ NULL
             notifyKeyspaceEvent(NOTIFY_STREAM,"xgroup-create",
                                 c->argv[2],c->db->id);
         } else {
-            addReplySds(c,
-                sdsnew("-BUSYGROUP Consumer Group name already exists\r\n"));
+            addReplyError(c,"-BUSYGROUP Consumer Group name already exists");
         }
     } else if (!strcasecmp(opt,"SETID") && c->argc == 5) {
         streamID id;
@@ -2237,7 +2236,7 @@ void xpendingCommand(client *c) {
 
     /* Start and stop, and the consumer, can be omitted. Also the IDLE modifier. */
     if (c->argc != 3 && (c->argc < 6 || c->argc > 9)) {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
 
@@ -2251,7 +2250,7 @@ void xpendingCommand(client *c) {
                 return;
             if (c->argc < 8) {
                 /* If IDLE was provided we must have at least 'start end count' */
-                addReply(c,shared.syntaxerr);
+                addReplyErrorObject(c,shared.syntaxerr);
                 return;
             }
             /* Search for rest of arguments after 'IDLE <idle>' */
@@ -2758,7 +2757,7 @@ void xtrimCommand(client *c) {
             i++;
             maxlen_arg_idx = i;
         } else {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             return;
         }
     }
diff --git a/src/t_string.c b/src/t_string.c
index 3ecc473bd..2792f5557 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -153,7 +153,7 @@ void setCommand(client *c) {
             expire = next;
             j++;
         } else {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             return;
         }
     }
@@ -528,7 +528,7 @@ void stralgoCommand(client *c) {
     if (!strcasecmp(c->argv[1]->ptr,"lcs")) {
         stralgoLCS(c);
     } else {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
     }
 }
 
@@ -589,7 +589,7 @@ void stralgoLCS(client *c) {
             b = objb->ptr;
             j += 2;
         } else {
-            addReply(c,shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             goto cleanup;
         }
     }
diff --git a/src/t_zset.c b/src/t_zset.c
index a9564828a..25c7dda4c 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -1701,7 +1701,7 @@ void zaddGenericCommand(client *c, int flags) {
      * we expect any number of score-element pairs. */
     elements = c->argc-scoreidx;
     if (elements % 2 || !elements) {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
     elements /= 2; /* Now this holds the number of score-element pairs. */
@@ -2525,7 +2525,7 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in
 
     /* test if the expected number of keys would overflow */
     if (setnum > (c->argc-(numkeysIndex+1))) {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
 
@@ -2536,7 +2536,7 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in
         if (obj != NULL) {
             if (obj->type != OBJ_ZSET && obj->type != OBJ_SET) {
                 zfree(src);
-                addReply(c,shared.wrongtypeerr);
+                addReplyErrorObject(c,shared.wrongtypeerr);
                 return;
             }
 
@@ -2582,7 +2582,7 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in
                     aggregate = REDIS_AGGR_MAX;
                 } else {
                     zfree(src);
-                    addReply(c,shared.syntaxerr);
+                    addReplyErrorObject(c,shared.syntaxerr);
                     return;
                 }
                 j++; remaining--;
@@ -2594,7 +2594,7 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in
                 withscores = 1;
             } else {
                 zfree(src);
-                addReply(c,shared.syntaxerr);
+                addReplyErrorObject(c,shared.syntaxerr);
                 return;
             }
         }
@@ -2793,7 +2793,7 @@ void zrangeGenericCommand(client *c, int reverse) {
     if (c->argc == 5 && !strcasecmp(c->argv[4]->ptr,"withscores")) {
         withscores = 1;
     } else if (c->argc >= 5) {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
 
@@ -2938,7 +2938,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
                 }
                 pos += 3; remaining -= 3;
             } else {
-                addReply(c,shared.syntaxerr);
+                addReplyErrorObject(c,shared.syntaxerr);
                 return;
             }
         }
@@ -3289,7 +3289,7 @@ void genericZrangebylexCommand(client *c, int reverse) {
                 pos += 3; remaining -= 3;
             } else {
                 zslFreeLexRange(&range);
-                addReply(c,shared.syntaxerr);
+                addReplyErrorObject(c,shared.syntaxerr);
                 return;
             }
         }
@@ -3632,7 +3632,7 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey
 /* ZPOPMIN key [<count>] */
 void zpopminCommand(client *c) {
     if (c->argc > 3) {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
     genericZpopCommand(c,&c->argv[1],1,ZSET_MIN,0,
@@ -3642,7 +3642,7 @@ void zpopminCommand(client *c) {
 /* ZMAXPOP key [<count>] */
 void zpopmaxCommand(client *c) {
     if (c->argc > 3) {
-        addReply(c,shared.syntaxerr);
+        addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
     genericZpopCommand(c,&c->argv[1],1,ZSET_MAX,0,

From 59ff42c42156dcf4a5d9b637d8cb20c37ddc5c65 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Wed, 23 Dec 2020 19:13:12 -0800
Subject: [PATCH 0904/1098] Cleanup key tracking documentation and table
 management (#8039)

Cleanup key tracking documentation, always cleanup the tracking table, and free the tracking table in an async manner when applicable.
---
 redis.conf        |  2 +-
 src/aof.c         |  4 +--
 src/bio.c         | 61 +++++++++++++++++++++--------------
 src/bio.h         |  6 +++-
 src/db.c          |  6 ++--
 src/lazyfree.c    | 81 +++++++++++++++++++++++++++++------------------
 src/replication.c |  4 +--
 src/server.h      |  5 +--
 src/tracking.c    | 27 ++++++++++------
 9 files changed, 121 insertions(+), 75 deletions(-)

diff --git a/redis.conf b/redis.conf
index af4b4be1f..891bb1663 100644
--- a/redis.conf
+++ b/redis.conf
@@ -691,7 +691,7 @@ replica-priority 100
 
 # Redis implements server assisted support for client side caching of values.
 # This is implemented using an invalidation table that remembers, using
-# 16 millions of slots, what clients may have certain subsets of keys. In turn
+# a radix key indexed by key name, what clients have which keys. In turn
 # this is used in order to send invalidation messages to clients. Please
 # check this page to understand more about the feature:
 #
diff --git a/src/aof.c b/src/aof.c
index 79b2f1284..2accc781e 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -206,7 +206,7 @@ int aofFsyncInProgress(void) {
 /* Starts a background task that performs fsync() against the specified
  * file descriptor (the one of the AOF file) in another thread. */
 void aof_background_fsync(int fd) {
-    bioCreateBackgroundJob(BIO_AOF_FSYNC,(void*)(long)fd,NULL,NULL);
+    bioCreateFsyncJob(fd);
 }
 
 /* Kills an AOFRW child process if exists */
@@ -1909,7 +1909,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
             server.aof_state = AOF_ON;
 
         /* Asynchronously close the overwritten AOF. */
-        if (oldfd != -1) bioCreateBackgroundJob(BIO_CLOSE_FILE,(void*)(long)oldfd,NULL,NULL);
+        if (oldfd != -1) bioCreateCloseJob(oldfd);
 
         serverLog(LL_VERBOSE,
             "Background AOF rewrite signal handler took %lldus", ustime()-now);
diff --git a/src/bio.c b/src/bio.c
index a11bcb18b..c6e17f49d 100644
--- a/src/bio.c
+++ b/src/bio.c
@@ -78,15 +78,13 @@ static unsigned long long bio_pending[BIO_NUM_OPS];
  * file as the API does not expose the internals at all. */
 struct bio_job {
     time_t time; /* Time at which the job was created. */
-    /* Job specific arguments pointers. If we need to pass more than three
-     * arguments we can just pass a pointer to a structure or alike. */
-    void *arg1, *arg2, *arg3;
+    /* Job specific arguments.*/
+    int fd; /* Fd for file based background jobs */
+    lazy_free_fn *free_fn; /* Function that will free the provided arguments */
+    void *free_args[]; /* List of arguments to be passed to the free function */
 };
 
 void *bioProcessBackgroundJobs(void *arg);
-void lazyfreeFreeObjectFromBioThread(robj *o);
-void lazyfreeFreeDatabaseFromBioThread(dict *ht1, dict *ht2);
-void lazyfreeFreeSlotsMapFromBioThread(rax *rt);
 
 /* Make sure we have enough stack to perform all the things we do in the
  * main thread. */
@@ -128,13 +126,8 @@ void bioInit(void) {
     }
 }
 
-void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3) {
-    struct bio_job *job = zmalloc(sizeof(*job));
-
+void bioSubmitJob(int type, struct bio_job *job) {
     job->time = time(NULL);
-    job->arg1 = arg1;
-    job->arg2 = arg2;
-    job->arg3 = arg3;
     pthread_mutex_lock(&bio_mutex[type]);
     listAddNodeTail(bio_jobs[type],job);
     bio_pending[type]++;
@@ -142,6 +135,35 @@ void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3) {
     pthread_mutex_unlock(&bio_mutex[type]);
 }
 
+void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...) {
+    va_list valist;
+    /* Allocate memory for the job structure and all required
+     * arguments */
+    struct bio_job *job = zmalloc(sizeof(*job) + sizeof(void *) * (arg_count));
+    job->free_fn = free_fn;
+
+    va_start(valist, arg_count);
+    for (int i = 0; i < arg_count; i++) {
+        job->free_args[i] = va_arg(valist, void *);
+    }
+    va_end(valist);
+    bioSubmitJob(BIO_LAZY_FREE, job);
+}
+
+void bioCreateCloseJob(int fd) {
+    struct bio_job *job = zmalloc(sizeof(*job));
+    job->fd = fd;
+
+    bioSubmitJob(BIO_CLOSE_FILE, job);
+}
+
+void bioCreateFsyncJob(int fd) {
+    struct bio_job *job = zmalloc(sizeof(*job));
+    job->fd = fd;
+
+    bioSubmitJob(BIO_AOF_FSYNC, job);
+}
+
 void *bioProcessBackgroundJobs(void *arg) {
     struct bio_job *job;
     unsigned long type = (unsigned long) arg;
@@ -196,20 +218,11 @@ void *bioProcessBackgroundJobs(void *arg) {
 
         /* Process the job accordingly to its type. */
         if (type == BIO_CLOSE_FILE) {
-            close((long)job->arg1);
+            close(job->fd);
         } else if (type == BIO_AOF_FSYNC) {
-            redis_fsync((long)job->arg1);
+            redis_fsync(job->fd);
         } else if (type == BIO_LAZY_FREE) {
-            /* What we free changes depending on what arguments are set:
-             * arg1 -> free the object at pointer.
-             * arg2 & arg3 -> free two dictionaries (a Redis DB).
-             * only arg3 -> free the radix tree. */
-            if (job->arg1)
-                lazyfreeFreeObjectFromBioThread(job->arg1);
-            else if (job->arg2 && job->arg3)
-                lazyfreeFreeDatabaseFromBioThread(job->arg2,job->arg3);
-            else if (job->arg3)
-                lazyfreeFreeSlotsMapFromBioThread(job->arg3);
+            job->free_fn(job->free_args);
         } else {
             serverPanic("Wrong job type in bioProcessBackgroundJobs().");
         }
diff --git a/src/bio.h b/src/bio.h
index 6c2155941..1e6e97297 100644
--- a/src/bio.h
+++ b/src/bio.h
@@ -30,13 +30,17 @@
 #ifndef __BIO_H
 #define __BIO_H
 
+typedef void lazy_free_fn(void *args[]);
+
 /* Exported API */
 void bioInit(void);
-void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3);
 unsigned long long bioPendingJobsOfType(int type);
 unsigned long long bioWaitStepOfType(int type);
 time_t bioOlderJobOfType(int type);
 void bioKillThreads(void);
+void bioCreateCloseJob(int fd);
+void bioCreateFsyncJob(int fd);
+void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...);
 
 /* Background job opcodes */
 #define BIO_CLOSE_FILE    0 /* Deferred close(2) syscall. */
diff --git a/src/db.c b/src/db.c
index d5e8b07d3..64a473bf3 100644
--- a/src/db.c
+++ b/src/db.c
@@ -433,7 +433,7 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) {
     /* Make sure the WATCHed keys are affected by the FLUSH* commands.
      * Note that we need to call the function while the keys are still
      * there. */
-    signalFlushedDb(dbnum);
+    signalFlushedDb(dbnum, async);
 
     /* Empty redis database structure. */
     removed = emptyDbStructure(server.db, dbnum, async, callback);
@@ -572,9 +572,9 @@ void signalModifiedKey(client *c, redisDb *db, robj *key) {
     trackingInvalidateKey(c,key);
 }
 
-void signalFlushedDb(int dbid) {
+void signalFlushedDb(int dbid, int async) {
     touchWatchedKeysOnFlush(dbid);
-    trackingInvalidateKeysOnFlush(dbid);
+    trackingInvalidateKeysOnFlush(async);
 }
 
 /*-----------------------------------------------------------------------------
diff --git a/src/lazyfree.c b/src/lazyfree.c
index 125e6a1b0..8b9f0e2dc 100644
--- a/src/lazyfree.c
+++ b/src/lazyfree.c
@@ -6,6 +6,49 @@
 static redisAtomic size_t lazyfree_objects = 0;
 static redisAtomic size_t lazyfreed_objects = 0;
 
+/* Release objects from the lazyfree thread. It's just decrRefCount()
+ * updating the count of objects to release. */
+void lazyfreeFreeObject(void *args[]) {
+    robj *o = (robj *) args[0];
+    decrRefCount(o);
+    atomicDecr(lazyfree_objects,1);
+    atomicIncr(lazyfreed_objects,1);
+}
+
+/* Release a database from the lazyfree thread. The 'db' pointer is the
+ * database which was substituted with a fresh one in the main thread
+ * when the database was logically deleted. */
+void lazyfreeFreeDatabase(void *args[]) {
+    dict *ht1 = (dict *) args[0];
+    dict *ht2 = (dict *) args[1];
+
+    size_t numkeys = dictSize(ht1);
+    dictRelease(ht1);
+    dictRelease(ht2);
+    atomicDecr(lazyfree_objects,numkeys);
+    atomicIncr(lazyfreed_objects,numkeys);
+}
+
+/* Release the skiplist mapping Redis Cluster keys to slots in the
+ * lazyfree thread. */
+void lazyfreeFreeSlotsMap(void *args[]) {
+    rax *rt = args[0];
+    size_t len = rt->numele;
+    raxFree(rt);
+    atomicDecr(lazyfree_objects,len);
+    atomicIncr(lazyfreed_objects,len);
+}
+
+/* Release the rax mapping Redis Cluster keys to slots in the
+ * lazyfree thread. */
+void lazyFreeTrackingTable(void *args[]) {
+    rax *rt = args[0];
+    size_t len = rt->numele;
+    raxFree(rt);
+    atomicDecr(lazyfree_objects,len);
+    atomicIncr(lazyfreed_objects,len);
+}
+
 /* Return the number of currently pending objects to free. */
 size_t lazyfreeGetPendingObjectsCount(void) {
     size_t aux;
@@ -120,7 +163,7 @@ int dbAsyncDelete(redisDb *db, robj *key) {
          * equivalent to just calling decrRefCount(). */
         if (free_effort > LAZYFREE_THRESHOLD && val->refcount == 1) {
             atomicIncr(lazyfree_objects,1);
-            bioCreateBackgroundJob(BIO_LAZY_FREE,val,NULL,NULL);
+            bioCreateLazyFreeJob(lazyfreeFreeObject,1, val);
             dictSetVal(db->dict,de,NULL);
         }
     }
@@ -141,7 +184,7 @@ void freeObjAsync(robj *key, robj *obj) {
     size_t free_effort = lazyfreeGetFreeEffort(key,obj);
     if (free_effort > LAZYFREE_THRESHOLD && obj->refcount == 1) {
         atomicIncr(lazyfree_objects,1);
-        bioCreateBackgroundJob(BIO_LAZY_FREE,obj,NULL,NULL);
+        bioCreateLazyFreeJob(lazyfreeFreeObject,1,obj);
     } else {
         decrRefCount(obj);
     }
@@ -155,39 +198,17 @@ void emptyDbAsync(redisDb *db) {
     db->dict = dictCreate(&dbDictType,NULL);
     db->expires = dictCreate(&dbExpiresDictType,NULL);
     atomicIncr(lazyfree_objects,dictSize(oldht1));
-    bioCreateBackgroundJob(BIO_LAZY_FREE,NULL,oldht1,oldht2);
+    bioCreateLazyFreeJob(lazyfreeFreeDatabase,2,oldht1,oldht2);
 }
 
 /* Release the radix tree mapping Redis Cluster keys to slots asynchronously. */
 void freeSlotsToKeysMapAsync(rax *rt) {
     atomicIncr(lazyfree_objects,rt->numele);
-    bioCreateBackgroundJob(BIO_LAZY_FREE,NULL,NULL,rt);
+    bioCreateLazyFreeJob(lazyfreeFreeSlotsMap,1,rt);
 }
 
-/* Release objects from the lazyfree thread. It's just decrRefCount()
- * updating the count of objects to release. */
-void lazyfreeFreeObjectFromBioThread(robj *o) {
-    decrRefCount(o);
-    atomicDecr(lazyfree_objects,1);
-    atomicIncr(lazyfreed_objects,1);
-}
-
-/* Release a database from the lazyfree thread. The 'db' pointer is the
- * database which was substituted with a fresh one in the main thread
- * when the database was logically deleted. */
-void lazyfreeFreeDatabaseFromBioThread(dict *ht1, dict *ht2) {
-    size_t numkeys = dictSize(ht1);
-    dictRelease(ht1);
-    dictRelease(ht2);
-    atomicDecr(lazyfree_objects,numkeys);
-    atomicIncr(lazyfreed_objects,numkeys);
-}
-
-/* Release the radix tree mapping Redis Cluster keys to slots in the
- * lazyfree thread. */
-void lazyfreeFreeSlotsMapFromBioThread(rax *rt) {
-    size_t len = rt->numele;
-    raxFree(rt);
-    atomicDecr(lazyfree_objects,len);
-    atomicIncr(lazyfreed_objects,len);
+/* Free an object, if the object is huge enough, free it in async way. */
+void freeTrackingRadixTreeAsync(rax *tracking) {
+    atomicIncr(lazyfree_objects,tracking->numele);
+    bioCreateLazyFreeJob(lazyFreeTrackingTable,1,tracking);
 }
diff --git a/src/replication.c b/src/replication.c
index 92d7294f5..8a15c01a7 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -102,7 +102,7 @@ int bg_unlink(const char *filename) {
             errno = old_errno;
             return -1;
         }
-        bioCreateBackgroundJob(BIO_CLOSE_FILE,(void*)(long)fd,NULL,NULL);
+        bioCreateCloseJob(fd);
         return 0; /* Success. */
     }
 }
@@ -1752,7 +1752,7 @@ void readSyncBulkPayload(connection *conn) {
             return;
         }
         /* Close old rdb asynchronously. */
-        if (old_rdb_fd != -1) bioCreateBackgroundJob(BIO_CLOSE_FILE,(void*)(long)old_rdb_fd,NULL,NULL);
+        if (old_rdb_fd != -1) bioCreateCloseJob(old_rdb_fd);
 
         if (rdbLoad(server.rdb_filename,&rsi,RDBFLAGS_REPLICATION) != C_OK) {
             serverLog(LL_WARNING,
diff --git a/src/server.h b/src/server.h
index 04fd709a8..887f7752e 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1826,7 +1826,8 @@ void enableTracking(client *c, uint64_t redirect_to, uint64_t options, robj **pr
 void disableTracking(client *c);
 void trackingRememberKeys(client *c);
 void trackingInvalidateKey(client *c, robj *keyobj);
-void trackingInvalidateKeysOnFlush(int dbid);
+void trackingInvalidateKeysOnFlush(int async);
+void freeTrackingRadixTreeAsync(rax *rt);
 void trackingLimitUsedSlots(void);
 uint64_t trackingGetTotalItems(void);
 uint64_t trackingGetTotalKeys(void);
@@ -2260,7 +2261,7 @@ void discardDbBackup(dbBackup *buckup, int flags, void(callback)(void*));
 
 int selectDb(client *c, int id);
 void signalModifiedKey(client *c, redisDb *db, robj *key);
-void signalFlushedDb(int dbid);
+void signalFlushedDb(int dbid, int async);
 unsigned int getKeysInSlot(unsigned int hashslot, robj **keys, unsigned int count);
 unsigned int countKeysInSlot(unsigned int hashslot);
 unsigned int delKeysInSlot(unsigned int hashslot);
diff --git a/src/tracking.c b/src/tracking.c
index 913577eab..852fa229b 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -350,19 +350,22 @@ void trackingInvalidateKey(client *c, robj *keyobj) {
 }
 
 /* This function is called when one or all the Redis databases are
- * flushed (dbid == -1 in case of FLUSHALL). Caching keys are not
- * specific for each DB but are global: currently what we do is send a
- * special notification to clients with tracking enabled, sending a
- * RESP NULL, which means, "all the keys", in order to avoid flooding
- * clients with many invalidation messages for all the keys they may
- * hold.
+ * flushed. Caching keys are not specific for each DB but are global: 
+ * currently what we do is send a special notification to clients with 
+ * tracking enabled, sending a RESP NULL, which means, "all the keys", 
+ * in order to avoid flooding clients with many invalidation messages 
+ * for all the keys they may hold.
  */
-void freeTrackingRadixTree(void *rt) {
+void freeTrackingRadixTreeCallback(void *rt) {
     raxFree(rt);
 }
 
+void freeTrackingRadixTree(rax *rt) {
+    raxFreeWithCallback(rt,freeTrackingRadixTreeCallback);
+}
+
 /* A RESP NULL is sent to indicate that all keys are invalid */
-void trackingInvalidateKeysOnFlush(int dbid) {
+void trackingInvalidateKeysOnFlush(int async) {
     if (server.tracking_clients) {
         listNode *ln;
         listIter li;
@@ -376,8 +379,12 @@ void trackingInvalidateKeysOnFlush(int dbid) {
     }
 
     /* In case of FLUSHALL, reclaim all the memory used by tracking. */
-    if (dbid == -1 && TrackingTable) {
-        raxFreeWithCallback(TrackingTable,freeTrackingRadixTree);
+    if (TrackingTable) {
+        if (async) {
+            freeTrackingRadixTreeAsync(TrackingTable);
+        } else {
+            freeTrackingRadixTree(TrackingTable);
+        }
         TrackingTable = raxNew();
         TrackingTableTotalItems = 0;
     }

From b3dc23c5a8365a5e61b9ef8113970c1c36de6a13 Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Tue, 9 Jun 2020 19:46:49 +0800
Subject: [PATCH 0905/1098] add a read-only variant for HELLO As discussed in
 https://github.com/antirez/redis/issues/7364, it is good to have a HELLO
 command variant, which does not switch the current proto version of a redis
 server.

While `HELLO` will work, it introduced a certain difficulty on parsing
options of the command. We will need to offset the index of authentication
and setname option by -1.

So 0 is marked a special version meaning non-switching. And we do not
need to change the code much.
---
 src/networking.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 8fee298c6..e16a39a14 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2763,7 +2763,7 @@ void helloCommand(client *c) {
     long long ver;
 
     if (getLongLongFromObject(c->argv[1],&ver) != C_OK ||
-        ver < 2 || ver > 3)
+        (ver != 0 && ver < 2) || ver > 3)
     {
         addReplyError(c,"-NOPROTO unsupported protocol version");
         return;
@@ -2797,7 +2797,7 @@ void helloCommand(client *c) {
     }
 
     /* Let's switch to the specified RESP mode. */
-    c->resp = ver;
+    if (ver != 0) c->resp = ver;
     addReplyMapLen(c,6 + !server.sentinel_mode);
 
     addReplyBulkCString(c,"server");
@@ -2807,7 +2807,7 @@ void helloCommand(client *c) {
     addReplyBulkCString(c,REDIS_VERSION);
 
     addReplyBulkCString(c,"proto");
-    addReplyLongLong(c,ver);
+    addReplyLongLong(c,c->resp);
 
     addReplyBulkCString(c,"id");
     addReplyLongLong(c,c->id);

From 2e8f8c9b0c08d8c7c8b4dc72d65285d2761c5183 Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Thu, 24 Dec 2020 13:13:25 +0800
Subject: [PATCH 0906/1098] HELLO without protover

Signed-off-by: xhe <xw897002528@gmail.com>
---
 src/networking.c | 11 +++++------
 src/sentinel.c   |  2 +-
 src/server.c     |  2 +-
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index e16a39a14..37d4bf3d3 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2758,18 +2758,17 @@ NULL
     }
 }
 
-/* HELLO <protocol-version> [AUTH <user> <password>] [SETNAME <name>] */
+/* HELLO [protocol-version] [AUTH <user> <password>] [SETNAME <name>] */
 void helloCommand(client *c) {
-    long long ver;
+    long long ver = 0;
 
-    if (getLongLongFromObject(c->argv[1],&ver) != C_OK ||
-        (ver != 0 && ver < 2) || ver > 3)
-    {
+    if (c->argc >= 2 && getLongLongFromObject(c->argv[1],&ver) == C_OK &&
+            (ver < 2 || ver > 3)) {
         addReplyError(c,"-NOPROTO unsupported protocol version");
         return;
     }
 
-    for (int j = 2; j < c->argc; j++) {
+    for (int j = ver == 0 ? 1 : 2; j < c->argc; j++) {
         int moreargs = (c->argc-1) - j;
         const char *opt = c->argv[j]->ptr;
         if (!strcasecmp(opt,"AUTH") && moreargs >= 2) {
diff --git a/src/sentinel.c b/src/sentinel.c
index 9764f9004..1c1b4bf5b 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -469,7 +469,7 @@ struct redisCommand sentinelcmds[] = {
     {"client",clientCommand,-2,"admin random @connection",0,NULL,0,0,0,0,0},
     {"shutdown",shutdownCommand,-1,"admin",0,NULL,0,0,0,0,0},
     {"auth",authCommand,-2,"no-auth fast @connection",0,NULL,0,0,0,0,0},
-    {"hello",helloCommand,-2,"no-auth fast @connection",0,NULL,0,0,0,0,0},
+    {"hello",helloCommand,-1,"no-auth fast @connection",0,NULL,0,0,0,0,0},
     {"acl",aclCommand,-2,"admin",0,NULL,0,0,0,0,0,0},
     {"command",commandCommand,-1, "random @connection", 0,NULL,0,0,0,0,0,0}
 };
diff --git a/src/server.c b/src/server.c
index f28c6e114..0eabe8ea2 100644
--- a/src/server.c
+++ b/src/server.c
@@ -869,7 +869,7 @@ struct redisCommand redisCommandTable[] = {
      "admin no-script random ok-loading ok-stale @connection",
      0,NULL,0,0,0,0,0,0},
 
-    {"hello",helloCommand,-2,
+    {"hello",helloCommand,-1,
      "no-auth no-script fast no-monitor ok-loading ok-stale no-slowlog @connection",
      0,NULL,0,0,0,0,0,0},
 

From 7a7c60459e0853b1880bd1692fb15c4b67e00fa9 Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Thu, 24 Dec 2020 15:26:24 +0800
Subject: [PATCH 0907/1098] add a test

Signed-off-by: xhe <xw897002528@gmail.com>
---
 tests/unit/tracking.tcl | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index fc2800791..deecc1d06 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -135,6 +135,26 @@ start_server {tags {"tracking"}} {
         assert {[lindex $reply 2] eq {proto 3}}
     }
 
+    test {HELLO without protover} {
+        set reply [r HELLO 3]
+        assert {[lindex $reply 2] eq {proto 3}}
+
+        set reply [r HELLO]
+        assert {[lindex $reply 2] eq {proto 3}}
+
+        set reply [r HELLO]
+        assert {[lindex $reply 2] eq {proto 3}}
+
+        set reply [r HELLO 2]
+        assert {[lindex $reply 2] eq {proto 2}}
+
+        set reply [r HELLO]
+        assert {[lindex $reply 2] eq {proto 2}}
+
+        set reply [r HELLO]
+        assert {[lindex $reply 2] eq {proto 2}}
+    }
+
     test {RESP3 based basic invalidation} {
         r CLIENT TRACKING off
         r CLIENT TRACKING on

From 50d750733e52dd9fff26fbcf8274d8d6c81ebfca Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Thu, 24 Dec 2020 15:29:17 +0800
Subject: [PATCH 0908/1098] prefer !

Signed-off-by: xhe <xw897002528@gmail.com>
---
 src/networking.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 37d4bf3d3..d66eb2cea 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2768,7 +2768,7 @@ void helloCommand(client *c) {
         return;
     }
 
-    for (int j = ver == 0 ? 1 : 2; j < c->argc; j++) {
+    for (int j = !ver ? 1 : 2; j < c->argc; j++) {
         int moreargs = (c->argc-1) - j;
         const char *opt = c->argv[j]->ptr;
         if (!strcasecmp(opt,"AUTH") && moreargs >= 2) {
@@ -2796,7 +2796,7 @@ void helloCommand(client *c) {
     }
 
     /* Let's switch to the specified RESP mode. */
-    if (ver != 0) c->resp = ver;
+    if (!!ver) c->resp = ver;
     addReplyMapLen(c,6 + !server.sentinel_mode);
 
     addReplyBulkCString(c,"server");

From 35fc7fda7aac1699cf6f7ba902cf414d07c1cd3b Mon Sep 17 00:00:00 2001
From: Brad Dunbar <dunbarb2@gmail.com>
Date: Thu, 24 Dec 2020 03:42:52 -0500
Subject: [PATCH 0909/1098] Typo: timout -> timeout (#8228)

---
 src/networking.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 8fee298c6..ef1210b99 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -856,7 +856,7 @@ void addReplySubcommandSyntaxError(client *c) {
     sdsfree(cmd);
 }
 
-/* Append 'src' client output buffers into 'dst' client output buffers. 
+/* Append 'src' client output buffers into 'dst' client output buffers.
  * This function clears the output buffers of 'src' */
 void AddReplyFromClient(client *dst, client *src) {
     /* If the source client contains a partial response due to client output
@@ -2380,7 +2380,7 @@ void clientCommand(client *c) {
 "LIST [options ...]     -- Return information about client connections. Options:",
 "     TYPE (normal|master|replica|pubsub) -- Return clients of specified type.",
 "     ID id [id ...]                      -- Return clients of specified IDs only.",
-"PAUSE <timeout>        -- Suspend all Redis clients for <timout> milliseconds.",
+"PAUSE <timeout>        -- Suspend all Redis clients for <timeout> milliseconds.",
 "REPLY (on|off|skip)    -- Control the replies sent to the current connection.",
 "SETNAME <name>         -- Assign the name <name> to the current connection.",
 "UNBLOCK <clientid> [TIMEOUT|ERROR] -- Unblock the specified blocked client.",

From 60f13e7a865f7c0d4908b25803f42058c7a19201 Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Thu, 24 Dec 2020 16:50:08 +0800
Subject: [PATCH 0910/1098] try to fix the test

Signed-off-by: xhe <xw897002528@gmail.com>
---
 tests/unit/tracking.tcl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index deecc1d06..4d7da755d 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -146,13 +146,13 @@ start_server {tags {"tracking"}} {
         assert {[lindex $reply 2] eq {proto 3}}
 
         set reply [r HELLO 2]
-        assert {[lindex $reply 2] eq {proto 2}}
+        assert {[lindex $reply 6] eq 2}
 
         set reply [r HELLO]
-        assert {[lindex $reply 2] eq {proto 2}}
+        assert {[lindex $reply 6] eq 2}
 
         set reply [r HELLO]
-        assert {[lindex $reply 2] eq {proto 2}}
+        assert {[lindex $reply 6] eq 2}
     }
 
     test {RESP3 based basic invalidation} {

From 456c347d4520bce417e48d0d01a87035483755a5 Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Thu, 24 Dec 2020 17:03:22 +0800
Subject: [PATCH 0911/1098] simplify

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/networking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index d66eb2cea..0953b8e5e 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2796,7 +2796,7 @@ void helloCommand(client *c) {
     }
 
     /* Let's switch to the specified RESP mode. */
-    if (!!ver) c->resp = ver;
+    if (ver) c->resp = ver;
     addReplyMapLen(c,6 + !server.sentinel_mode);
 
     addReplyBulkCString(c,"server");

From c07d3bd8dd8ba108f743c09b059f157e9799d5c5 Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Thu, 24 Dec 2020 17:03:36 +0800
Subject: [PATCH 0912/1098] simplify

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/networking.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/networking.c b/src/networking.c
index 0953b8e5e..132596f9e 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2761,6 +2761,7 @@ NULL
 /* HELLO [protocol-version] [AUTH <user> <password>] [SETNAME <name>] */
 void helloCommand(client *c) {
     long long ver = 0;
+    int next_arg = 1;
 
     if (c->argc >= 2 && getLongLongFromObject(c->argv[1],&ver) == C_OK &&
             (ver < 2 || ver > 3)) {

From 723b4a15a34ce25490882f40ef51b24a5b61304c Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Thu, 24 Dec 2020 17:03:45 +0800
Subject: [PATCH 0913/1098] simplify

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/networking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index 132596f9e..32ffb3ce8 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2763,7 +2763,7 @@ void helloCommand(client *c) {
     long long ver = 0;
     int next_arg = 1;
 
-    if (c->argc >= 2 && getLongLongFromObject(c->argv[1],&ver) == C_OK &&
+    if (c->argc >= 2 && getLongLongFromObject(c->argv[next_arg++],&ver) == C_OK &&
             (ver < 2 || ver > 3)) {
         addReplyError(c,"-NOPROTO unsupported protocol version");
         return;

From 98f39a37fb14f2b1c1965de143011402728dafbc Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Thu, 24 Dec 2020 17:03:53 +0800
Subject: [PATCH 0914/1098] simplify

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/networking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index 32ffb3ce8..e29256488 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2769,7 +2769,7 @@ void helloCommand(client *c) {
         return;
     }
 
-    for (int j = !ver ? 1 : 2; j < c->argc; j++) {
+    for (int j = next_arg; j < c->argc; j++) {
         int moreargs = (c->argc-1) - j;
         const char *opt = c->argv[j]->ptr;
         if (!strcasecmp(opt,"AUTH") && moreargs >= 2) {

From ef14c18c8e83f3f11f5436e3dbeee9dfd5665de1 Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Thu, 24 Dec 2020 17:31:50 +0800
Subject: [PATCH 0915/1098] fix the test

Signed-off-by: xhe <xw897002528@gmail.com>
---
 tests/unit/tracking.tcl | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index 4d7da755d..1beaf71a9 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -146,13 +146,16 @@ start_server {tags {"tracking"}} {
         assert {[lindex $reply 2] eq {proto 3}}
 
         set reply [r HELLO 2]
-        assert {[lindex $reply 6] eq 2}
+        assert {[lindex $reply 4] eq "proto"}
+        assert {[lindex $reply 5] eq 2}
 
         set reply [r HELLO]
-        assert {[lindex $reply 6] eq 2}
+        assert {[lindex $reply 4] eq "proto"}
+        assert {[lindex $reply 5] eq 2}
 
         set reply [r HELLO]
-        assert {[lindex $reply 6] eq 2}
+        assert {[lindex $reply 4] eq "proto"}
+        assert {[lindex $reply 5] eq 2}
     }
 
     test {RESP3 based basic invalidation} {

From 9bd212cf241154d22849ef2addd0d5a7551b3a55 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 18 Dec 2020 00:02:57 +0200
Subject: [PATCH 0916/1098] syncWithMaster: sendSynchronousCommand split to
 send, and receive

This is just a refactoring commit.

This function was never actually used as a synchronous (do both send or
receive), it was always used only ine one of the two modes, which meant it
has to take extra arguments that are not relevant for the other.

Besides that, a tool that sends a synchronous command, it not something
we want in our toolbox (synchronous IO in single threaded app is evil).

sendSynchronousCommand was now refactored into separate sending and
receiving APIs, and the sending part has two variants, one taking vaargs,
and the other taking argc+argv (and an optional length array which means
you can use binary sds strings).
---
 src/replication.c | 173 ++++++++++++++++++++++++++--------------------
 1 file changed, 98 insertions(+), 75 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index 8a15c01a7..4977bba42 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1827,72 +1827,94 @@ error:
     return;
 }
 
-/* Send a synchronous command to the master. Used to send AUTH and
- * REPLCONF commands before starting the replication with SYNC.
+char *receiveSynchronousResponse(connection *conn) {
+    char buf[256];
+    /* Read the reply from the server. */
+    if (connSyncReadLine(conn,buf,sizeof(buf),server.repl_syncio_timeout*1000) == -1)
+    {
+        return sdscatprintf(sdsempty(),"-Reading from master: %s",
+                strerror(errno));
+    }
+    server.repl_transfer_lastio = server.unixtime;
+    return sdsnew(buf);
+}
+
+/* Send a pre-formatted multi-bulk command to the connection. */
+char* sendCommandRaw(connection *conn, sds cmd) {
+    if (connSyncWrite(conn,cmd,sdslen(cmd),server.repl_syncio_timeout*1000) == -1) {
+        return sdscatprintf(sdsempty(),"-Writing to master: %s",
+                connGetLastError(conn));
+    }
+    return NULL;
+}
+
+/* Compose a multi-bulk command and send it to the connection.
+ * Used to send AUTH and REPLCONF commands to the master before starting the
+ * replication.
+ *
+ * Takes a list of char* arguments, terminated by a NULL argument.
  *
  * The command returns an sds string representing the result of the
  * operation. On error the first byte is a "-".
  */
-#define SYNC_CMD_READ (1<<0)
-#define SYNC_CMD_WRITE (1<<1)
-#define SYNC_CMD_WRITE_SDS (1<<2)
-#define SYNC_CMD_FULL (SYNC_CMD_READ|SYNC_CMD_WRITE)
-char *sendSynchronousCommand(int flags, connection *conn, ...) {
+char *sendCommand(connection *conn, ...) {
+    va_list ap;
+    sds cmd = sdsempty();
+    sds cmdargs = sdsempty();
+    size_t argslen = 0;
+    char *arg;
 
     /* Create the command to send to the master, we use redis binary
      * protocol to make sure correct arguments are sent. This function
      * is not safe for all binary data. */
-    if (flags & SYNC_CMD_WRITE) {
-        char *arg;
-        va_list ap;
-        sds cmd = sdsempty();
-        sds cmdargs = sdsempty();
-        size_t argslen = 0;
-        va_start(ap,conn);
-
-        while(1) {
-            arg = va_arg(ap, char*);
-            if (arg == NULL) break;
-            if (flags & SYNC_CMD_WRITE_SDS) {
-                cmdargs = sdscatprintf(cmdargs,"$%zu\r\n", sdslen((sds)arg));
-                cmdargs = sdscatsds(cmdargs, (sds)arg);
-                cmdargs = sdscat(cmdargs, "\r\n");
-            } else {
-                cmdargs = sdscatprintf(cmdargs,"$%zu\r\n%s\r\n",strlen(arg),arg);
-            }
-            argslen++;
-        }
-
-        va_end(ap);
-
-        cmd = sdscatprintf(cmd,"*%zu\r\n",argslen);
-        cmd = sdscatsds(cmd,cmdargs);
-        sdsfree(cmdargs);
-
-        /* Transfer command to the server. */
-        if (connSyncWrite(conn,cmd,sdslen(cmd),server.repl_syncio_timeout*1000)
-            == -1)
-        {
-            sdsfree(cmd);
-            return sdscatprintf(sdsempty(),"-Writing to master: %s",
-                    connGetLastError(conn));
-        }
-        sdsfree(cmd);
+    va_start(ap,conn);
+    while(1) {
+        arg = va_arg(ap, char*);
+        if (arg == NULL) break;
+        cmdargs = sdscatprintf(cmdargs,"$%zu\r\n%s\r\n",strlen(arg),arg);
+        argslen++;
     }
 
-    /* Read the reply from the server. */
-    if (flags & SYNC_CMD_READ) {
-        char buf[256];
+    cmd = sdscatprintf(cmd,"*%zu\r\n",argslen);
+    cmd = sdscatsds(cmd,cmdargs);
+    sdsfree(cmdargs);
 
-        if (connSyncReadLine(conn,buf,sizeof(buf),server.repl_syncio_timeout*1000)
-            == -1)
-        {
-            return sdscatprintf(sdsempty(),"-Reading from master: %s",
-                    strerror(errno));
-        }
-        server.repl_transfer_lastio = server.unixtime;
-        return sdsnew(buf);
+    va_end(ap);
+    char* err = sendCommandRaw(conn, cmd);
+    sdsfree(cmd);
+    if(err)
+        return err;
+    return NULL;
+}
+
+/* Compose a multi-bulk command and send it to the connection. 
+ * Used to send AUTH and REPLCONF commands to the master before starting the
+ * replication.
+ *
+ * argv_lens is optional, when NULL, strlen is used.
+ *
+ * The command returns an sds string representing the result of the
+ * operation. On error the first byte is a "-".
+ */
+char *sendCommandArgv(connection *conn, int argc, char **argv, size_t *argv_lens) {
+    sds cmd = sdsempty();
+    char *arg;
+    int i;
+
+    /* Create the command to send to the master. */
+    cmd = sdscatfmt(cmd,"*%i\r\n",argc);
+    for (i=0; i<argc; i++) {
+        int len;
+        arg = argv[i];
+        len = argv_lens ? argv_lens[i] : strlen(arg);
+        cmd = sdscatfmt(cmd,"$%i\r\n",len);
+        cmd = sdscatlen(cmd,arg,len);
+        cmd = sdscatlen(cmd,"\r\n",2);
     }
+    char* err = sendCommandRaw(conn, cmd);
+    sdsfree(cmd);
+    if (err)
+        return err;
     return NULL;
 }
 
@@ -1975,7 +1997,7 @@ int slaveTryPartialResynchronization(connection *conn, int read_reply) {
         }
 
         /* Issue the PSYNC command */
-        reply = sendSynchronousCommand(SYNC_CMD_WRITE,conn,"PSYNC",psync_replid,psync_offset,NULL);
+        reply = sendCommand(conn,"PSYNC",psync_replid,psync_offset,NULL);
         if (reply != NULL) {
             serverLog(LL_WARNING,"Unable to send PSYNC to master: %s",reply);
             sdsfree(reply);
@@ -1986,7 +2008,7 @@ int slaveTryPartialResynchronization(connection *conn, int read_reply) {
     }
 
     /* Reading half */
-    reply = sendSynchronousCommand(SYNC_CMD_READ,conn,NULL);
+    reply = receiveSynchronousResponse(conn);
     if (sdslen(reply) == 0) {
         /* The master may send empty newlines after it receives PSYNC
          * and before to reply, just to keep the connection alive. */
@@ -2140,14 +2162,14 @@ void syncWithMaster(connection *conn) {
         server.repl_state = REPL_STATE_RECEIVE_PONG;
         /* Send the PING, don't check for errors at all, we have the timeout
          * that will take care about this. */
-        err = sendSynchronousCommand(SYNC_CMD_WRITE,conn,"PING",NULL);
+        err = sendCommand(conn,"PING",NULL);
         if (err) goto write_error;
         return;
     }
 
     /* Receive the PONG command. */
     if (server.repl_state == REPL_STATE_RECEIVE_PONG) {
-        err = sendSynchronousCommand(SYNC_CMD_READ,conn,NULL);
+        err = receiveSynchronousResponse(conn);
 
         /* We accept only two replies as valid, a positive +PONG reply
          * (we just check for "+") or an authentication error.
@@ -2173,17 +2195,18 @@ void syncWithMaster(connection *conn) {
     /* AUTH with the master if required. */
     if (server.repl_state == REPL_STATE_SEND_AUTH) {
         if (server.masterauth) {
-            sds auth = sdsnew("AUTH");
+            char *args[3] = {"AUTH",NULL,NULL};
+            size_t lens[3] = {4,0,0};
+            int argc = 1;
             if (server.masteruser) {
-                sds masteruser = sdsnew(server.masteruser);
-                err = sendSynchronousCommand(SYNC_CMD_WRITE | SYNC_CMD_WRITE_SDS, conn, auth,
-                                             masteruser, server.masterauth, NULL);
-                sdsfree(masteruser);
-            } else {
-                err = sendSynchronousCommand(SYNC_CMD_WRITE | SYNC_CMD_WRITE_SDS, conn, auth,
-                                             server.masterauth, NULL);
+                args[argc] = server.masteruser;
+                lens[argc] = strlen(server.masteruser);
+                argc++;
             }
-            sdsfree(auth);
+            args[argc] = server.masterauth;
+            lens[argc] = sdslen(server.masterauth);
+            argc++;
+            err = sendCommandArgv(conn, argc, args, lens);
             if (err) goto write_error;
             server.repl_state = REPL_STATE_RECEIVE_AUTH;
             return;
@@ -2194,7 +2217,7 @@ void syncWithMaster(connection *conn) {
 
     /* Receive AUTH reply. */
     if (server.repl_state == REPL_STATE_RECEIVE_AUTH) {
-        err = sendSynchronousCommand(SYNC_CMD_READ,conn,NULL);
+        err = receiveSynchronousResponse(conn);
         if (err[0] == '-') {
             serverLog(LL_WARNING,"Unable to AUTH to MASTER: %s",err);
             sdsfree(err);
@@ -2212,7 +2235,7 @@ void syncWithMaster(connection *conn) {
         else if (server.tls_replication && server.tls_port) port = server.tls_port;
         else port = server.port;
         sds portstr = sdsfromlonglong(port);
-        err = sendSynchronousCommand(SYNC_CMD_WRITE,conn,"REPLCONF",
+        err = sendCommand(conn,"REPLCONF",
                 "listening-port",portstr, NULL);
         sdsfree(portstr);
         if (err) goto write_error;
@@ -2223,7 +2246,7 @@ void syncWithMaster(connection *conn) {
 
     /* Receive REPLCONF listening-port reply. */
     if (server.repl_state == REPL_STATE_RECEIVE_PORT) {
-        err = sendSynchronousCommand(SYNC_CMD_READ,conn,NULL);
+        err = receiveSynchronousResponse(conn);
         /* Ignore the error if any, not all the Redis versions support
          * REPLCONF listening-port. */
         if (err[0] == '-') {
@@ -2244,7 +2267,7 @@ void syncWithMaster(connection *conn) {
     /* Set the slave ip, so that Master's INFO command can list the
      * slave IP address port correctly in case of port forwarding or NAT. */
     if (server.repl_state == REPL_STATE_SEND_IP) {
-        err = sendSynchronousCommand(SYNC_CMD_WRITE,conn,"REPLCONF",
+        err = sendCommand(conn,"REPLCONF",
                 "ip-address",server.slave_announce_ip, NULL);
         if (err) goto write_error;
         sdsfree(err);
@@ -2254,7 +2277,7 @@ void syncWithMaster(connection *conn) {
 
     /* Receive REPLCONF ip-address reply. */
     if (server.repl_state == REPL_STATE_RECEIVE_IP) {
-        err = sendSynchronousCommand(SYNC_CMD_READ,conn,NULL);
+        err = receiveSynchronousResponse(conn);
         /* Ignore the error if any, not all the Redis versions support
          * REPLCONF listening-port. */
         if (err[0] == '-') {
@@ -2272,7 +2295,7 @@ void syncWithMaster(connection *conn) {
      *
      * The master will ignore capabilities it does not understand. */
     if (server.repl_state == REPL_STATE_SEND_CAPA) {
-        err = sendSynchronousCommand(SYNC_CMD_WRITE,conn,"REPLCONF",
+        err = sendCommand(conn,"REPLCONF",
                 "capa","eof","capa","psync2",NULL);
         if (err) goto write_error;
         sdsfree(err);
@@ -2282,7 +2305,7 @@ void syncWithMaster(connection *conn) {
 
     /* Receive CAPA reply. */
     if (server.repl_state == REPL_STATE_RECEIVE_CAPA) {
-        err = sendSynchronousCommand(SYNC_CMD_READ,conn,NULL);
+        err = receiveSynchronousResponse(conn);
         /* Ignore the error if any, not all the Redis versions support
          * REPLCONF capa. */
         if (err[0] == '-') {
@@ -2403,7 +2426,7 @@ error:
     server.repl_state = REPL_STATE_CONNECT;
     return;
 
-write_error: /* Handle sendSynchronousCommand(SYNC_CMD_WRITE) errors. */
+write_error: /* Handle sendCommand() errors. */
     serverLog(LL_WARNING,"Sending command to master in replication handshake: %s", err);
     sdsfree(err);
     goto error;

From e87c31de66482e83ac36857e500847bfe8c13574 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 18 Dec 2020 22:10:31 +0200
Subject: [PATCH 0917/1098] syncWithMaster: use pipeline for AUTH+REPLCONF*3

The commit deals with the syncWithMaster and the ugly state machine in it.
It attempts to clean it a bit, but more importantly it uses pipeline for
part of the work (rather than 7 round trips, we now have 4).
i.e. the connect and PING are separate, then AUTH + 3 REPLCONF in one pipeline,
and finally the PSYNC (must be separate since the master has to have an empty
output buffer).
---
 src/replication.c | 131 ++++++++++++++++++++++------------------------
 src/server.h      |  35 ++++++-------
 2 files changed, 81 insertions(+), 85 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index 4977bba42..520c43fa4 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1334,8 +1334,8 @@ void shiftReplicationId(void) {
 /* Returns 1 if the given replication state is a handshake state,
  * 0 otherwise. */
 int slaveIsInHandshakeState(void) {
-    return server.repl_state >= REPL_STATE_RECEIVE_PONG &&
-           server.repl_state <= REPL_STATE_RECEIVE_PSYNC;
+    return server.repl_state >= REPL_STATE_RECEIVE_PING_REPLY &&
+           server.repl_state <= REPL_STATE_RECEIVE_PSYNC_REPLY;
 }
 
 /* Avoid the master to detect the slave is timing out while loading the
@@ -2159,7 +2159,7 @@ void syncWithMaster(connection *conn) {
          * registered and we can wait for the PONG reply. */
         connSetReadHandler(conn, syncWithMaster);
         connSetWriteHandler(conn, NULL);
-        server.repl_state = REPL_STATE_RECEIVE_PONG;
+        server.repl_state = REPL_STATE_RECEIVE_PING_REPLY;
         /* Send the PING, don't check for errors at all, we have the timeout
          * that will take care about this. */
         err = sendCommand(conn,"PING",NULL);
@@ -2168,7 +2168,7 @@ void syncWithMaster(connection *conn) {
     }
 
     /* Receive the PONG command. */
-    if (server.repl_state == REPL_STATE_RECEIVE_PONG) {
+    if (server.repl_state == REPL_STATE_RECEIVE_PING_REPLY) {
         err = receiveSynchronousResponse(conn);
 
         /* We accept only two replies as valid, a positive +PONG reply
@@ -2189,11 +2189,12 @@ void syncWithMaster(connection *conn) {
                 "Master replied to PING, replication can continue...");
         }
         sdsfree(err);
-        server.repl_state = REPL_STATE_SEND_AUTH;
+        err = NULL;
+        server.repl_state = REPL_STATE_SEND_HANDSHAKE;
     }
 
-    /* AUTH with the master if required. */
-    if (server.repl_state == REPL_STATE_SEND_AUTH) {
+    if (server.repl_state == REPL_STATE_SEND_HANDSHAKE) {
+        /* AUTH with the master if required. */
         if (server.masterauth) {
             char *args[3] = {"AUTH",NULL,NULL};
             size_t lens[3] = {4,0,0};
@@ -2208,15 +2209,53 @@ void syncWithMaster(connection *conn) {
             argc++;
             err = sendCommandArgv(conn, argc, args, lens);
             if (err) goto write_error;
-            server.repl_state = REPL_STATE_RECEIVE_AUTH;
-            return;
-        } else {
-            server.repl_state = REPL_STATE_SEND_PORT;
         }
+
+        /* Set the slave port, so that Master's INFO command can list the
+         * slave listening port correctly. */
+        {
+            int port;
+            if (server.slave_announce_port)
+                port = server.slave_announce_port;
+            else if (server.tls_replication && server.tls_port)
+                port = server.tls_port;
+            else
+                port = server.port;
+            sds portstr = sdsfromlonglong(port);
+            err = sendCommand(conn,"REPLCONF",
+                    "listening-port",portstr, NULL);
+            sdsfree(portstr);
+            if (err) goto write_error;
+        }
+
+        /* Set the slave ip, so that Master's INFO command can list the
+         * slave IP address port correctly in case of port forwarding or NAT.
+         * Skip REPLCONF ip-address if there is no slave-announce-ip option set. */
+        if (server.slave_announce_ip) {
+            err = sendCommand(conn,"REPLCONF",
+                    "ip-address",server.slave_announce_ip, NULL);
+            if (err) goto write_error;
+        }
+
+        /* Inform the master of our (slave) capabilities.
+         *
+         * EOF: supports EOF-style RDB transfer for diskless replication.
+         * PSYNC2: supports PSYNC v2, so understands +CONTINUE <new repl ID>.
+         *
+         * The master will ignore capabilities it does not understand. */
+        err = sendCommand(conn,"REPLCONF",
+                "capa","eof","capa","psync2",NULL);
+        if (err) goto write_error;
+
+        server.repl_state = REPL_STATE_RECEIVE_AUTH_REPLY;
+        return;
     }
 
+    if (server.repl_state == REPL_STATE_RECEIVE_AUTH_REPLY && !server.masterauth)
+        server.repl_state = REPL_STATE_RECEIVE_PORT_REPLY;
+
     /* Receive AUTH reply. */
-    if (server.repl_state == REPL_STATE_RECEIVE_AUTH) {
+    if (server.repl_state == REPL_STATE_RECEIVE_AUTH_REPLY) {
         err = receiveSynchronousResponse(conn);
         if (err[0] == '-') {
             serverLog(LL_WARNING,"Unable to AUTH to MASTER: %s",err);
@@ -2224,28 +2263,13 @@ void syncWithMaster(connection *conn) {
             goto error;
         }
         sdsfree(err);
-        server.repl_state = REPL_STATE_SEND_PORT;
-    }
-
-    /* Set the slave port, so that Master's INFO command can list the
-     * slave listening port correctly. */
-    if (server.repl_state == REPL_STATE_SEND_PORT) {
-        int port;
-        if (server.slave_announce_port) port = server.slave_announce_port;
-        else if (server.tls_replication && server.tls_port) port = server.tls_port;
-        else port = server.port;
-        sds portstr = sdsfromlonglong(port);
-        err = sendCommand(conn,"REPLCONF",
-                "listening-port",portstr, NULL);
-        sdsfree(portstr);
-        if (err) goto write_error;
-        sdsfree(err);
-        server.repl_state = REPL_STATE_RECEIVE_PORT;
+        err = NULL;
+        server.repl_state = REPL_STATE_RECEIVE_PORT_REPLY;
         return;
     }
 
     /* Receive REPLCONF listening-port reply. */
-    if (server.repl_state == REPL_STATE_RECEIVE_PORT) {
+    if (server.repl_state == REPL_STATE_RECEIVE_PORT_REPLY) {
         err = receiveSynchronousResponse(conn);
         /* Ignore the error if any, not all the Redis versions support
          * REPLCONF listening-port. */
@@ -2254,29 +2278,15 @@ void syncWithMaster(connection *conn) {
                                 "REPLCONF listening-port: %s", err);
         }
         sdsfree(err);
-        server.repl_state = REPL_STATE_SEND_IP;
-    }
-
-    /* Skip REPLCONF ip-address if there is no slave-announce-ip option set. */
-    if (server.repl_state == REPL_STATE_SEND_IP &&
-        server.slave_announce_ip == NULL)
-    {
-            server.repl_state = REPL_STATE_SEND_CAPA;
-    }
-
-    /* Set the slave ip, so that Master's INFO command can list the
-     * slave IP address port correctly in case of port forwarding or NAT. */
-    if (server.repl_state == REPL_STATE_SEND_IP) {
-        err = sendCommand(conn,"REPLCONF",
-                "ip-address",server.slave_announce_ip, NULL);
-        if (err) goto write_error;
-        sdsfree(err);
-        server.repl_state = REPL_STATE_RECEIVE_IP;
+        server.repl_state = REPL_STATE_RECEIVE_IP_REPLY;
         return;
     }
 
+    if (server.repl_state == REPL_STATE_RECEIVE_IP_REPLY && !server.slave_announce_ip)
+        server.repl_state = REPL_STATE_RECEIVE_CAPA_REPLY;
+
     /* Receive REPLCONF ip-address reply. */
-    if (server.repl_state == REPL_STATE_RECEIVE_IP) {
+    if (server.repl_state == REPL_STATE_RECEIVE_IP_REPLY) {
         err = receiveSynchronousResponse(conn);
         /* Ignore the error if any, not all the Redis versions support
          * REPLCONF listening-port. */
@@ -2285,26 +2295,12 @@ void syncWithMaster(connection *conn) {
                                 "REPLCONF ip-address: %s", err);
         }
         sdsfree(err);
-        server.repl_state = REPL_STATE_SEND_CAPA;
-    }
-
-    /* Inform the master of our (slave) capabilities.
-     *
-     * EOF: supports EOF-style RDB transfer for diskless replication.
-     * PSYNC2: supports PSYNC v2, so understands +CONTINUE <new repl ID>.
-     *
-     * The master will ignore capabilities it does not understand. */
-    if (server.repl_state == REPL_STATE_SEND_CAPA) {
-        err = sendCommand(conn,"REPLCONF",
-                "capa","eof","capa","psync2",NULL);
-        if (err) goto write_error;
-        sdsfree(err);
-        server.repl_state = REPL_STATE_RECEIVE_CAPA;
+        server.repl_state = REPL_STATE_RECEIVE_CAPA_REPLY;
         return;
     }
 
     /* Receive CAPA reply. */
-    if (server.repl_state == REPL_STATE_RECEIVE_CAPA) {
+    if (server.repl_state == REPL_STATE_RECEIVE_CAPA_REPLY) {
         err = receiveSynchronousResponse(conn);
         /* Ignore the error if any, not all the Redis versions support
          * REPLCONF capa. */
@@ -2313,6 +2309,7 @@ void syncWithMaster(connection *conn) {
                                   "REPLCONF capa: %s", err);
         }
         sdsfree(err);
+        err = NULL;
         server.repl_state = REPL_STATE_SEND_PSYNC;
     }
 
@@ -2326,12 +2323,12 @@ void syncWithMaster(connection *conn) {
             err = sdsnew("Write error sending the PSYNC command.");
             goto write_error;
         }
-        server.repl_state = REPL_STATE_RECEIVE_PSYNC;
+        server.repl_state = REPL_STATE_RECEIVE_PSYNC_REPLY;
         return;
     }
 
     /* If reached this point, we should be in REPL_STATE_RECEIVE_PSYNC. */
-    if (server.repl_state != REPL_STATE_RECEIVE_PSYNC) {
+    if (server.repl_state != REPL_STATE_RECEIVE_PSYNC_REPLY) {
         serverLog(LL_WARNING,"syncWithMaster(): state machine error, "
                              "state should be RECEIVE_PSYNC but is %d",
                              server.repl_state);
diff --git a/src/server.h b/src/server.h
index 887f7752e..a19b5ad9e 100644
--- a/src/server.h
+++ b/src/server.h
@@ -299,24 +299,23 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
 
 /* Slave replication state. Used in server.repl_state for slaves to remember
  * what to do next. */
-#define REPL_STATE_NONE 0 /* No active replication */
-#define REPL_STATE_CONNECT 1 /* Must connect to master */
-#define REPL_STATE_CONNECTING 2 /* Connecting to master */
-/* --- Handshake states, must be ordered --- */
-#define REPL_STATE_RECEIVE_PONG 3 /* Wait for PING reply */
-#define REPL_STATE_SEND_AUTH 4 /* Send AUTH to master */
-#define REPL_STATE_RECEIVE_AUTH 5 /* Wait for AUTH reply */
-#define REPL_STATE_SEND_PORT 6 /* Send REPLCONF listening-port */
-#define REPL_STATE_RECEIVE_PORT 7 /* Wait for REPLCONF reply */
-#define REPL_STATE_SEND_IP 8 /* Send REPLCONF ip-address */
-#define REPL_STATE_RECEIVE_IP 9 /* Wait for REPLCONF reply */
-#define REPL_STATE_SEND_CAPA 10 /* Send REPLCONF capa */
-#define REPL_STATE_RECEIVE_CAPA 11 /* Wait for REPLCONF reply */
-#define REPL_STATE_SEND_PSYNC 12 /* Send PSYNC */
-#define REPL_STATE_RECEIVE_PSYNC 13 /* Wait for PSYNC reply */
-/* --- End of handshake states --- */
-#define REPL_STATE_TRANSFER 14 /* Receiving .rdb from master */
-#define REPL_STATE_CONNECTED 15 /* Connected to master */
+typedef enum {
+    REPL_STATE_NONE = 0,            /* No active replication */
+    REPL_STATE_CONNECT,             /* Must connect to master */
+    REPL_STATE_CONNECTING,          /* Connecting to master */
+    /* --- Handshake states, must be ordered --- */
+    REPL_STATE_RECEIVE_PING_REPLY,  /* Wait for PING reply */
+    REPL_STATE_SEND_HANDSHAKE,      /* Send handshake sequance to master */
+    REPL_STATE_RECEIVE_AUTH_REPLY,  /* Wait for AUTH reply */
+    REPL_STATE_RECEIVE_PORT_REPLY,  /* Wait for REPLCONF reply */
+    REPL_STATE_RECEIVE_IP_REPLY,    /* Wait for REPLCONF reply */
+    REPL_STATE_RECEIVE_CAPA_REPLY,  /* Wait for REPLCONF reply */
+    REPL_STATE_SEND_PSYNC,          /* Send PSYNC */
+    REPL_STATE_RECEIVE_PSYNC_REPLY, /* Wait for PSYNC reply */
+    /* --- End of handshake states --- */
+    REPL_STATE_TRANSFER,        /* Receiving .rdb from master */
+    REPL_STATE_CONNECTED,       /* Connected to master */
+} repl_state;
 
 /* State of slaves from the POV of the master. Used in client->replstate.
  * In SEND_BULK and ONLINE state the slave receives new updates

From c4b52fc7c9e77fb1f2719d2cb5b9977b90698721 Mon Sep 17 00:00:00 2001
From: huangzhw <huang_zhw@126.com>
Date: Thu, 24 Dec 2020 17:58:43 +0800
Subject: [PATCH 0918/1098] cleanup: ziplist prev entry large length use
 sizeof(uint32_t) instead 4 (#8241)

This is just a cleanup, no bugs in the real world.

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/ziplist.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/ziplist.c b/src/ziplist.c
index 866078613..96ba47e13 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -431,19 +431,21 @@ unsigned int zipStoreEntryEncoding(unsigned char *p, unsigned char encoding, uns
 /* Encode the length of the previous entry and write it to "p". This only
  * uses the larger encoding (required in __ziplistCascadeUpdate). */
 int zipStorePrevEntryLengthLarge(unsigned char *p, unsigned int len) {
+    uint32_t u32;
     if (p != NULL) {
         p[0] = ZIP_BIG_PREVLEN;
-        memcpy(p+1,&len,sizeof(len));
+        u32 = len;
+        memcpy(p+1,&u32,sizeof(u32));
         memrev32ifbe(p+1);
     }
-    return 1+sizeof(len);
+    return 1 + sizeof(uint32_t);
 }
 
 /* Encode the length of the previous entry and write it to "p". Return the
  * number of bytes needed to encode this length if "p" is NULL. */
 unsigned int zipStorePrevEntryLength(unsigned char *p, unsigned int len) {
     if (p == NULL) {
-        return (len < ZIP_BIG_PREVLEN) ? 1 : sizeof(len)+1;
+        return (len < ZIP_BIG_PREVLEN) ? 1 : sizeof(uint32_t) + 1;
     } else {
         if (len < ZIP_BIG_PREVLEN) {
             p[0] = len;

From 4e36925c662bba9c83661caa166d4699bcbef17c Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Thu, 24 Dec 2020 19:16:28 +0800
Subject: [PATCH 0919/1098] correction

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/networking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index e29256488..6d571aad1 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2758,7 +2758,7 @@ NULL
     }
 }
 
-/* HELLO [protocol-version] [AUTH <user> <password>] [SETNAME <name>] */
+/* HELLO [<protocol-version> [AUTH <user> <password>] [SETNAME <name>] ] */
 void helloCommand(client *c) {
     long long ver = 0;
     int next_arg = 1;

From 955e00fbec2bf9cb2dbe53fceef00941681bfbd4 Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Thu, 24 Dec 2020 19:23:30 +0800
Subject: [PATCH 0920/1098] ask protover for authentication

Signed-off-by: xhe <xw897002528@gmail.com>
---
 src/networking.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/networking.c b/src/networking.c
index 6d571aad1..66e91c521 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2769,6 +2769,11 @@ void helloCommand(client *c) {
         return;
     }
 
+    if (!ver && next_arg < c->argc) {
+        addReplyError(c,"Authentication needs to provide an protocol version");
+        return;
+    }
+
     for (int j = next_arg; j < c->argc; j++) {
         int moreargs = (c->argc-1) - j;
         const char *opt = c->argv[j]->ptr;

From f6711b7da5445f3e4ba3595c9c0153183184078e Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Thu, 24 Dec 2020 19:25:30 +0800
Subject: [PATCH 0921/1098] reword

Signed-off-by: xhe <xw897002528@gmail.com>
---
 src/networking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index 66e91c521..0760a4669 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2770,7 +2770,7 @@ void helloCommand(client *c) {
     }
 
     if (!ver && next_arg < c->argc) {
-        addReplyError(c,"Authentication needs to provide an protocol version");
+        addReplyError(c,"Need to provide an protocol version for other arguments");
         return;
     }
 

From 78eaf503fdbafe94d1905301ba47294326b278e4 Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Thu, 24 Dec 2020 20:13:57 +0800
Subject: [PATCH 0922/1098] address comment

Signed-off-by: xhe <xw897002528@gmail.com>
---
 src/networking.c | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 0760a4669..c44976d12 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2763,15 +2763,16 @@ void helloCommand(client *c) {
     long long ver = 0;
     int next_arg = 1;
 
-    if (c->argc >= 2 && getLongLongFromObject(c->argv[next_arg++],&ver) == C_OK &&
-            (ver < 2 || ver > 3)) {
-        addReplyError(c,"-NOPROTO unsupported protocol version");
-        return;
-    }
+    if (c->argc >= 2) {
+        if (getLongLongFromObjectOrReply(c, c->argv[next_arg++], &ver,
+	    "The second argument should the protocol version if provided") != C_OK) {
+            return;
+	}
 
-    if (!ver && next_arg < c->argc) {
-        addReplyError(c,"Need to provide an protocol version for other arguments");
-        return;
+        if (ver < 2 || ver > 3) {
+            addReplyError(c,"-NOPROTO unsupported protocol version");
+            return;
+        }
     }
 
     for (int j = next_arg; j < c->argc; j++) {

From 4617960863c2baf3545ca74009b03358fa2363e1 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Thu, 24 Dec 2020 14:33:53 +0200
Subject: [PATCH 0923/1098] resolve hung test.

---
 tests/unit/tracking.tcl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index 1beaf71a9..b315293a0 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -156,6 +156,9 @@ start_server {tags {"tracking"}} {
         set reply [r HELLO]
         assert {[lindex $reply 4] eq "proto"}
         assert {[lindex $reply 5] eq 2}
+        
+        # restore RESP3 for next test
+        r HELLO 3
     }
 
     test {RESP3 based basic invalidation} {

From fae5ceef2ab65797af7d7778143efb8e4d598301 Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Fri, 25 Dec 2020 01:40:06 +0800
Subject: [PATCH 0924/1098] reword

Co-authored-by: Itamar Haber <itamar@redislabs.com>
---
 src/networking.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index c44976d12..187fafbb4 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2765,7 +2765,7 @@ void helloCommand(client *c) {
 
     if (c->argc >= 2) {
         if (getLongLongFromObjectOrReply(c, c->argv[next_arg++], &ver,
-	    "The second argument should the protocol version if provided") != C_OK) {
+	    "Protocol version is not an integer or out of range") != C_OK) {
             return;
 	}
 

From e6c1aeaf08b14076ccdf26aaef65b76b1597ee77 Mon Sep 17 00:00:00 2001
From: xhe <xw897002528@gmail.com>
Date: Fri, 25 Dec 2020 10:17:55 +0800
Subject: [PATCH 0925/1098] fix the format

Signed-off-by: xhe <xw897002528@gmail.com>
---
 src/networking.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 187fafbb4..ff71559a4 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2765,9 +2765,9 @@ void helloCommand(client *c) {
 
     if (c->argc >= 2) {
         if (getLongLongFromObjectOrReply(c, c->argv[next_arg++], &ver,
-	    "Protocol version is not an integer or out of range") != C_OK) {
+            "Protocol version is not an integer or out of range") != C_OK) {
             return;
-	}
+        }
 
         if (ver < 2 || ver > 3) {
             addReplyError(c,"-NOPROTO unsupported protocol version");

From e18068d9d83e6c7298b5501ffabfdb7709984fc0 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Fri, 25 Dec 2020 18:27:30 +0200
Subject: [PATCH 0926/1098] Use addReplyErrorObject with shared.syntaxerror
 (#8248)

---
 src/db.c  | 2 +-
 src/geo.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/db.c b/src/db.c
index 64a473bf3..495300613 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1208,7 +1208,7 @@ void copyCommand(client *c) {
             selectDb(c,srcid); /* Back to the source DB */
             j++; /* Consume additional arg. */
         } else {
-            addReply(c, shared.syntaxerr);
+            addReplyErrorObject(c,shared.syntaxerr);
             return;
         }
     }
diff --git a/src/geo.c b/src/geo.c
index 75a88398e..417bf897e 100644
--- a/src/geo.c
+++ b/src/geo.c
@@ -620,7 +620,7 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
                 bybox = 1;
                 i += 3;
             } else {
-                addReply(c, shared.syntaxerr);
+                addReplyErrorObject(c,shared.syntaxerr);
                 return;
             }
         }

From f44186e5755e943f39c1446ad9576ab97de5039d Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Fri, 25 Dec 2020 21:49:24 +0200
Subject: [PATCH 0927/1098] Adds count to L/RPOP (#8179)

Adds: `L/RPOP <key> [count]`

Implements no. 2 of the following strategies:

1. Loop on listTypePop - this would result in multiple calls for memory freeing and allocating (see https://github.com/redis/redis/pull/8179/commits/769167a079b0e110d28e4a8099dce1ecd45682b5)
2. Iterate the range to build the reply, then call quickListDelRange - this requires two iterations and **is the current choice**
3. Refactor quicklist to have a pop variant of quickListDelRange - probably optimal but more complex

Also:
* There's a historical check for NULL after calling listTypePop that was converted to an assert.
* This refactors common logic shared between LRANGE and the new form of LPOP/RPOP into addListRangeReply (adds test for b/w compat)
* Consequently, it may have made sense to have `LRANGE l -1 -2` and `LRANGE l 9 0` be legit and return a reverse reply. Due to historical reasons that would be, however, a breaking change.
* Added minimal comments to existing commands to adhere to the style, make core dev life easier and get commit karma, naturally.
---
 src/server.c             |   4 +-
 src/server.h             |   2 +-
 src/t_list.c             | 168 +++++++++++++++++++++++++++------------
 tests/unit/type/list.tcl |  17 ++++
 4 files changed, 138 insertions(+), 53 deletions(-)

diff --git a/src/server.c b/src/server.c
index f28c6e114..ad78abeba 100644
--- a/src/server.c
+++ b/src/server.c
@@ -288,11 +288,11 @@ struct redisCommand redisCommandTable[] = {
      "write use-memory @list",
      0,NULL,1,1,1,0,0,0},
 
-    {"rpop",rpopCommand,2,
+    {"rpop",rpopCommand,-2,
      "write fast @list",
      0,NULL,1,1,1,0,0,0},
 
-    {"lpop",lpopCommand,2,
+    {"lpop",lpopCommand,-2,
      "write fast @list",
      0,NULL,1,1,1,0,0,0},
 
diff --git a/src/server.h b/src/server.h
index a19b5ad9e..1dc761959 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1849,7 +1849,7 @@ void listTypeConvert(robj *subject, int enc);
 robj *listTypeDup(robj *o);
 void unblockClientWaitingData(client *c);
 void popGenericCommand(client *c, int where);
-void listElementsRemoved(client *c, robj *key, int where, robj *o);
+void listElementsRemoved(client *c, robj *key, int where, robj *o, long count);
 
 /* MULTI/EXEC/WATCH... */
 void unwatchAllKeys(client *c);
diff --git a/src/t_list.c b/src/t_list.c
index c8323c612..b64b37a8a 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -206,7 +206,7 @@ robj *listTypeDup(robj *o) {
             lobj->encoding = OBJ_ENCODING_QUICKLIST;
             break;
         default:
-            serverPanic("Wrong encoding.");
+            serverPanic("Unknown list encoding");
             break;
     }
     return lobj;
@@ -216,6 +216,7 @@ robj *listTypeDup(robj *o) {
  * List Commands
  *----------------------------------------------------------------------------*/
 
+/* Implements LPUSH/RPUSH. */
 void pushGenericCommand(client *c, int where) {
     int j, pushed = 0;
     robj *lobj = lookupKeyWrite(c->db,c->argv[1]);
@@ -244,14 +245,17 @@ void pushGenericCommand(client *c, int where) {
     server.dirty += pushed;
 }
 
+/* LPUSH <key> <element> [<element> ...] */
 void lpushCommand(client *c) {
     pushGenericCommand(c,LIST_HEAD);
 }
 
+/* RPUSH <key> <element> [<element> ...] */
 void rpushCommand(client *c) {
     pushGenericCommand(c,LIST_TAIL);
 }
 
+/* Implements LPUSHX/RPUSHX. */
 void pushxGenericCommand(client *c, int where) {
     int j, pushed = 0;
     robj *subject;
@@ -274,14 +278,17 @@ void pushxGenericCommand(client *c, int where) {
     server.dirty += pushed;
 }
 
+/* LPUSHX <key> <element> [<element> ...] */
 void lpushxCommand(client *c) {
     pushxGenericCommand(c,LIST_HEAD);
 }
 
+/* RPUSH <key> <element> [<element> ...] */
 void rpushxCommand(client *c) {
     pushxGenericCommand(c,LIST_TAIL);
 }
 
+/* LINSERT <key> (BEFORE|AFTER) <pivot> <element> */
 void linsertCommand(client *c) {
     int where;
     robj *subject;
@@ -326,12 +333,14 @@ void linsertCommand(client *c) {
     addReplyLongLong(c,listTypeLength(subject));
 }
 
+/* LLEN <key> */
 void llenCommand(client *c) {
     robj *o = lookupKeyReadOrReply(c,c->argv[1],shared.czero);
     if (o == NULL || checkType(c,o,OBJ_LIST)) return;
     addReplyLongLong(c,listTypeLength(o));
 }
 
+/* LINDEX <key> <index> */
 void lindexCommand(client *c) {
     robj *o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp]);
     if (o == NULL || checkType(c,o,OBJ_LIST)) return;
@@ -359,6 +368,7 @@ void lindexCommand(client *c) {
     }
 }
 
+/* LSET <key> <index> <element> */
 void lsetCommand(client *c) {
     robj *o = lookupKeyWriteOrReply(c,c->argv[1],shared.nokeyerr);
     if (o == NULL || checkType(c,o,OBJ_LIST)) return;
@@ -385,53 +395,15 @@ void lsetCommand(client *c) {
     }
 }
 
-void listElementsRemoved(client *c, robj *key, int where, robj *o) {
-    char *event = (where == LIST_HEAD) ? "lpop" : "rpop";
+/* A helper for replying with a list's range between the inclusive start and end
+ * indexes as multi-bulk, with support for negative indexes. Note that start
+ * must be less than end or an empty array is returned. When the reverse
+ * argument is set to a non-zero value, the reply is reversed so that elements
+ * are returned from end to start. */
+void addListRangeReply(client *c, robj *o, long start, long end, int reverse) {
+    long rangelen, llen = listTypeLength(o);
 
-    notifyKeyspaceEvent(NOTIFY_LIST, event, key, c->db->id);
-    if (listTypeLength(o) == 0) {
-        notifyKeyspaceEvent(NOTIFY_GENERIC, "del", key, c->db->id);
-        dbDelete(c->db, key);
-    }
-    signalModifiedKey(c, c->db, key);
-    server.dirty++;
-}
-
-void popGenericCommand(client *c, int where) {
-    robj *o = lookupKeyWriteOrReply(c, c->argv[1], shared.null[c->resp]);
-    if (o == NULL || checkType(c, o, OBJ_LIST))
-        return;
-
-    robj *value = listTypePop(o, where);
-    if (value == NULL) {
-        addReplyNull(c);
-    } else {
-        addReplyBulk(c,value);
-        decrRefCount(value);
-        listElementsRemoved(c,c->argv[1],where,o);
-    }
-}
-
-void lpopCommand(client *c) {
-    popGenericCommand(c,LIST_HEAD);
-}
-
-void rpopCommand(client *c) {
-    popGenericCommand(c,LIST_TAIL);
-}
-
-void lrangeCommand(client *c) {
-    robj *o;
-    long start, end, llen, rangelen;
-
-    if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != C_OK) ||
-        (getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != C_OK)) return;
-
-    if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptyarray)) == NULL
-         || checkType(c,o,OBJ_LIST)) return;
-    llen = listTypeLength(o);
-
-    /* convert negative indexes */
+    /* Convert negative indexes. */
     if (start < 0) start = llen+start;
     if (end < 0) end = llen+end;
     if (start < 0) start = 0;
@@ -448,7 +420,9 @@ void lrangeCommand(client *c) {
     /* Return the result in form of a multi-bulk reply */
     addReplyArrayLen(c,rangelen);
     if (o->encoding == OBJ_ENCODING_QUICKLIST) {
-        listTypeIterator *iter = listTypeInitIterator(o, start, LIST_TAIL);
+        int from = reverse ? end : start;
+        int direction = reverse ? LIST_HEAD : LIST_TAIL;
+        listTypeIterator *iter = listTypeInitIterator(o,from,direction);
 
         while(rangelen--) {
             listTypeEntry entry;
@@ -462,10 +436,98 @@ void lrangeCommand(client *c) {
         }
         listTypeReleaseIterator(iter);
     } else {
-        serverPanic("List encoding is not QUICKLIST!");
+        serverPanic("Unknown list encoding");
     }
 }
 
+/* A housekeeping helper for list elements popping tasks. */
+void listElementsRemoved(client *c, robj *key, int where, robj *o, long count) {
+    char *event = (where == LIST_HEAD) ? "lpop" : "rpop";
+
+    notifyKeyspaceEvent(NOTIFY_LIST, event, key, c->db->id);
+    if (listTypeLength(o) == 0) {
+        notifyKeyspaceEvent(NOTIFY_GENERIC, "del", key, c->db->id);
+        dbDelete(c->db, key);
+    }
+    signalModifiedKey(c, c->db, key);
+    server.dirty += count;
+}
+
+/* Implements the generic list pop operation for LPOP/RPOP.
+ * The where argument specifies which end of the list is operated on. An
+ * optional count may be provided as the third argument of the client's
+ * command. */
+void popGenericCommand(client *c, int where) {
+    long count = 0;
+    robj *value;
+
+    if (c->argc > 3) {
+        addReplyErrorFormat(c,"wrong number of arguments for '%s' command",
+                            c->cmd->name);
+        return;
+    } else if (c->argc == 3) {
+        /* Parse the optional count argument. */
+        if (getPositiveLongFromObjectOrReply(c,c->argv[2],&count,NULL) != C_OK) 
+            return;
+        if (count == 0) {
+            /* Fast exit path. */
+            addReplyNullArray(c);
+            return;
+        }
+    }
+
+    robj *o = lookupKeyWriteOrReply(c, c->argv[1], shared.null[c->resp]);
+    if (o == NULL || checkType(c, o, OBJ_LIST))
+        return;
+
+    if (!count) {
+        /* Pop a single element. This is POP's original behavior that replies
+         * with a bulk string. */
+        value = listTypePop(o,where);
+        serverAssert(value != NULL);
+        addReplyBulk(c,value);
+        decrRefCount(value);
+        listElementsRemoved(c,c->argv[1],where,o,1);
+    } else {
+        /* Pop a range of elements. An addition to the original POP command,
+         *  which replies with a multi-bulk. */
+        long llen = listTypeLength(o);
+        long rangelen = (count > llen) ? llen : count;
+        long rangestart = (where == LIST_HEAD) ? 0 : -rangelen;
+        long rangeend = (where == LIST_HEAD) ? rangelen - 1 : -1;
+        int reverse = (where == LIST_HEAD) ? 0 : 1;
+
+        addListRangeReply(c,o,rangestart,rangeend,reverse);
+        quicklistDelRange(o->ptr,rangestart,rangelen);
+        listElementsRemoved(c,c->argv[1],where,o,rangelen);
+    }
+}
+
+/* LPOP <key> [count] */
+void lpopCommand(client *c) {
+    popGenericCommand(c,LIST_HEAD);
+}
+
+/* RPOP <key> [count] */
+void rpopCommand(client *c) {
+    popGenericCommand(c,LIST_TAIL);
+}
+
+/* LRANGE <key> <start> <stop> */
+void lrangeCommand(client *c) {
+    robj *o;
+    long start, end;
+
+    if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != C_OK) ||
+        (getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != C_OK)) return;
+
+    if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptyarray)) == NULL
+         || checkType(c,o,OBJ_LIST)) return;
+
+    addListRangeReply(c,o,start,end,0);
+}
+
+/* LTRIM <key> <start> <stop> */
 void ltrimCommand(client *c) {
     robj *o;
     long start, end, llen, ltrim, rtrim;
@@ -629,6 +691,7 @@ void lposCommand(client *c) {
     }
 }
 
+/* LREM <key> <count> <element> */
 void lremCommand(client *c) {
     robj *subject, *obj;
     obj = c->argv[3];
@@ -756,6 +819,7 @@ void lmoveGenericCommand(client *c, int wherefrom, int whereto) {
     }
 }
 
+/* LMOVE <source> <destination> (LEFT|RIGHT) (LEFT|RIGHT) */
 void lmoveCommand(client *c) {
     int wherefrom, whereto;
     if (getListPositionFromObjectOrReply(c,c->argv[3],&wherefrom)
@@ -888,7 +952,7 @@ void blockingPopGenericCommand(client *c, int where) {
                     addReplyBulk(c,c->argv[j]);
                     addReplyBulk(c,value);
                     decrRefCount(value);
-                    listElementsRemoved(c,c->argv[j],where,o);
+                    listElementsRemoved(c,c->argv[j],where,o,1);
 
                     /* Replicate it as an [LR]POP instead of B[LR]POP. */
                     rewriteClientCommandVector(c,2,
@@ -912,10 +976,12 @@ void blockingPopGenericCommand(client *c, int where) {
     blockForKeys(c,BLOCKED_LIST,c->argv + 1,c->argc - 2,timeout,NULL,&pos,NULL);
 }
 
+/* BLPOP <key> [<key> ...] <timeout> */
 void blpopCommand(client *c) {
     blockingPopGenericCommand(c,LIST_HEAD);
 }
 
+/* BLPOP <key> [<key> ...] <timeout> */
 void brpopCommand(client *c) {
     blockingPopGenericCommand(c,LIST_TAIL);
 }
@@ -942,6 +1008,7 @@ void blmoveGenericCommand(client *c, int wherefrom, int whereto, mstime_t timeou
     }
 }
 
+/* BLMOVE <source> <destination> (LEFT|RIGHT) (LEFT|RIGHT) <timeout> */
 void blmoveCommand(client *c) {
     mstime_t timeout;
     int wherefrom, whereto;
@@ -954,6 +1021,7 @@ void blmoveCommand(client *c) {
     blmoveGenericCommand(c,wherefrom,whereto,timeout);
 }
 
+/* BRPOPLPUSH <source> <destination> <timeout> */
 void brpoplpushCommand(client *c) {
     mstime_t timeout;
     if (getTimeoutFromObjectOrReply(c,c->argv[3],&timeout,UNIT_SECONDS)
diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl
index 61ca23377..9be5dd93b 100644
--- a/tests/unit/type/list.tcl
+++ b/tests/unit/type/list.tcl
@@ -123,6 +123,17 @@ start_server {
     test {R/LPOP against empty list} {
         r lpop non-existing-list
     } {}
+    
+    test {R/LPOP with the optional count argument} {
+        assert_equal 7 [r lpush listcount aa bb cc dd ee ff gg]
+        assert_equal {} [r lpop listcount 0]
+        assert_equal {gg} [r lpop listcount 1]
+        assert_equal {ff ee} [r lpop listcount 2]
+        assert_equal {aa bb} [r rpop listcount 2]
+        assert_equal {cc} [r rpop listcount 1]
+        assert_equal {dd} [r rpop listcount 123]
+        assert_error "*ERR*range*" {r lpop forbarqaz -123}
+    }
 
     test {Variadic RPUSH/LPUSH} {
         r del mylist
@@ -947,6 +958,12 @@ start_server {
         assert_equal {} [r lrange nosuchkey 0 1]
     }
 
+    test {LRANGE with start > end yields an empty array for backward compatibility} {
+        create_list mylist "1 2 3"
+        assert_equal {} [r lrange mylist 1 0]
+        assert_equal {} [r lrange mylist -1 -2]
+    }
+
     foreach {type large} [array get largevalue] {
         proc trim_list {type min max} {
             upvar 1 large large

From 299f9ebffa5d0a79245a6e989cde775daf30d6a8 Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <276441700@qq.com>
Date: Sun, 27 Dec 2020 19:14:39 +0800
Subject: [PATCH 0928/1098] Tracking: add CLIENT TRACKINGINFO subcommand
 (#7309)

Add CLIENT TRACKINGINFO subcommand

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/networking.c        | 64 ++++++++++++++++++++++++++++++++-
 tests/unit/tracking.tcl | 79 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 142 insertions(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index ef1210b99..4f72e144a 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2384,9 +2384,10 @@ void clientCommand(client *c) {
 "REPLY (on|off|skip)    -- Control the replies sent to the current connection.",
 "SETNAME <name>         -- Assign the name <name> to the current connection.",
 "UNBLOCK <clientid> [TIMEOUT|ERROR] -- Unblock the specified blocked client.",
-"TRACKING (on|off) [REDIRECT <id>] [BCAST] [PREFIX first] [PREFIX second] [OPTIN] [OPTOUT]... -- Enable client keys tracking for client side caching.",
+"TRACKING (on|off) [REDIRECT <id>] [BCAST] [PREFIX first] [PREFIX second] [OPTIN] [OPTOUT] [NOLOOP]... -- Enable client keys tracking for client side caching.",
 "CACHING  (yes|no)      -- Enable/Disable tracking of the keys for next command in OPTIN/OPTOUT mode.",
 "GETREDIR               -- Return the client ID we are redirecting to when tracking is enabled.",
+"TRACKINGINFO           -- Return information about current client's tracking status.",
 NULL
         };
         addReplyHelp(c, help);
@@ -2753,6 +2754,67 @@ NULL
         } else {
             addReplyLongLong(c,-1);
         }
+    } else if (!strcasecmp(c->argv[1]->ptr,"trackinginfo") && c->argc == 2) {
+        addReplyMapLen(c,3);
+
+        /* Flags */
+        addReplyBulkCString(c,"flags");
+        void *arraylen_ptr = addReplyDeferredLen(c);
+        int numflags = 0;
+        addReplyBulkCString(c,c->flags & CLIENT_TRACKING ? "on" : "off");
+        numflags++;
+        if (c->flags & CLIENT_TRACKING_BCAST) {
+            addReplyBulkCString(c,"bcast");
+            numflags++;
+        }
+        if (c->flags & CLIENT_TRACKING_OPTIN) {
+            addReplyBulkCString(c,"optin");
+            numflags++;
+            if (c->flags & CLIENT_TRACKING_CACHING) {
+                addReplyBulkCString(c,"caching-yes");
+                numflags++;        
+            }
+        }
+        if (c->flags & CLIENT_TRACKING_OPTOUT) {
+            addReplyBulkCString(c,"optout");
+            numflags++;
+            if (c->flags & CLIENT_TRACKING_CACHING) {
+                addReplyBulkCString(c,"caching-no");
+                numflags++;        
+            }
+        }
+        if (c->flags & CLIENT_TRACKING_NOLOOP) {
+            addReplyBulkCString(c,"noloop");
+            numflags++;
+        }
+        if (c->flags & CLIENT_TRACKING_BROKEN_REDIR) {
+            addReplyBulkCString(c,"broken_redirect");
+            numflags++;
+        }
+        setDeferredSetLen(c,arraylen_ptr,numflags);
+
+        /* Redirect */
+        addReplyBulkCString(c,"redirect");
+        if (c->flags & CLIENT_TRACKING) {
+            addReplyLongLong(c,c->client_tracking_redirection);
+        } else {
+            addReplyLongLong(c,-1);
+        }
+
+        /* Prefixes */
+        addReplyBulkCString(c,"prefixes");
+        if (c->client_tracking_prefixes) {
+            addReplyArrayLen(c,raxSize(c->client_tracking_prefixes));
+            raxIterator ri;
+            raxStart(&ri,c->client_tracking_prefixes);
+            raxSeek(&ri,"^",NULL,0);
+            while(raxNext(&ri)) {
+                addReplyBulkCBuffer(c,ri.key,ri.key_len);
+            }
+            raxStop(&ri);
+        } else {
+            addReplyArrayLen(c,0);
+        }
     } else {
         addReplyErrorFormat(c, "Unknown subcommand or wrong number of arguments for '%s'. Try CLIENT HELP", (char*)c->argv[1]->ptr);
     }
diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index fc2800791..555c4e75e 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -398,6 +398,85 @@ start_server {tags {"tracking"}} {
         assert {$total_prefixes == 1}
     }
 
+    test {CLIENT TRACKINGINFO provides reasonable results when tracking off} {
+        r CLIENT TRACKING off
+        set res [r client trackinginfo]
+        set flags [dict get $res flags]
+        assert_equal {off} $flags
+        set redirect [dict get $res redirect]
+        assert_equal {-1} $redirect
+        set prefixes [dict get $res prefixes]
+        assert_equal {} $prefixes
+    }
+
+    test {CLIENT TRACKINGINFO provides reasonable results when tracking on} {
+        r CLIENT TRACKING on
+        set res [r client trackinginfo]
+        set flags [dict get $res flags]
+        assert_equal {on} $flags
+        set redirect [dict get $res redirect]
+        assert_equal {0} $redirect
+        set prefixes [dict get $res prefixes]
+        assert_equal {} $prefixes
+    }
+
+    test {CLIENT TRACKINGINFO provides reasonable results when tracking on with options} {
+        r CLIENT TRACKING on REDIRECT $redir_id noloop
+        set res [r client trackinginfo]
+        set flags [dict get $res flags]
+        assert_equal {on noloop} $flags
+        set redirect [dict get $res redirect]
+        assert_equal $redir_id $redirect
+        set prefixes [dict get $res prefixes]
+        assert_equal {} $prefixes
+    }
+
+    test {CLIENT TRACKINGINFO provides reasonable results when tracking optin} {
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on optin
+        set res [r client trackinginfo]
+        set flags [dict get $res flags]
+        assert_equal {on optin} $flags
+        set redirect [dict get $res redirect]
+        assert_equal {0} $redirect
+        set prefixes [dict get $res prefixes]
+        assert_equal {} $prefixes
+
+        r CLIENT CACHING yes
+        set res [r client trackinginfo]
+        set flags [dict get $res flags]
+        assert_equal {on optin caching-yes} $flags
+    }
+
+    test {CLIENT TRACKINGINFO provides reasonable results when tracking optout} {
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on optout
+        set res [r client trackinginfo]
+        set flags [dict get $res flags]
+        assert_equal {on optout} $flags
+        set redirect [dict get $res redirect]
+        assert_equal {0} $redirect
+        set prefixes [dict get $res prefixes]
+        assert_equal {} $prefixes
+
+        r CLIENT CACHING no
+        set res [r client trackinginfo]
+        set flags [dict get $res flags]
+        assert_equal {on optout caching-no} $flags
+    }
+
+    test {CLIENT TRACKINGINFO provides reasonable results when tracking bcast mode} {
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on BCAST PREFIX foo PREFIX bar
+        set res [r client trackinginfo]
+        set flags [dict get $res flags]
+        assert_equal {on bcast} $flags
+        set redirect [dict get $res redirect]
+        assert_equal {0} $redirect
+        set prefixes [lsort [dict get $res prefixes]]
+        assert_equal {bar foo} $prefixes
+    }
+
     $rd_redirection close
     $rd close
 }

From 049cf8cdf4e9e0abecf137dc1e3362089439f414 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 27 Dec 2020 21:40:12 +0200
Subject: [PATCH 0929/1098] Fix memory leaks in error replies due to recent
 change (#8249)

Recently efaf09ee4 started using addReplyErrorSds in place of
addReplySds the later takes ownership of the string but the former did
not.
This introduced memory leaks when a script returns an error to redis,
and also in clusterRedirectClient (two new usages of
addReplyErrorSds which was mostly unused till now.

This commit chagnes two thanks.
1. change addReplyErrorSds to take ownership of the error string.
2. scripting.c doesn't actually need to use addReplyErrorSds, it's a
perfect match for addReplyErrorFormat (replaces newlines with spaces)
---
 src/networking.c | 2 ++
 src/scripting.c  | 5 +----
 src/server.c     | 3 ++-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 24a601630..34e8b8481 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -458,9 +458,11 @@ void addReplyError(client *c, const char *err) {
 }
 
 /* See addReplyErrorLength for expectations from the input string. */
+/* As a side effect the SDS string is freed. */
 void addReplyErrorSds(client *c, sds err) {
     addReplyErrorLength(c,err,sdslen(err));
     afterErrorReply(c,err,sdslen(err));
+    sdsfree(err);
 }
 
 /* See addReplyErrorLength for expectations from the formatted string.
diff --git a/src/scripting.c b/src/scripting.c
index 2ac8268ed..e75892046 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -366,10 +366,7 @@ void luaReplyToRedisReply(client *c, lua_State *lua) {
         lua_gettable(lua,-2);
         t = lua_type(lua,-1);
         if (t == LUA_TSTRING) {
-            sds err = sdsnew(lua_tostring(lua,-1));
-            sdsmapchars(err,"\r\n","  ",2);
-            addReplyErrorSds(c,sdscatprintf(sdsempty(),"-%s",err));
-            sdsfree(err);
+            addReplyErrorFormat(c,"-%s",lua_tostring(lua,-1));
             lua_pop(lua,2);
             return;
         }
diff --git a/src/server.c b/src/server.c
index 90f669bdb..257a39f71 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3689,10 +3689,11 @@ void rejectCommandFormat(client *c, const char *fmt, ...) {
     sdsmapchars(s, "\r\n", "  ",  2);
     if (c->cmd && c->cmd->proc == execCommand) {
         execCommandAbort(c, s);
+        sdsfree(s);
     } else {
+        /* The following frees 's'. */
         addReplyErrorSds(c, s);
     }
-    sdsfree(s);
 }
 
 /* Returns 1 for commands that may have key names in their arguments, but have

From 1f5a73a530915f6f6326047effc796218af22cf6 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Wed, 30 Dec 2020 14:37:37 +0800
Subject: [PATCH 0930/1098] Merge pushGenericCommand and pushxGenericCommand
 (#8255)

Merge pushGenericCommand and pushxGenericCommand to remove redundancy and simplify code.
---
 src/t_list.c | 73 +++++++++++++++++++---------------------------------
 1 file changed, 26 insertions(+), 47 deletions(-)

diff --git a/src/t_list.c b/src/t_list.c
index b64b37a8a..f019a7ec0 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -216,76 +216,55 @@ robj *listTypeDup(robj *o) {
  * List Commands
  *----------------------------------------------------------------------------*/
 
-/* Implements LPUSH/RPUSH. */
-void pushGenericCommand(client *c, int where) {
-    int j, pushed = 0;
-    robj *lobj = lookupKeyWrite(c->db,c->argv[1]);
+/* Implements LPUSH/RPUSH/LPUSHX/RPUSHX. 
+ * 'xx': push if key exists. */
+void pushGenericCommand(client *c, int where, int xx) {
+    int j;
 
-    if (checkType(c,lobj,OBJ_LIST)) {
-        return;
+    robj *lobj = lookupKeyWrite(c->db, c->argv[1]);
+    if (checkType(c,lobj,OBJ_LIST)) return;
+    if (!lobj) {
+        if (xx) {
+            addReply(c, shared.czero);
+            return;
+        }
+
+        lobj = createQuicklistObject();
+        quicklistSetOptions(lobj->ptr, server.list_max_ziplist_size,
+                            server.list_compress_depth);
+        dbAdd(c->db,c->argv[1],lobj);
     }
 
     for (j = 2; j < c->argc; j++) {
-        if (!lobj) {
-            lobj = createQuicklistObject();
-            quicklistSetOptions(lobj->ptr, server.list_max_ziplist_size,
-                                server.list_compress_depth);
-            dbAdd(c->db,c->argv[1],lobj);
-        }
         listTypePush(lobj,c->argv[j],where);
-        pushed++;
+        server.dirty++;
     }
-    addReplyLongLong(c, (lobj ? listTypeLength(lobj) : 0));
-    if (pushed) {
-        char *event = (where == LIST_HEAD) ? "lpush" : "rpush";
 
-        signalModifiedKey(c,c->db,c->argv[1]);
-        notifyKeyspaceEvent(NOTIFY_LIST,event,c->argv[1],c->db->id);
-    }
-    server.dirty += pushed;
+    addReplyLongLong(c, listTypeLength(lobj));
+
+    char *event = (where == LIST_HEAD) ? "lpush" : "rpush";
+    signalModifiedKey(c,c->db,c->argv[1]);
+    notifyKeyspaceEvent(NOTIFY_LIST,event,c->argv[1],c->db->id);
 }
 
 /* LPUSH <key> <element> [<element> ...] */
 void lpushCommand(client *c) {
-    pushGenericCommand(c,LIST_HEAD);
+    pushGenericCommand(c,LIST_HEAD,0);
 }
 
 /* RPUSH <key> <element> [<element> ...] */
 void rpushCommand(client *c) {
-    pushGenericCommand(c,LIST_TAIL);
-}
-
-/* Implements LPUSHX/RPUSHX. */
-void pushxGenericCommand(client *c, int where) {
-    int j, pushed = 0;
-    robj *subject;
-
-    if ((subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL ||
-        checkType(c,subject,OBJ_LIST)) return;
-
-    for (j = 2; j < c->argc; j++) {
-        listTypePush(subject,c->argv[j],where);
-        pushed++;
-    }
-
-    addReplyLongLong(c,listTypeLength(subject));
-
-    if (pushed) {
-        char *event = (where == LIST_HEAD) ? "lpush" : "rpush";
-        signalModifiedKey(c,c->db,c->argv[1]);
-        notifyKeyspaceEvent(NOTIFY_LIST,event,c->argv[1],c->db->id);
-    }
-    server.dirty += pushed;
+    pushGenericCommand(c,LIST_TAIL,0);
 }
 
 /* LPUSHX <key> <element> [<element> ...] */
 void lpushxCommand(client *c) {
-    pushxGenericCommand(c,LIST_HEAD);
+    pushGenericCommand(c,LIST_HEAD,1);
 }
 
 /* RPUSH <key> <element> [<element> ...] */
 void rpushxCommand(client *c) {
-    pushxGenericCommand(c,LIST_TAIL);
+    pushGenericCommand(c,LIST_TAIL,1);
 }
 
 /* LINSERT <key> (BEFORE|AFTER) <pivot> <element> */

From 90b9f08e5d1657e7bfffe43f31f6663bf469ee75 Mon Sep 17 00:00:00 2001
From: filipe oliveira <filipecosta.90@gmail.com>
Date: Thu, 31 Dec 2020 14:53:43 +0000
Subject: [PATCH 0931/1098] Add errorstats info section, Add failed_calls and
 rejected_calls to commandstats (#8217)

This Commit pushes forward the observability on overall error statistics and command statistics within redis-server:

It extends INFO COMMANDSTATS to have
- failed_calls in - so we can keep track of errors that happen from the command itself, broken by command.
- rejected_calls - so we can keep track of errors that were triggered outside the commmand processing per se

Adds a new section to INFO, named ERRORSTATS that enables keeping track of the different errors that
occur within redis ( within processCommand and call ) based on the reply Error Prefix ( The first word
after the "-", up to the first space ).

This commit also fixes RM_ReplyWithError so that it can be correctly identified as an error reply.
---
 src/config.c                    |   1 +
 src/module.c                    |  24 +++--
 src/networking.c                |  18 ++++
 src/server.c                    |  63 ++++++++++++-
 src/server.h                    |  10 ++-
 tests/cluster/tests/18-info.tcl |  36 ++++++++
 tests/support/util.tcl          |   6 ++
 tests/test_helper.tcl           |   1 +
 tests/unit/info.tcl             | 153 ++++++++++++++++++++++++++++++++
 9 files changed, 294 insertions(+), 18 deletions(-)
 create mode 100644 tests/cluster/tests/18-info.tcl
 create mode 100644 tests/unit/info.tcl

diff --git a/src/config.c b/src/config.c
index a92b41624..9d078bfe3 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2560,6 +2560,7 @@ NULL
     } else if (!strcasecmp(c->argv[1]->ptr,"resetstat") && c->argc == 2) {
         resetServerStats();
         resetCommandTableStats();
+        resetErrorTableStats();
         addReply(c,shared.ok);
     } else if (!strcasecmp(c->argv[1]->ptr,"rewrite") && c->argc == 2) {
         if (server.configfile == NULL) {
diff --git a/src/module.c b/src/module.c
index 11f5f4489..913b4de5d 100644
--- a/src/module.c
+++ b/src/module.c
@@ -1368,18 +1368,6 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) {
     return REDISMODULE_OK;
 }
 
-/* Reply with an error or simple string (status message). Used to implement
- * ReplyWithSimpleString() and ReplyWithError().
- * The function always returns REDISMODULE_OK. */
-int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) {
-    client *c = moduleGetReplyClient(ctx);
-    if (c == NULL) return REDISMODULE_OK;
-    addReplyProto(c,prefix,strlen(prefix));
-    addReplyProto(c,msg,strlen(msg));
-    addReplyProto(c,"\r\n",2);
-    return REDISMODULE_OK;
-}
-
 /* Reply with the error 'err'.
  *
  * Note that 'err' must contain all the error, including
@@ -1395,7 +1383,10 @@ int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) {
  * The function always returns REDISMODULE_OK.
  */
 int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) {
-    return replyWithStatus(ctx,err,"-");
+    client *c = moduleGetReplyClient(ctx);
+    if (c == NULL) return REDISMODULE_OK;
+    addReplyErrorFormat(c,"-%s",err);
+    return REDISMODULE_OK;
 }
 
 /* Reply with a simple string (+... \r\n in RESP protocol). This replies
@@ -1404,7 +1395,12 @@ int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) {
  *
  * The function always returns REDISMODULE_OK. */
 int RM_ReplyWithSimpleString(RedisModuleCtx *ctx, const char *msg) {
-    return replyWithStatus(ctx,msg,"+");
+    client *c = moduleGetReplyClient(ctx);
+    if (c == NULL) return REDISMODULE_OK;
+    addReplyProto(c,"+",1);
+    addReplyProto(c,msg,strlen(msg));
+    addReplyProto(c,"\r\n",2);
+    return REDISMODULE_OK;
 }
 
 /* Reply with an array type of 'len' elements. However 'len' other calls
diff --git a/src/networking.c b/src/networking.c
index 34e8b8481..7957cc82f 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -405,6 +405,24 @@ void addReplyErrorLength(client *c, const char *s, size_t len) {
 
 /* Do some actions after an error reply was sent (Log if needed, updates stats, etc.) */
 void afterErrorReply(client *c, const char *s, size_t len) {
+    /* Increment the global error counter */
+    server.stat_total_error_replies++;
+    /* Increment the error stats
+     * If the string already starts with "-..." then the error prefix
+     * is provided by the caller ( we limit the search to 32 chars). Otherwise we use "-ERR". */
+    if (s[0] != '-') {
+        incrementErrorCount("ERR", 3);
+    } else {
+        char *spaceloc = memchr(s, ' ', len < 32 ? len : 32);
+        if (spaceloc) {
+            const size_t errEndPos = (size_t)(spaceloc - s);
+            incrementErrorCount(s+1, errEndPos-1);
+        } else {
+            /* Fallback to ERR if we can't retrieve the error prefix */
+            incrementErrorCount("ERR", 3);
+        }
+    }
+
     /* Sometimes it could be normal that a slave replies to a master with
      * an error and this function gets called. Actually the error will never
      * be sent because addReply*() against master clients has no effect...
diff --git a/src/server.c b/src/server.c
index 257a39f71..dbcaa767d 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2952,6 +2952,7 @@ void resetServerStats(void) {
     atomicSet(server.stat_net_input_bytes, 0);
     atomicSet(server.stat_net_output_bytes, 0);
     server.stat_unexpected_error_replies = 0;
+    server.stat_total_error_replies = 0;
     server.stat_dump_payload_sanitizations = 0;
     server.aof_delayed_fsync = 0;
     server.blocked_last_cron = 0;
@@ -2985,6 +2986,7 @@ void initServer(void) {
     server.in_fork_child = CHILD_TYPE_NONE;
     server.main_thread_id = pthread_self();
     server.current_client = NULL;
+    server.errors = raxNew();
     server.fixed_time_expire = 0;
     server.clients = listCreate();
     server.clients_index = raxNew();
@@ -3291,11 +3293,18 @@ void resetCommandTableStats(void) {
         c = (struct redisCommand *) dictGetVal(de);
         c->microseconds = 0;
         c->calls = 0;
+        c->rejected_calls = 0;
+        c->failed_calls = 0;
     }
     dictReleaseIterator(di);
 
 }
 
+void resetErrorTableStats(void) {
+    raxFree(server.errors);
+    server.errors = raxNew();
+}
+
 /* ========================== Redis OP Array API ============================ */
 
 void redisOpArrayInit(redisOpArray *oa) {
@@ -3490,6 +3499,7 @@ void call(client *c, int flags) {
     ustime_t start, duration;
     int client_old_flags = c->flags;
     struct redisCommand *real_cmd = c->cmd;
+    static long long prev_err_count;
 
     server.fixed_time_expire++;
 
@@ -3510,6 +3520,7 @@ void call(client *c, int flags) {
 
     /* Call the command. */
     dirty = server.dirty;
+    prev_err_count = server.stat_total_error_replies;
     updateCachedTime(0);
     start = server.ustime;
     c->cmd->proc(c);
@@ -3517,6 +3528,14 @@ void call(client *c, int flags) {
     dirty = server.dirty-dirty;
     if (dirty < 0) dirty = 0;
 
+    /* Update failed command calls if required.
+     * We leverage a static variable (prev_err_count) to retain
+     * the counter across nested function calls and avoid logging
+     * the same error twice. */
+    if ((server.stat_total_error_replies - prev_err_count) > 0) {
+        real_cmd->failed_calls++;
+    }
+
     /* After executing command, we will close the client after writing entire
      * reply if it is set 'CLIENT_CLOSE_AFTER_COMMAND' flag. */
     if (c->flags & CLIENT_CLOSE_AFTER_COMMAND) {
@@ -3655,6 +3674,7 @@ void call(client *c, int flags) {
 
     server.fixed_time_expire--;
     server.stat_numcommands++;
+    prev_err_count = server.stat_total_error_replies;
 
     /* Record peak memory after each command and before the eviction that runs
      * before the next command. */
@@ -3670,6 +3690,7 @@ void call(client *c, int flags) {
  * Note: 'reply' is expected to end with \r\n */
 void rejectCommand(client *c, robj *reply) {
     flagTransaction(c);
+    if (c->cmd) c->cmd->rejected_calls++;
     if (c->cmd && c->cmd->proc == execCommand) {
         execCommandAbort(c, reply->ptr);
     } else {
@@ -3679,6 +3700,7 @@ void rejectCommand(client *c, robj *reply) {
 }
 
 void rejectCommandFormat(client *c, const char *fmt, ...) {
+    if (c->cmd) c->cmd->rejected_calls++;
     flagTransaction(c);
     va_list ap;
     va_start(ap,fmt);
@@ -3805,6 +3827,7 @@ int processCommand(client *c) {
                 flagTransaction(c);
             }
             clusterRedirectClient(c,n,hashslot,error_code);
+            c->cmd->rejected_calls++;
             return C_OK;
         }
     }
@@ -3962,9 +3985,22 @@ int processCommand(client *c) {
         if (listLength(server.ready_keys))
             handleClientsBlockedOnKeys();
     }
+
     return C_OK;
 }
 
+/* ====================== Error lookup and execution ===================== */
+
+void incrementErrorCount(const char *fullerr, size_t namelen) {
+    struct redisError *error = raxFind(server.errors,(unsigned char*)fullerr,namelen);
+    if (error == raxNotFound) {
+        error = zmalloc(sizeof(*error));
+        error->count = 0;
+        raxInsert(server.errors,(unsigned char*)fullerr,namelen,error,NULL);
+    }
+    error->count++;
+}
+
 /*================================== Shutdown =============================== */
 
 /* Close listening sockets. Also unlink the unix domain socket if
@@ -4681,6 +4717,7 @@ sds genRedisInfoString(const char *section) {
             "tracking_total_items:%lld\r\n"
             "tracking_total_prefixes:%lld\r\n"
             "unexpected_error_replies:%lld\r\n"
+            "total_error_replies:%lld\r\n"
             "dump_payload_sanitizations:%lld\r\n"
             "total_reads_processed:%lld\r\n"
             "total_writes_processed:%lld\r\n"
@@ -4718,6 +4755,7 @@ sds genRedisInfoString(const char *section) {
             (unsigned long long) trackingGetTotalItems(),
             (unsigned long long) trackingGetTotalPrefixes(),
             server.stat_unexpected_error_replies,
+            server.stat_total_error_replies,
             server.stat_dump_payload_sanitizations,
             stat_total_reads_processed,
             stat_total_writes_processed,
@@ -4908,14 +4946,33 @@ sds genRedisInfoString(const char *section) {
         di = dictGetSafeIterator(server.commands);
         while((de = dictNext(di)) != NULL) {
             c = (struct redisCommand *) dictGetVal(de);
-            if (!c->calls) continue;
+            if (!c->calls && !c->failed_calls && !c->rejected_calls)
+                continue;
             info = sdscatprintf(info,
-                "cmdstat_%s:calls=%lld,usec=%lld,usec_per_call=%.2f\r\n",
+                "cmdstat_%s:calls=%lld,usec=%lld,usec_per_call=%.2f"
+                ",rejected_calls=%lld,failed_calls=%lld\r\n",
                 c->name, c->calls, c->microseconds,
-                (c->calls == 0) ? 0 : ((float)c->microseconds/c->calls));
+                (c->calls == 0) ? 0 : ((float)c->microseconds/c->calls),
+                c->rejected_calls, c->failed_calls);
         }
         dictReleaseIterator(di);
     }
+    /* Error statistics */
+    if (allsections || defsections || !strcasecmp(section,"errorstats")) {
+        if (sections++) info = sdscat(info,"\r\n");
+        info = sdscat(info, "# Errorstats\r\n");
+        raxIterator ri;
+        raxStart(&ri,server.errors);
+        raxSeek(&ri,"^",NULL,0);
+        struct redisError *e;
+        while(raxNext(&ri)) {
+            e = (struct redisError *) ri.data;
+            info = sdscatprintf(info,
+                "errorstat_%.*s:count=%lld\r\n",
+                (int)ri.key_len, ri.key, e->count);
+        }
+        raxStop(&ri);
+    }
 
     /* Cluster */
     if (allsections || defsections || !strcasecmp(section,"cluster")) {
diff --git a/src/server.h b/src/server.h
index 1dc761959..5f6b21ec4 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1122,6 +1122,7 @@ struct redisServer {
     dict *commands;             /* Command table */
     dict *orig_commands;        /* Command table before command renaming. */
     aeEventLoop *el;
+    rax *errors;                /* Errors table */
     redisAtomic unsigned int lruclock; /* Clock for LRU eviction */
     volatile sig_atomic_t shutdown_asap; /* SHUTDOWN needed ASAP */
     int activerehashing;        /* Incremental rehash in serverCron() */
@@ -1231,6 +1232,7 @@ struct redisServer {
     size_t stat_module_cow_bytes;   /* Copy on write bytes during module fork. */
     uint64_t stat_clients_type_memory[CLIENT_TYPE_COUNT];/* Mem usage by type */
     long long stat_unexpected_error_replies; /* Number of unexpected (aof-loading, replica to master, etc.) error replies */
+    long long stat_total_error_replies; /* Total number of issued error replies ( command + rejected errors ) */
     long long stat_dump_payload_sanitizations; /* Number deep dump payloads integrity validations. */
     long long stat_io_reads_processed; /* Number of read events processed by IO / Main threads */
     long long stat_io_writes_processed; /* Number of write events processed by IO / Main threads */
@@ -1579,7 +1581,7 @@ struct redisCommand {
     int firstkey; /* The first argument that's a key (0 = no keys) */
     int lastkey;  /* The last argument that's a key */
     int keystep;  /* The step between first and last key */
-    long long microseconds, calls;
+    long long microseconds, calls, rejected_calls, failed_calls;
     int id;     /* Command ID. This is a progressive ID starting from 0 that
                    is assigned at runtime, and is used in order to check
                    ACLs. A connection is able to execute a given command if
@@ -1587,6 +1589,10 @@ struct redisCommand {
                    bit set in the bitmap of allowed commands. */
 };
 
+struct redisError {
+    long long count;
+};
+
 struct redisFunctionSym {
     char *name;
     unsigned long pointer;
@@ -2132,7 +2138,9 @@ void updateDictResizePolicy(void);
 int htNeedsResize(dict *dict);
 void populateCommandTable(void);
 void resetCommandTableStats(void);
+void resetErrorTableStats(void);
 void adjustOpenFilesLimit(void);
+void incrementErrorCount(const char *fullerr, size_t namelen);
 void closeListeningSockets(int unlink_unix_socket);
 void updateCachedTime(int update_daylight_info);
 void resetServerStats(void);
diff --git a/tests/cluster/tests/18-info.tcl b/tests/cluster/tests/18-info.tcl
new file mode 100644
index 000000000..83dbf833f
--- /dev/null
+++ b/tests/cluster/tests/18-info.tcl
@@ -0,0 +1,36 @@
+# Check cluster info stats
+
+source "../tests/includes/init-tests.tcl"
+
+test "Create a primary with a replica" {
+    create_cluster 2 0
+}
+
+test "Cluster should start ok" {
+    assert_cluster_state ok
+}
+
+set primary1 [Rn 0]
+set primary2 [Rn 1]
+
+proc cmdstat {instace cmd} {
+    return [cmdrstat $cmd $instace]
+}
+
+proc errorstat {instace cmd} {
+    return [errorrstat $cmd $instace]
+}
+
+test "errorstats: rejected call due to MOVED Redirection" {
+    $primary1 config resetstat
+    $primary2 config resetstat
+    assert_match {} [errorstat $primary1 MOVED]
+    assert_match {} [errorstat $primary2 MOVED]
+    # we know that the primary 2 will have a MOVED reply
+    catch {$primary1 set key{0x0000} b} replyP1
+    catch {$primary2 set key{0x0000} b} replyP2
+    assert_match {OK} $replyP1
+    assert_match {} [errorstat $primary1 MOVED]
+    assert_match {*count=1*} [errorstat $primary2 MOVED]
+    assert_match {*calls=0,*,rejected_calls=1,failed_calls=0} [cmdstat $primary2 set]
+}
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index e4b70ed20..86f2753c2 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -561,6 +561,12 @@ proc cmdrstat {cmd r} {
     }
 }
 
+proc errorrstat {cmd r} {
+    if {[regexp "\r\nerrorstat_$cmd:(.*?)\r\n" [$r info errorstats] _ value]} {
+        set _ $value
+    }
+}
+
 proc generate_fuzzy_traffic_on_key {key duration} {
     # Commands per type, blocking commands removed
     # TODO: extract these from help.h or elsewhere, and improve to include other types
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 3b8dc16da..4bef921ff 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -18,6 +18,7 @@ set ::all_tests {
     unit/protocol
     unit/keyspace
     unit/scan
+    unit/info
     unit/type/string
     unit/type/incr
     unit/type/list
diff --git a/tests/unit/info.tcl b/tests/unit/info.tcl
new file mode 100644
index 000000000..5a44c0647
--- /dev/null
+++ b/tests/unit/info.tcl
@@ -0,0 +1,153 @@
+proc cmdstat {cmd} {
+    return [cmdrstat $cmd r]
+}
+
+proc errorstat {cmd} {
+    return [errorrstat $cmd r]
+}
+
+start_server {tags {"info"}} {
+    start_server {} {
+
+        test {errorstats: failed call authentication error} {
+            r config resetstat
+            assert_match {} [errorstat ERR]
+            assert_equal [s total_error_replies] 0
+            catch {r auth k} e
+            assert_match {ERR AUTH*} $e
+            assert_match {*count=1*} [errorstat ERR]
+            assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat auth]
+            assert_equal [s total_error_replies] 1
+            r config resetstat
+            assert_match {} [errorstat ERR]
+        }
+
+        test {errorstats: failed call within MULTI/EXEC} {
+            r config resetstat
+            assert_match {} [errorstat ERR]
+            assert_equal [s total_error_replies] 0
+            r multi
+            r set a b
+            r auth a
+            catch {r exec} e
+            assert_match {ERR AUTH*} $e
+            assert_match {*count=1*} [errorstat ERR]
+            assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} [cmdstat set]
+            assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat auth]
+            assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} [cmdstat exec]
+            assert_equal [s total_error_replies] 1
+
+            # MULTI/EXEC command errors should still be pinpointed to him
+            catch {r exec} e
+            assert_match {ERR EXEC without MULTI} $e
+            assert_match {*calls=2,*,rejected_calls=0,failed_calls=1} [cmdstat exec]
+            assert_match {*count=2*} [errorstat ERR]
+            assert_equal [s total_error_replies] 2
+        }
+
+        test {errorstats: failed call within LUA} {
+            r config resetstat
+            assert_match {} [errorstat ERR]
+            assert_equal [s total_error_replies] 0
+            catch {r eval {redis.pcall('XGROUP', 'CREATECONSUMER', 's1', 'mygroup', 'consumer') return } 0} e
+            assert_match {*count=1*} [errorstat ERR]
+            assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat xgroup]
+            assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} [cmdstat eval]
+
+            # EVAL command errors should still be pinpointed to him
+            catch {r eval a} e
+            assert_match {ERR wrong*} $e
+            assert_match {*calls=1,*,rejected_calls=1,failed_calls=0} [cmdstat eval]
+            assert_match {*count=2*} [errorstat ERR]
+            assert_equal [s total_error_replies] 2
+        }
+
+        test {errorstats: failed call NOGROUP error} {
+            r config resetstat
+            assert_match {} [errorstat NOGROUP]
+            r del mystream
+            r XADD mystream * f v
+            catch {r XGROUP CREATECONSUMER mystream mygroup consumer} e
+            assert_match {NOGROUP*} $e
+            assert_match {*count=1*} [errorstat NOGROUP]
+            assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat xgroup]
+            r config resetstat
+            assert_match {} [errorstat NOGROUP]
+        }
+
+        test {errorstats: rejected call unknown command} {
+            r config resetstat
+            assert_equal [s total_error_replies] 0
+            assert_match {} [errorstat ERR]
+            catch {r asdf} e
+            assert_match {ERR unknown*} $e
+            assert_match {*count=1*} [errorstat ERR]
+            assert_equal [s total_error_replies] 1
+            r config resetstat
+            assert_match {} [errorstat ERR]
+        }
+
+        test {errorstats: rejected call within MULTI/EXEC} {
+            r config resetstat
+            assert_equal [s total_error_replies] 0
+            assert_match {} [errorstat ERR]
+            r multi
+            catch {r set} e
+            assert_match {ERR wrong number of arguments*} $e
+            catch {r exec} e
+            assert_match {EXECABORT*} $e
+            assert_match {*count=1*} [errorstat ERR]
+            assert_equal [s total_error_replies] 1
+            assert_match {*calls=0,*,rejected_calls=1,failed_calls=0} [cmdstat set]
+            assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} [cmdstat multi]
+            assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} [cmdstat exec]
+            assert_equal [s total_error_replies] 1
+            r config resetstat
+            assert_match {} [errorstat ERR]
+        }
+
+        test {errorstats: rejected call due to wrong arity} {
+            r config resetstat
+            assert_equal [s total_error_replies] 0
+            assert_match {} [errorstat ERR]
+            catch {r set k} e
+            assert_match {ERR wrong number of arguments*} $e
+            assert_match {*count=1*} [errorstat ERR]
+            assert_match {*calls=0,*,rejected_calls=1,failed_calls=0} [cmdstat set]
+            # ensure that after a rejected command, valid ones are counted properly
+            r set k1 v1
+            r set k2 v2
+            assert_match {calls=2,*,rejected_calls=1,failed_calls=0} [cmdstat set]
+            assert_equal [s total_error_replies] 1
+        }
+
+        test {errorstats: rejected call by OOM error} {
+            r config resetstat
+            assert_equal [s total_error_replies] 0
+            assert_match {} [errorstat OOM]
+            r config set maxmemory 1
+            catch {r set a b} e
+            assert_match {OOM*} $e
+            assert_match {*count=1*} [errorstat OOM]
+            assert_match {*calls=0,*,rejected_calls=1,failed_calls=0} [cmdstat set]
+            assert_equal [s total_error_replies] 1
+            r config resetstat
+            assert_match {} [errorstat OOM]
+        }
+
+        test {errorstats: rejected call by authorization error} {
+            r config resetstat
+            assert_equal [s total_error_replies] 0
+            assert_match {} [errorstat NOPERM]
+            r ACL SETUSER alice on >p1pp0 ~cached:* +get +info +config
+            r auth alice p1pp0
+            catch {r set a b} e
+            assert_match {NOPERM*} $e
+            assert_match {*count=1*} [errorstat NOPERM]
+            assert_match {*calls=0,*,rejected_calls=1,failed_calls=0} [cmdstat set]
+            assert_equal [s total_error_replies] 1
+            r config resetstat
+            assert_match {} [errorstat NOPERM]
+        }
+    }
+}

From 152b5d46c4a76f2d16031ef794092bfc8d322f8a Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 1 Jan 2021 10:23:30 +0200
Subject: [PATCH 0932/1098] Crash log would crash half way on commands with no
 arguments (#8260)

The crash log attempts to print the current client info, and when it
does that it attempts to check if the first argument happens to be a key
but it did so for commands with no arguments too, which caused the crash
log to crash half way and not reach its end.
---
 src/debug.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/debug.c b/src/debug.c
index f95aaba0d..de6e8c13a 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -1569,7 +1569,7 @@ void logCurrentClient(void) {
     }
     /* Check if the first argument, usually a key, is found inside the
      * selected DB, and if so print info about the associated object. */
-    if (cc->argc >= 1) {
+    if (cc->argc > 1) {
         robj *val, *key;
         dictEntry *de;
 

From 71fbe6e800a10dabc81b1aec96bcca3b74af7bf4 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sat, 2 Jan 2021 08:37:19 +0200
Subject: [PATCH 0933/1098] Fix leak in new errorstats commit, and a flaky test
 (#8278)

---
 src/server.c                    |  2 +-
 tests/cluster/tests/18-info.tcl | 23 ++++++++++++++++-------
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/src/server.c b/src/server.c
index dbcaa767d..49ce319c4 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3301,7 +3301,7 @@ void resetCommandTableStats(void) {
 }
 
 void resetErrorTableStats(void) {
-    raxFree(server.errors);
+    raxFreeWithCallback(server.errors, zfree);
     server.errors = raxNew();
 }
 
diff --git a/tests/cluster/tests/18-info.tcl b/tests/cluster/tests/18-info.tcl
index 83dbf833f..978d9d1da 100644
--- a/tests/cluster/tests/18-info.tcl
+++ b/tests/cluster/tests/18-info.tcl
@@ -26,11 +26,20 @@ test "errorstats: rejected call due to MOVED Redirection" {
     $primary2 config resetstat
     assert_match {} [errorstat $primary1 MOVED]
     assert_match {} [errorstat $primary2 MOVED]
-    # we know that the primary 2 will have a MOVED reply
-    catch {$primary1 set key{0x0000} b} replyP1
-    catch {$primary2 set key{0x0000} b} replyP2
-    assert_match {OK} $replyP1
-    assert_match {} [errorstat $primary1 MOVED]
-    assert_match {*count=1*} [errorstat $primary2 MOVED]
-    assert_match {*calls=0,*,rejected_calls=1,failed_calls=0} [cmdstat $primary2 set]
+    # we know that one will have a MOVED reply and one will succeed
+    catch {$primary1 set key b} replyP1
+    catch {$primary2 set key b} replyP2
+    # sort servers so we know which one failed
+    if {$replyP1 eq {OK}} {
+        assert_match {MOVED*} $replyP2
+        set pok $primary1
+        set perr $primary2
+    } else {
+        assert_match {MOVED*} $replyP1
+        set pok $primary2
+        set perr $primary1
+    }
+    assert_match {} [errorstat $pok MOVED]
+    assert_match {*count=1*} [errorstat $perr MOVED]
+    assert_match {*calls=0,*,rejected_calls=1,failed_calls=0} [cmdstat $perr set]
 }

From 41b2ed2bbc0671e43101feecc48cac26a5e312cb Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 3 Jan 2021 11:56:26 +0200
Subject: [PATCH 0934/1098] fix crash in redis-cli after making cluster backup
 (#8267)

getRDB is "designed" to work in two modes: one for redis-cli --rdb and
one for redis-cli --cluster backup.
in the later case it uses the hiredis connection from the cluster nodes
and it used to free it without nullifying the context, so a later
attempt to free the context would crash.

I suppose the reason it seems to want to free the hiredis context ASAP
is that it wants to disconnect the replica link, so that replication
buffers will not be accumulated.
---
 src/redis-cli.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index ff29de748..31d2360c9 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -7132,7 +7132,9 @@ static void getRDB(clusterManagerNode *node) {
     } else {
         fprintf(stderr,"Transfer finished with success.\n");
     }
-    redisFree(s); /* Close the file descriptor ASAP as fsync() may take time. */
+    redisFree(s); /* Close the connection ASAP as fsync() may take time. */
+    if (node)
+        node->context = NULL;
     fsync(fd);
     close(fd);
     fprintf(stderr,"Transfer finished with success.\n");

From 7896debe6c16ec212c2b8ce169227988ccd9a2d8 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 3 Jan 2021 16:09:29 +0200
Subject: [PATCH 0935/1098] Fix rare assertion as a result of: active defrag
 while loading (#8281)

In #7726 (part of 6.2), we added a mechanism for whileBlockedCron, this
mechanism has an assertion to make sure the timestamp in
whileBlockedCron was always set correctly before the blocking operation
starts.

I now found (thanks to our CI) two bugs in that area:
1) CONFIG RESETSTAT (if it was allowed during loading) would have
   cleared this var
2) the call stopLoading (which calls whileBlockedCron) was made too
   early, while the rio is still in use, in which case the update_cksum
   (rdbLoadProgressCallback) may still be called and whileBlockedCron
   can assert.
---
 src/replication.c | 4 +++-
 src/server.c      | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/replication.c b/src/replication.c
index 520c43fa4..20eb83e72 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -1690,7 +1690,6 @@ void readSyncBulkPayload(connection *conn) {
              * gets promoted. */
             return;
         }
-        stopLoading(1);
 
         /* RDB loading succeeded if we reach this point. */
         if (server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB) {
@@ -1705,6 +1704,7 @@ void readSyncBulkPayload(connection *conn) {
             if (!rioRead(&rdb,buf,CONFIG_RUN_ID_SIZE) ||
                 memcmp(buf,eofmark,CONFIG_RUN_ID_SIZE) != 0)
             {
+                stopLoading(0);
                 serverLog(LL_WARNING,"Replication stream EOF marker is broken");
                 cancelReplicationHandshake(1);
                 rioFreeConn(&rdb, NULL);
@@ -1712,6 +1712,8 @@ void readSyncBulkPayload(connection *conn) {
             }
         }
 
+        stopLoading(1);
+
         /* Cleanup and restore the socket to the original state to continue
          * with the normal replication. */
         rioFreeConn(&rdb, NULL);
diff --git a/src/server.c b/src/server.c
index 49ce319c4..7ecfa8d1b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2955,7 +2955,6 @@ void resetServerStats(void) {
     server.stat_total_error_replies = 0;
     server.stat_dump_payload_sanitizations = 0;
     server.aof_delayed_fsync = 0;
-    server.blocked_last_cron = 0;
 }
 
 /* Make the thread killable at any time, so that kill threads functions
@@ -3004,6 +3003,7 @@ void initServer(void) {
     server.clients_paused = 0;
     server.events_processed_while_blocked = 0;
     server.system_memory_size = zmalloc_get_memory_size();
+    server.blocked_last_cron = 0;
 
     if ((server.tls_port || server.tls_replication || server.tls_cluster)
                 && tlsConfigure(&server.tls_ctx_config) == C_ERR) {

From 91690a29202b8ff3fbc9305273463fd5a575e572 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 3 Jan 2021 16:14:52 +0200
Subject: [PATCH 0936/1098] Fix uninitialized variable in new errorstats commit
 (#8280)

---
 src/module.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/module.c b/src/module.c
index 913b4de5d..abad38e78 100644
--- a/src/module.c
+++ b/src/module.c
@@ -851,6 +851,8 @@ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc c
     cp->rediscmd->keystep = keystep;
     cp->rediscmd->microseconds = 0;
     cp->rediscmd->calls = 0;
+    cp->rediscmd->rejected_calls = 0;
+    cp->rediscmd->failed_calls = 0;
     dictAdd(server.commands,sdsdup(cmdname),cp->rediscmd);
     dictAdd(server.orig_commands,sdsdup(cmdname),cp->rediscmd);
     cp->rediscmd->id = ACLGetCommandID(cmdname); /* ID used for ACL. */

From 33fb6170531435b891faf0ed7b21a95ac65dd0a1 Mon Sep 17 00:00:00 2001
From: kukey <wei.kukey@gmail.com>
Date: Sun, 3 Jan 2021 23:13:37 +0800
Subject: [PATCH 0937/1098] GEOADD - add [CH] [NX|XX] options (#8227)

New command flags similar to what SADD already has.

Co-authored-by: huangwei03 <huangwei03@kuaishou.com>
Co-authored-by: Itamar Haber <itamar@redislabs.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/geo.c          | 42 ++++++++++++++++++++++++++++--------------
 tests/unit/geo.tcl | 43 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+), 14 deletions(-)

diff --git a/src/geo.c b/src/geo.c
index 417bf897e..e9c8bf5cb 100644
--- a/src/geo.c
+++ b/src/geo.c
@@ -428,31 +428,45 @@ static int sort_gp_desc(const void *a, const void *b) {
  * Commands
  * ==================================================================== */
 
-/* GEOADD key long lat name [long2 lat2 name2 ... longN latN nameN] */
+/* GEOADD key [CH] [NX|XX] long lat name [long2 lat2 name2 ... longN latN nameN] */
 void geoaddCommand(client *c) {
-    /* Check arguments number for sanity. */
-    if ((c->argc - 2) % 3 != 0) {
+    int xx = 0, nx = 0, longidx = 2;
+    int i;
+
+    /* Parse options. At the end 'longidx' is set to the argument position
+     * of the longitude of the first element. */
+    while (longidx < c->argc) {
+        char *opt = c->argv[longidx]->ptr;
+        if (!strcasecmp(opt,"nx")) nx = 1;
+        else if (!strcasecmp(opt,"xx")) xx = 1;
+        else if (!strcasecmp(opt,"ch")) {}
+        else break;
+        longidx++;
+    }
+
+    if ((c->argc - longidx) % 3 || (xx && nx)) {
         /* Need an odd number of arguments if we got this far... */
-        addReplyError(c, "syntax error. Try GEOADD key [x1] [y1] [name1] "
-                         "[x2] [y2] [name2] ... ");
+            addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
 
-    int elements = (c->argc - 2) / 3;
-    int argc = 2+elements*2; /* ZADD key score ele ... */
+    /* Set up the vector for calling ZADD. */
+    int elements = (c->argc - longidx) / 3;
+    int argc = longidx+elements*2; /* ZADD key [CH] [NX|XX] score ele ... */
     robj **argv = zcalloc(argc*sizeof(robj*));
     argv[0] = createRawStringObject("zadd",4);
-    argv[1] = c->argv[1]; /* key */
-    incrRefCount(argv[1]);
+    for (i = 1; i < longidx; i++) {
+        argv[i] = c->argv[i];
+        incrRefCount(argv[i]);
+    }
 
     /* Create the argument vector to call ZADD in order to add all
      * the score,value pairs to the requested zset, where score is actually
      * an encoded version of lat,long. */
-    int i;
     for (i = 0; i < elements; i++) {
         double xy[2];
 
-        if (extractLongLatOrReply(c, (c->argv+2)+(i*3),xy) == C_ERR) {
+        if (extractLongLatOrReply(c, (c->argv+longidx)+(i*3),xy) == C_ERR) {
             for (i = 0; i < argc; i++)
                 if (argv[i]) decrRefCount(argv[i]);
             zfree(argv);
@@ -464,9 +478,9 @@ void geoaddCommand(client *c) {
         geohashEncodeWGS84(xy[0], xy[1], GEO_STEP_MAX, &hash);
         GeoHashFix52Bits bits = geohashAlign52Bits(hash);
         robj *score = createObject(OBJ_STRING, sdsfromlonglong(bits));
-        robj *val = c->argv[2 + i * 3 + 2];
-        argv[2+i*2] = score;
-        argv[3+i*2] = val;
+        robj *val = c->argv[longidx + i * 3 + 2];
+        argv[longidx+i*2] = score;
+        argv[longidx+1+i*2] = val;
         incrRefCount(val);
     }
 
diff --git a/tests/unit/geo.tcl b/tests/unit/geo.tcl
index 119d2bd64..6c1f2ef11 100644
--- a/tests/unit/geo.tcl
+++ b/tests/unit/geo.tcl
@@ -74,6 +74,49 @@ start_server {tags {"geo"}} {
         r geoadd nyc -73.9454966 40.747533 "lic market"
     } {0}
 
+    test {GEOADD update with CH option} {
+        assert_equal 1 [r geoadd nyc CH 40.747533 -73.9454966 "lic market"]
+        lassign [lindex [r geopos nyc "lic market"] 0] x1 y1
+        assert {abs($x1) - 40.747 < 0.001}
+        assert {abs($y1) - 73.945 < 0.001}
+    } {}
+
+    test {GEOADD update with NX option} {
+        assert_equal 0 [r geoadd nyc NX -73.9454966 40.747533 "lic market"]
+        lassign [lindex [r geopos nyc "lic market"] 0] x1 y1
+        assert {abs($x1) - 40.747 < 0.001}
+        assert {abs($y1) - 73.945 < 0.001}
+    } {}
+
+    test {GEOADD update with XX option} {
+        assert_equal 0 [r geoadd nyc XX -83.9454966 40.747533 "lic market"]
+        lassign [lindex [r geopos nyc "lic market"] 0] x1 y1
+        assert {abs($x1) - 83.945 < 0.001}
+        assert {abs($y1) - 40.747 < 0.001}
+    } {}
+
+    test {GEOADD update with CH NX option} {
+        r geoadd nyc CH NX -73.9454966 40.747533 "lic market"
+    } {0}
+
+    test {GEOADD update with CH XX option} {
+        r geoadd nyc CH XX -73.9454966 40.747533 "lic market"
+    } {1}
+
+    test {GEOADD update with XX NX option will return syntax error} {
+        catch {
+            r geoadd nyc xx nx -73.9454966 40.747533 "lic market"
+        } err
+        set err
+    } {ERR*syntax*}
+
+    test {GEOADD update with invalid option} {
+        catch {
+            r geoadd nyc ch xx foo -73.9454966 40.747533 "lic market"
+        } err
+        set err
+    } {ERR*syntax*}
+
     test {GEOADD invalid coordinates} {
         catch {
             r geoadd nyc -73.9454966 40.747533 "lic market" \

From 08ad6abd04c5aafe5471fa754000e512ae6b0f05 Mon Sep 17 00:00:00 2001
From: huangzhw <huang_zhw@126.com>
Date: Mon, 4 Jan 2021 16:28:47 +0800
Subject: [PATCH 0938/1098] sort Command lookupKeyRead and lookupKeyWrite are
 used on the opposite (#8283)

This is a recent problem, introduced by 7471743 (redis 6.0)

The implications are:
The sole difference between LookupKeyRead and LookupKeyWrite is for command
executed on a replica, which are not received from its master client. (for the master,
and for the master client on the replica, these two functions behave the same)!

Since SORT is a write command, this bug only implicates a writable-replica.
And these are its implications:

- SORT STORE will behave as it did before the above mentioned commit (like before
  redis 6.0). on a writable-replica an already logically expired the key would have
  appeared missing. (store dest key would be deleted, instead of being populated
  with the data from the already logically expired key)
- SORT (the non store variant, which in theory could have been executed on
  read-only-replica if it weren't for the write flag), will (in redis 6.0) have a new bug
  and return the data from the already logically expired key instead of empty response.
---
 src/sort.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sort.c b/src/sort.c
index 44637720b..3b67cc639 100644
--- a/src/sort.c
+++ b/src/sort.c
@@ -270,7 +270,7 @@ void sortCommand(client *c) {
     }
 
     /* Lookup the key to sort. It must be of the right types */
-    if (storekey)
+    if (!storekey)
         sortval = lookupKeyRead(c->db,c->argv[1]);
     else
         sortval = lookupKeyWrite(c->db,c->argv[1]);

From ecd53518700f5b2be19c18e7a19f87c9f25a7ff5 Mon Sep 17 00:00:00 2001
From: "Meir Shpilraien (Spielrein)" <meir@redislabs.com>
Date: Mon, 4 Jan 2021 13:42:17 +0200
Subject: [PATCH 0939/1098] Fix assertion on loading AOF with timed out script.
 (#8284)

If AOF file contains a long Lua script that timed out, then the `evalCommand` calls
`blockingOperationEnds` which sets `server.blocked_last_cron` to 0. later on,
the AOF `whileBlockedCron` function asserts that this value is not 0.

The fix allows nesting call to `blockingOperationStarts` and `blockingOperationEnds`.

The issue was first introduce in this commit: 9ef8d2f67 (Redis 6.2 RC1)
---
 src/server.c | 11 ++++++++---
 src/server.h |  1 +
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/server.c b/src/server.c
index 7ecfa8d1b..1fee70789 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2187,12 +2187,16 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
 
 
 void blockingOperationStarts() {
-    updateCachedTime(0);
-    server.blocked_last_cron = server.mstime;
+    if(!server.blocking_op_nesting++){
+        updateCachedTime(0);
+        server.blocked_last_cron = server.mstime;
+    }
 }
 
 void blockingOperationEnds() {
-    server.blocked_last_cron = 0;
+    if(!(--server.blocking_op_nesting)){
+        server.blocked_last_cron = 0;
+    }
 }
 
 /* This function fill in the role of serverCron during RDB or AOF loading, and
@@ -3004,6 +3008,7 @@ void initServer(void) {
     server.events_processed_while_blocked = 0;
     server.system_memory_size = zmalloc_get_memory_size();
     server.blocked_last_cron = 0;
+    server.blocking_op_nesting = 0;
 
     if ((server.tls_port || server.tls_replication || server.tls_cluster)
                 && tlsConfigure(&server.tls_ctx_config) == C_ERR) {
diff --git a/src/server.h b/src/server.h
index 5f6b21ec4..aaee5a35d 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1468,6 +1468,7 @@ struct redisServer {
     int daylight_active;        /* Currently in daylight saving time. */
     mstime_t mstime;            /* 'unixtime' in milliseconds. */
     ustime_t ustime;            /* 'unixtime' in microseconds. */
+    size_t blocking_op_nesting; /* Nesting level of blocking operation, used to reset blocked_last_cron. */
     long long blocked_last_cron; /* Indicate the mstime of the last time we did cron jobs from a blocking operation */
     /* Pubsub */
     dict *pubsub_channels;  /* Map channels to list of subscribed clients */

From 10f94b0ab12f9315939dcccf39d64b9388c0c7fa Mon Sep 17 00:00:00 2001
From: Yang Bodong <bodong.ybd@alibaba-inc.com>
Date: Mon, 4 Jan 2021 20:48:28 +0800
Subject: [PATCH 0940/1098] Swapdb should make transaction fail if there is any
 client watching keys (#8239)

This PR not only fixes the problem that swapdb does not make the
transaction fail, but also optimizes the FLUSHALL and FLUSHDB command to
set the CLIENT_DIRTY_CAS flag to avoid unnecessary traversal of clients.

FLUSHDB was changed to first iterate on all watched keys, and then on the
clients watching each key.
Instead of iterating though all clients, and for each iterate on watched keys.

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/db.c             | 20 ++++++++++++++++++--
 src/multi.c          | 43 ++++++++++++++++++++++++-------------------
 src/server.h         |  2 +-
 tests/unit/multi.tcl | 23 +++++++++++++++++++++++
 4 files changed, 66 insertions(+), 22 deletions(-)

diff --git a/src/db.c b/src/db.c
index 495300613..62ffda101 100644
--- a/src/db.c
+++ b/src/db.c
@@ -573,7 +573,18 @@ void signalModifiedKey(client *c, redisDb *db, robj *key) {
 }
 
 void signalFlushedDb(int dbid, int async) {
-    touchWatchedKeysOnFlush(dbid);
+    int startdb, enddb;
+    if (dbid == -1) {
+        startdb = 0;
+        enddb = server.dbnum-1;
+    } else {
+        startdb = enddb = dbid;
+    }
+
+    for (int j = startdb; j <= enddb; j++) {
+        touchAllWatchedKeysInDb(&server.db[j], NULL);
+    }
+
     trackingInvalidateKeysOnFlush(async);
 }
 
@@ -1331,9 +1342,14 @@ int dbSwapDatabases(long id1, long id2) {
      * However normally we only do this check for efficiency reasons
      * in dbAdd() when a list is created. So here we need to rescan
      * the list of clients blocked on lists and signal lists as ready
-     * if needed. */
+     * if needed.
+     *
+     * Also the swapdb should make transaction fail if there is any
+     * client watching keys */
     scanDatabaseForReadyLists(db1);
+    touchAllWatchedKeysInDb(db1, db2);
     scanDatabaseForReadyLists(db2);
+    touchAllWatchedKeysInDb(db2, db1);
     return C_OK;
 }
 
diff --git a/src/multi.c b/src/multi.c
index af0b0c612..d88c5f1b8 100644
--- a/src/multi.c
+++ b/src/multi.c
@@ -374,31 +374,36 @@ void touchWatchedKey(redisDb *db, robj *key) {
     }
 }
 
-/* On FLUSHDB or FLUSHALL all the watched keys that are present before the
- * flush but will be deleted as effect of the flushing operation should
- * be touched. "dbid" is the DB that's getting the flush. -1 if it is
- * a FLUSHALL operation (all the DBs flushed). */
-void touchWatchedKeysOnFlush(int dbid) {
-    listIter li1, li2;
+/* Set CLIENT_DIRTY_CAS to all clients of DB when DB is dirty.
+ * It may happen in the following situations:
+ * FLUSHDB, FLUSHALL, SWAPDB
+ *
+ * replaced_with: for SWAPDB, the WATCH should be invalidated if
+ * the key exists in either of them, and skipped only if it
+ * doesn't exist in both. */
+void touchAllWatchedKeysInDb(redisDb *emptied, redisDb *replaced_with) {
+    listIter li;
     listNode *ln;
+    dictEntry *de;
 
-    /* For every client, check all the waited keys */
-    listRewind(server.clients,&li1);
-    while((ln = listNext(&li1))) {
-        client *c = listNodeValue(ln);
-        listRewind(c->watched_keys,&li2);
-        while((ln = listNext(&li2))) {
-            watchedKey *wk = listNodeValue(ln);
+    if (dictSize(emptied->watched_keys) == 0) return;
 
-            /* For every watched key matching the specified DB, if the
-             * key exists, mark the client as dirty, as the key will be
-             * removed. */
-            if (dbid == -1 || wk->db->id == dbid) {
-                if (dictFind(wk->db->dict, wk->key->ptr) != NULL)
-                    c->flags |= CLIENT_DIRTY_CAS;
+    dictIterator *di = dictGetSafeIterator(emptied->watched_keys);
+    while((de = dictNext(di)) != NULL) {
+        robj *key = dictGetKey(de);
+        list *clients = dictGetVal(de);
+        if (!clients) continue;
+        listRewind(clients,&li);
+        while((ln = listNext(&li))) {
+            client *c = listNodeValue(ln);
+            if (dictFind(emptied->dict, key->ptr)) {
+                c->flags |= CLIENT_DIRTY_CAS;
+            } else if (replaced_with && dictFind(replaced_with->dict, key->ptr)) {
+                c->flags |= CLIENT_DIRTY_CAS;
             }
         }
     }
+    dictReleaseIterator(di);
 }
 
 void watchCommand(client *c) {
diff --git a/src/server.h b/src/server.h
index aaee5a35d..d79bf5b23 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1864,7 +1864,7 @@ void initClientMultiState(client *c);
 void freeClientMultiState(client *c);
 void queueMultiCommand(client *c);
 void touchWatchedKey(redisDb *db, robj *key);
-void touchWatchedKeysOnFlush(int dbid);
+void touchAllWatchedKeysInDb(redisDb *emptied, redisDb *replaced_with);
 void discardTransaction(client *c);
 void flagTransaction(client *c);
 void execCommandAbort(client *c, sds error);
diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl
index 25e417055..e22b6d43d 100644
--- a/tests/unit/multi.tcl
+++ b/tests/unit/multi.tcl
@@ -196,6 +196,29 @@ start_server {tags {"multi"}} {
         r exec
     } {PONG}
 
+    test {SWAPDB is able to touch the watched keys that exist} {
+        r flushall
+        r select 0
+        r set x 30
+        r watch x ;# make sure x (set to 30) doesn't change (SWAPDB will "delete" it)
+        r swapdb 0 1
+        r multi
+        r ping
+        r exec
+    } {}
+
+    test {SWAPDB is able to touch the watched keys that do not exist} {
+        r flushall
+        r select 1
+        r set x 30
+        r select 0
+        r watch x ;# make sure the key x (currently missing) doesn't change (SWAPDB will create it)
+        r swapdb 0 1
+        r multi
+        r ping
+        r exec
+    } {}
+
     test {WATCH is able to remember the DB a key belongs to} {
         r select 5
         r set x 30

From 9dcdc7e79a25968fcdfde09c7ca72a2012a1febf Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Mon, 4 Jan 2021 17:02:57 +0200
Subject: [PATCH 0941/1098] HELP subcommand, continued (#5531)

* man-like consistent long formatting
* Uppercases commands, subcommands and options
* Adds 'HELP' to HELP for all
* Lexicographical order
* Uses value notation and other .md likeness
* Moves const char *help to top
* Keeps it under 80 chars
* Misc help typos, consistent conjuctioning (i.e return and not returns)
* Uses addReplySubcommandSyntaxError(c) all over

Signed-off-by: Itamar Haber <itamar@redislabs.com>
---
 src/acl.c        |  44 ++++++++++++-------
 src/cluster.c    |  65 +++++++++++++++++----------
 src/config.c     |  13 ++++--
 src/debug.c      | 111 ++++++++++++++++++++++++++++++++++-------------
 src/latency.c    |  24 +++++-----
 src/module.c     |  15 +++++--
 src/networking.c |  70 ++++++++++++++++++++----------
 src/object.c     |  36 ++++++++++-----
 src/pubsub.c     |  10 +++--
 src/scripting.c  |  21 +++++----
 src/sentinel.c   |  58 +++++++++++++++++--------
 src/server.c     |  15 ++++---
 src/slowlog.c    |  14 +++---
 src/t_stream.c   |  55 ++++++++++++-----------
 14 files changed, 363 insertions(+), 188 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 606b61cb7..14d023cc3 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -174,15 +174,15 @@ sds ACLHashPassword(unsigned char *cleartext, size_t len) {
     return sdsnewlen(hex,HASH_PASSWORD_LEN);
 }
 
-/* Given a hash and the hash length, returns C_OK if it is a valid password 
+/* Given a hash and the hash length, returns C_OK if it is a valid password
  * hash, or C_ERR otherwise. */
 int ACLCheckPasswordHash(unsigned char *hash, int hashlen) {
     if (hashlen != HASH_PASSWORD_LEN) {
-        return C_ERR;      
+        return C_ERR;
     }
- 
+
     /* Password hashes can only be characters that represent
-     * hexadecimal values, which are numbers and lowercase 
+     * hexadecimal values, which are numbers and lowercase
      * characters 'a' through 'f'. */
     for(int i = 0; i < HASH_PASSWORD_LEN; i++) {
         char c = hash[i];
@@ -2184,18 +2184,30 @@ void aclCommand(client *c) {
         }
     } else if (c->argc == 2 && !strcasecmp(sub,"help")) {
         const char *help[] = {
-"LOAD                             -- Reload users from the ACL file.",
-"SAVE                             -- Save the current config to the ACL file.",
-"LIST                             -- Show user details in config file format.",
-"USERS                            -- List all the registered usernames.",
-"SETUSER <username> [attribs ...] -- Create or modify a user.",
-"GETUSER <username>               -- Get the user details.",
-"DELUSER <username> [...]         -- Delete a list of users.",
-"CAT                              -- List available categories.",
-"CAT <category>                   -- List commands inside category.",
-"GENPASS [<bits>]                 -- Generate a secure user password.",
-"WHOAMI                           -- Return the current connection username.",
-"LOG [<count> | RESET]            -- Show the ACL log entries.",
+"CAT [<category>]",
+"    List all commands that belong to <category>, or all command categories",
+"    when no category is specified.",
+"DELUSER <username> [<username> ...]",
+"    Delete a list of users.",
+"GETUSER <username>",
+"    Get the user's details.",
+"GENPASS [<bits>]",
+"    Generate a secure 256-bit user password. The optional `bits` argument can",
+"    be used to specify a different size.",
+"LIST",
+"    Show users details in config file format.",
+"LOAD",
+"    Reload users from the ACL file.",
+"LOG [<count> | RESET]",
+"    Show the ACL log entries.",
+"SAVE",
+"    Save the current config to the ACL file.",
+"SETUSER <username> <attribute> [<attribute> ...]",
+"    Create or modify a user with the specified attributes.",
+"USERS",
+"    List all the registered usernames.",
+"WHOAMI",
+"    Return the current connection username.",
 NULL
         };
         addReplyHelp(c,help);
diff --git a/src/cluster.c b/src/cluster.c
index 29bdb2109..d9f1a66d7 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -4357,28 +4357,49 @@ void clusterCommand(client *c) {
 
     if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
         const char *help[] = {
-"ADDSLOTS <slot> [slot ...] -- Assign slots to current node.",
-"BUMPEPOCH -- Advance the cluster config epoch.",
-"COUNT-failure-reports <node-id> -- Return number of failure reports for <node-id>.",
-"COUNTKEYSINSLOT <slot> - Return the number of keys in <slot>.",
-"DELSLOTS <slot> [slot ...] -- Delete slots information from current node.",
-"FAILOVER [force|takeover] -- Promote current replica node to being a master.",
-"FORGET <node-id> -- Remove a node from the cluster.",
-"GETKEYSINSLOT <slot> <count> -- Return key names stored by current node in a slot.",
-"FLUSHSLOTS -- Delete current node own slots information.",
-"INFO - Return information about the cluster.",
-"KEYSLOT <key> -- Return the hash slot for <key>.",
-"MEET <ip> <port> [bus-port] -- Connect nodes into a working cluster.",
-"MYID -- Return the node id.",
-"NODES -- Return cluster configuration seen by node. Output format:",
-"    <id> <ip:port> <flags> <master> <pings> <pongs> <epoch> <link> <slot> ... <slot>",
-"REPLICATE <node-id> -- Configure current node as replica to <node-id>.",
-"RESET [hard|soft] -- Reset current node (default: soft).",
-"SET-config-epoch <epoch> - Set config epoch of current node.",
-"SETSLOT <slot> (importing|migrating|stable|node <node-id>) -- Set slot state.",
-"REPLICAS <node-id> -- Return <node-id> replicas.",
-"SAVECONFIG - Force saving cluster configuration on disk.",
-"SLOTS -- Return information about slots range mappings. Each range is made of:",
+"ADDSLOTS <slot> [<slot> ...]",
+"    Assign slots to current node.",
+"BUMPEPOCH",
+"    Advance the cluster config epoch.",
+"COUNT-FAILURE-REPORTS <node-id>",
+"    Return number of failure reports for <node-id>.",
+"COUNTKEYSINSLOT <slot>",
+"    Return the number of keys in <slot>.",
+"DELSLOTS <slot> [<slot> ...]",
+"    Delete slots information from current node.",
+"FAILOVER [FORCE|TAKEOVER]",
+"    Promote current replica node to being a master.",
+"FORGET <node-id>",
+"    Remove a node from the cluster.",
+"GETKEYSINSLOT <slot> <count>",
+"    Return key names stored by current node in a slot.",
+"FLUSHSLOTS",
+"    Delete current node own slots information.",
+"INFO",
+"    Return information about the cluster.",
+"KEYSLOT <key>",
+"    Return the hash slot for <key>.",
+"MEET <ip> <port> [<bus-port>]",
+"    Connect nodes into a working cluster.",
+"MYID",
+"    Return the node id.",
+"NODES",
+"    Return cluster configuration seen by node. Output format:",
+"    <id> <ip:port> <flags> <master> <pings> <pongs> <epoch> <link> <slot> ...",
+"REPLICATE <node-id>",
+"    Configure current node as replica to <node-id>.",
+"RESET [HARD|SOFT]",
+"    Reset current node (default: soft).",
+"SET-CONFIG-EPOCH <epoch>",
+"    Set config epoch of current node.",
+"SETSLOT <slot> (IMPORTING|MIGRATING|STABLE|NODE <node-id>)",
+"    Set slot state.",
+"REPLICAS <node-id>",
+"    Return <node-id> replicas.",
+"SAVECONFIG",
+"    Force saving cluster configuration on disk.",
+"SLOTS",
+"    Return information about slots range mappings. Each range is made of:",
 "    start, end, master and replicas IP addresses, ports and ids",
 NULL
         };
diff --git a/src/config.c b/src/config.c
index 9d078bfe3..d804ff0ef 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2546,12 +2546,17 @@ void configCommand(client *c) {
 
     if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
         const char *help[] = {
-"GET <pattern> -- Return parameters matching the glob-like <pattern> and their values.",
-"SET <parameter> <value> -- Set parameter to value.",
-"RESETSTAT -- Reset statistics reported by INFO.",
-"REWRITE -- Rewrite the configuration file.",
+"GET <pattern>",
+"    Return parameters matching the glob-like <pattern> and their values.",
+"SET <directive> <value>",
+"    Set the configuration <directive> to <value>.",
+"RESETSTAT",
+"    Reset statistics reported by the INFO command.",
+"REWRITE",
+"    Rewrite the configuration file.",
 NULL
         };
+
         addReplyHelp(c, help);
     } else if (!strcasecmp(c->argv[1]->ptr,"set") && c->argc == 4) {
         configSetCommand(c);
diff --git a/src/debug.c b/src/debug.c
index de6e8c13a..a725e5a30 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -381,39 +381,88 @@ void mallctl_string(client *c, robj **argv, int argc) {
 void debugCommand(client *c) {
     if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
         const char *help[] = {
-"ASSERT -- Crash by assertion failed.",
-"CHANGE-REPL-ID -- Change the replication IDs of the instance. Dangerous, should be used only for testing the replication subsystem.",
-"CRASH-AND-RECOVER <milliseconds> -- Hard crash and restart after <milliseconds> delay.",
-"DIGEST -- Output a hex signature representing the current DB content.",
-"DIGEST-VALUE <key-1> ... <key-N>-- Output a hex signature of the values of all the specified keys.",
-"DEBUG PROTOCOL [string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false]",
-"ERROR <string> -- Return a Redis protocol error with <string> as message. Useful for clients unit tests to simulate Redis errors.",
-"LOG <message> -- write message to the server log.",
-"LEAK <string> -- Create a memory leak of the input string.",
-"HTSTATS <dbid> -- Return hash table statistics of the specified Redis database.",
-"HTSTATS-KEY <key> -- Like htstats but for the hash table stored as key's value.",
-"LOADAOF -- Flush the AOF buffers on disk and reload the AOF in memory.",
-"LUA-ALWAYS-REPLICATE-COMMANDS <0|1> -- Setting it to 1 makes Lua replication defaulting to replicating single commands, without the script having to enable effects replication.",
-"OBJECT <key> -- Show low level info about key and associated value.",
-"OOM -- Crash the server simulating an out-of-memory error.",
-"PANIC -- Crash the server simulating a panic.",
-"POPULATE <count> [prefix] [size] -- Create <count> string keys named key:<num>. If a prefix is specified is used instead of the 'key' prefix.",
-"RELOAD [MERGE] [NOFLUSH] [NOSAVE] -- Save the RDB on disk and reload it back in memory. By default it will save the RDB file and load it back. With the NOFLUSH option the current database is not removed before loading the new one, but conflicts in keys will kill the server with an exception. When MERGE is used, conflicting keys will be loaded (the key in the loaded RDB file will win). When NOSAVE is used, the server will not save the current dataset in the RDB file before loading. Use DEBUG RELOAD NOSAVE when you want just to load the RDB file you placed in the Redis working directory in order to replace the current dataset in memory. Use DEBUG RELOAD NOSAVE NOFLUSH MERGE when you want to add what is in the current RDB file placed in the Redis current directory, with the current memory content. Use DEBUG RELOAD when you want to verify Redis is able to persist the current dataset in the RDB file, flush the memory content, and load it back.",
-"RESTART -- Graceful restart: save config, db, restart.",
-"SDSLEN <key> -- Show low level SDS string info representing key and value.",
-"SEGFAULT -- Crash the server with sigsegv.",
-"SET-ACTIVE-EXPIRE <0|1> -- Setting it to 0 disables expiring keys in background when they are not accessed (otherwise the Redis behavior). Setting it to 1 reenables back the default.",
-"SET-SKIP-CHECKSUM-VALIDATION <0|1> -- Enables or disables checksum checks for rdb or RESTORE payload.",
-"AOF-FLUSH-SLEEP <microsec> -- Server will sleep before flushing the AOF, this is used for testing",
-"SLEEP <seconds> -- Stop the server for <seconds>. Decimals allowed.",
-"STRUCTSIZE -- Return the size of different Redis core C structures.",
-"ZIPLIST <key> -- Show low level info about the ziplist encoding.",
-"STRINGMATCH-TEST -- Run a fuzz tester against the stringmatchlen() function.",
-"CONFIG-REWRITE-FORCE-ALL -- Like CONFIG REWRITE but writes all configuration options, including keywords not listed in original configuration file or default values.",
+"AOF-FLUSH-SLEEP <microsec>",
+"    Server will sleep before flushing the AOF, this is used for testing.",
+"ASSERT",
+"    Crash by assertion failed.",
+"CHANGE-REPL-ID"
+"    Change the replication IDs of the instance.",
+"    Dangerous: should be used only for testing the replication subsystem.",
+"CONFIG-REWRITE-FORCE-ALL",
+"    Like CONFIG REWRITE but writes all configuration options, including",
+"    keywords not listed in original configuration file or default values.",
+"CRASH-AND-RECOVER <milliseconds>",
+"    Hard crash and restart after a <milliseconds> delay.",
+"DIGEST",
+"    Output a hex signature representing the current DB content.",
+"DIGEST-VALUE <key> [<key> ...]",
+"    Output a hex signature of the values of all the specified keys.",
+"ERROR <string>",
+"    Return a Redis protocol error with <string> as message. Useful for clients",
+"    unit tests to simulate Redis errors.",
+"LOG <message>",
+"    Write <message> to the server log.",
+"HTSTATS <dbid>",
+"    Return hash table statistics of the specified Redis database.",
+"HTSTATS-KEY <key>",
+"    Like HTSTATS but for the hash table stored at <key>'s value.",
+"LOADAOF",
+"    Flush the AOF buffers on disk and reload the AOF in memory.",
+"LUA-ALWAYS-REPLICATE-COMMANDS <0|1>",
+"    Setting it to 1 makes Lua replication defaulting to replicating single",
+"    commands, without the script having to enable effects replication.",
 #ifdef USE_JEMALLOC
-"MALLCTL <key> [<val>] -- Get or set a malloc tunning integer.",
-"MALLCTL-STR <key> [<val>] -- Get or set a malloc tunning string.",
+"MALLCTL <key> [<val>]",
+"    Get or set a malloc tuning integer.",
+"MALLCTL-STR <key> [<val>]",
+"    Get or set a malloc tuning string.",
 #endif
+"OBJECT <key>",
+"    Show low level info about `key` and associated value.",
+"OOM",
+"    Crash the server simulating an out-of-memory error.",
+"PANIC",
+"    Crash the server simulating a panic.",
+"POPULATE <count> [<prefix>] [<size>]",
+"    Create <count> string keys named key:<num>. If <prefix> is specified then",
+"    it is used instead of the 'key' prefix.",
+"DEBUG PROTOCOL <type>",
+"    Reply with a test value of the specified type. <type> can be: string,",
+"    integer, double, bignum, null, array, set, map, attrib, push, verbatim,",
+"    true, false.",
+"RELOAD [option ...]",
+"    Save the RDB on disk and reload it back to memory. Valid <option> values:",
+"    * MERGE: conflicting keys will be loaded from RDB.",
+"    * NOFLUSH: the existing database will not be removed before load, but",
+"      conflicting keys will generate an exception and kill the server."
+"    * NOSAVE: the database will be loaded from an existing RDB file.",
+"    Examples:",
+"    * DEBUG RELOAD: verify that the server is able to persist, flsuh and reload",
+"      the database.",
+"    * DEBUG RELOAD NOSAVE: replace the current database with the contents of an",
+"      existing RDB file.",
+"    * DEBUG RELOAD NOSAVE NOFLUSH MERGE: add the contents of an existing RDB",
+"      file to the database.",
+"RESTART",
+"    Graceful restart: save config, db, restart.",
+"SDSLEN <key>",
+"    Show low level SDS string info representing `key` and value.",
+"SEGFAULT",
+"    Crash the server with sigsegv.",
+"SET-ACTIVE-EXPIRE <0|1>",
+"    Setting it to 0 disables expiring keys in background when they are not",
+"    accessed (otherwise the Redis behavior). Setting it to 1 reenables back the",
+"    default.",
+"SET-SKIP-CHECKSUM-VALIDATION <0|1>",
+"    Enables or disables checksum checks for RDB files and RESTORE's payload.",
+"SLEEP <seconds>",
+"    Stop the server for <seconds>. Decimals allowed.",
+"STRINGMATCH-TEST",
+"    Run a fuzz tester against the stringmatchlen() function.",
+"STRUCTSIZE",
+"    Return the size of different Redis core C structures.",
+"ZIPLIST <key>",
+"    Show low level info about the ziplist encoding of <key>.",
 NULL
         };
         addReplyHelp(c, help);
diff --git a/src/latency.c b/src/latency.c
index 535e48c2b..d447b2b5b 100644
--- a/src/latency.c
+++ b/src/latency.c
@@ -584,16 +584,6 @@ sds latencyCommandGenSparkeline(char *event, struct latencyTimeSeries *ts) {
  * LATENCY RESET: reset data of a specified event or all the data if no event provided.
  */
 void latencyCommand(client *c) {
-    const char *help[] = {
-"DOCTOR              -- Returns a human readable latency analysis report.",
-"GRAPH   <event>     -- Returns an ASCII latency graph for the event class.",
-"HISTORY <event>     -- Returns time-latency samples for the event class.",
-"LATEST              -- Returns the latest latency samples for all events.",
-"RESET   [event ...] -- Resets latency data of one or more event classes.",
-"                       (default: reset all data for all event classes)",
-"HELP                -- Prints this help.",
-NULL
-    };
     struct latencyTimeSeries *ts;
 
     if (!strcasecmp(c->argv[1]->ptr,"history") && c->argc == 3) {
@@ -639,6 +629,20 @@ NULL
             addReplyLongLong(c,resets);
         }
     } else if (!strcasecmp(c->argv[1]->ptr,"help") && c->argc == 2) {
+        const char *help[] = {
+"DOCTOR",
+"    Return a human readable latency analysis report.",
+"GRAPH <event>",
+"    Return an ASCII latency graph for the <event> class.",
+"HISTORY <event>",
+"    Return time-latency samples for the <event> class.",
+"LATEST",
+"    Return the latest latency samples for all events.",
+"RESET [<event> ...]",
+"    Reset latency data of one or more <event> classes.",
+"    (default: reset all data for all event classes)",
+NULL
+        };
         addReplyHelp(c, help);
     } else {
         addReplySubcommandSyntaxError(c);
diff --git a/src/module.c b/src/module.c
index abad38e78..56ff63869 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7936,14 +7936,21 @@ sds genModulesInfoString(sds info) {
 
 /* Redis MODULE command.
  *
- * MODULE LOAD <path> [args...] */
+ * MODULE LIST
+ * MODULE LOAD <path> [args...]
+ * MODULE UNLOAD <name>
+ */
 void moduleCommand(client *c) {
     char *subcmd = c->argv[1]->ptr;
+
     if (c->argc == 2 && !strcasecmp(subcmd,"help")) {
         const char *help[] = {
-"LIST -- Return a list of loaded modules.",
-"LOAD <path> [arg ...] -- Load a module library from <path>.",
-"UNLOAD <name> -- Unload a module.",
+"LIST",
+"    Return a list of loaded modules.",
+"LOAD <path> [<arg> ...]",
+"    Load a module library from <path>, passing to it any optional arguments.",
+"UNLOAD <name>",
+"    Unload a module.",
 NULL
         };
         addReplyHelp(c, help);
diff --git a/src/networking.c b/src/networking.c
index 7957cc82f..77b25d0b7 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -855,12 +855,16 @@ void addReplyHelp(client *c, const char **help) {
 
     sdstoupper(cmd);
     addReplyStatusFormat(c,
-        "%s <subcommand> arg arg ... arg. Subcommands are:",cmd);
+        "%s <subcommand> [<arg> [value] [opt] ...]. Subcommands are:",cmd);
     sdsfree(cmd);
 
     while (help[blen]) addReplyStatus(c,help[blen++]);
 
-    blen++;  /* Account for the header line(s). */
+    addReplyStatus(c,"HELP");
+    addReplyStatus(c,"    Prints this help.");
+
+    blen += 1;  /* Account for the header. */
+    blen += 2;  /* Account for the footer. */
     setDeferredArrayLen(c,blenp,blen);
 }
 
@@ -2387,27 +2391,45 @@ void clientCommand(client *c) {
 
     if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
         const char *help[] = {
-"ID                     -- Return the ID of the current connection.",
-"INFO                   -- Return information about the current client connection.",
-"GETNAME                -- Return the name of the current connection.",
-"KILL <ip:port>         -- Kill connection made from <ip:port>.",
-"KILL <option> <value> [option value ...] -- Kill connections. Options are:",
-"     ADDR <ip:port>                      -- Kill connection made from <ip:port>",
-"     LADDR <ip:port>                     -- Kill connection made to <ip:port>",
-"     TYPE (normal|master|replica|pubsub) -- Kill connections by type.",
-"     USER <username>   -- Kill connections authenticated with such user.",
-"     SKIPME (yes|no)   -- Skip killing current connection (default: yes).",
-"LIST [options ...]     -- Return information about client connections. Options:",
-"     TYPE (normal|master|replica|pubsub) -- Return clients of specified type.",
-"     ID id [id ...]                      -- Return clients of specified IDs only.",
-"PAUSE <timeout>        -- Suspend all Redis clients for <timeout> milliseconds.",
-"REPLY (on|off|skip)    -- Control the replies sent to the current connection.",
-"SETNAME <name>         -- Assign the name <name> to the current connection.",
-"UNBLOCK <clientid> [TIMEOUT|ERROR] -- Unblock the specified blocked client.",
-"TRACKING (on|off) [REDIRECT <id>] [BCAST] [PREFIX first] [PREFIX second] [OPTIN] [OPTOUT] [NOLOOP]... -- Enable client keys tracking for client side caching.",
-"CACHING  (yes|no)      -- Enable/Disable tracking of the keys for next command in OPTIN/OPTOUT mode.",
-"GETREDIR               -- Return the client ID we are redirecting to when tracking is enabled.",
-"TRACKINGINFO           -- Return information about current client's tracking status.",
+"CACHING (YES|NO)",
+"    Enable/disable tracking of the keys for next command in OPTIN/OPTOUT modes.",
+"GETREDIR",
+"    Return the client ID we are redirecting to when tracking is enabled.",
+"GETNAME",
+"    Return the name of the current connection.",
+"ID",
+"    Return the ID of the current connection.",
+"INFO",
+"    Return information about the current client connection.",
+"KILL <ip:port>",
+"    Kill connection made from <ip:port>.",
+"KILL <option> <value> [<option> <value> [...]]",
+"    Kill connections. Options are:",
+"    * ADDR <ip:port>",
+"      Kill connection made from <ip:port>",
+"    * TYPE (normal|master|replica|pubsub)",
+"      Kill connections by type.",
+"    * USER <username>",
+"      Kill connections authenticated by <username>.",
+"    * SKIPME (YES|NO)",
+"      Skip killing current connection (default: yes).",
+"LIST [options ...]",
+"    Return information about client connections. Options:",
+"    * TYPE (NORMAL|MASTER|REPLICA|PUBSUB)",
+"      Return clients of specified type.",
+"PAUSE <timeout>",
+"    Suspend all clients for <timout> milliseconds.",
+"REPLY (ON|OFF|SKIP)",
+"    Control the replies sent to the current connection.",
+"SETNAME <name>",
+"    Assign the name <name> to the current connection.",
+"UNBLOCK <clientid> [TIMEOUT|ERROR]",
+"    Unblock the specified blocked client.",
+"TRACKING (ON|OFF) [REDIRECT <id>] [BCAST] [PREFIX <prefix> [...]]",
+"         [OPTIN] [OPTOUT]",
+"    Control server assisted client side caching.",
+"TRACKINGINFO",
+"    Report tracking status for the current connection.",
 NULL
         };
         addReplyHelp(c, help);
@@ -2836,7 +2858,7 @@ NULL
             addReplyArrayLen(c,0);
         }
     } else {
-        addReplyErrorFormat(c, "Unknown subcommand or wrong number of arguments for '%s'. Try CLIENT HELP", (char*)c->argv[1]->ptr);
+        addReplySubcommandSyntaxError(c);
     }
 }
 
diff --git a/src/object.c b/src/object.c
index 583d13737..8573ef0b5 100644
--- a/src/object.c
+++ b/src/object.c
@@ -1035,7 +1035,7 @@ struct redisMemOverhead *getMemoryOverheadData(void) {
     mem += dictSize(server.repl_scriptcache_dict) * sizeof(dictEntry) +
         dictSlots(server.repl_scriptcache_dict) * sizeof(dictEntry*);
     if (listLength(server.repl_scriptcache_fifo) > 0) {
-        mem += listLength(server.repl_scriptcache_fifo) * (sizeof(listNode) + 
+        mem += listLength(server.repl_scriptcache_fifo) * (sizeof(listNode) +
             sdsZmallocSize(listNodeValue(listFirst(server.repl_scriptcache_fifo))));
     }
     mh->lua_caches = mem;
@@ -1256,10 +1256,18 @@ void objectCommand(client *c) {
 
     if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
         const char *help[] = {
-"ENCODING <key> -- Return the kind of internal representation used in order to store the value associated with a key.",
-"FREQ <key> -- Return the access frequency index of the key. The returned integer is proportional to the logarithm of the recent access frequency of the key.",
-"IDLETIME <key> -- Return the idle time of the key, that is the approximated number of seconds elapsed since the last access to the key.",
-"REFCOUNT <key> -- Return the number of references of the value associated with the specified key.",
+"ENCODING <key>",
+"    Return the kind of internal representation used in order to store the value",
+"    associated with a <key>.",
+"FREQ <key>",
+"    Return the access frequency index of the <key>. The returned integer is",
+"    proportional to the logarithm of the recent access frequency of the key.",
+"IDLETIME <key>",
+"    Return the idle time of the <key>, that is the approximated number of",
+"    seconds elapsed since the last access to the key.",
+"REFCOUNT <key>",
+"    Return the number of references of the value associated with the specified",
+"    <key>.",
 NULL
         };
         addReplyHelp(c, help);
@@ -1303,11 +1311,17 @@ NULL
 void memoryCommand(client *c) {
     if (!strcasecmp(c->argv[1]->ptr,"help") && c->argc == 2) {
         const char *help[] = {
-"DOCTOR - Return memory problems reports.",
-"MALLOC-STATS -- Return internal statistics report from the memory allocator.",
-"PURGE -- Attempt to purge dirty pages for reclamation by the allocator.",
-"STATS -- Return information about the memory usage of the server.",
-"USAGE <key> [SAMPLES <count>] -- Return memory in bytes used by <key> and its value. Nested values are sampled up to <count> times (default: 5).",
+"DOCTOR",
+"    Return memory problems reports.",
+"MALLOC-STATS"
+"    Return internal statistics report from the memory allocator.",
+"PURGE",
+"    Attempt to purge dirty pages for reclamation by the allocator.",
+"STATS",
+"    Return information about the memory usage of the server.",
+"USAGE <key> [SAMPLES <count>]",
+"    Return memory in bytes used by <key> and its value. Nested values are",
+"    sampled up to <count> times (default: 5).",
 NULL
         };
         addReplyHelp(c, help);
@@ -1452,6 +1466,6 @@ NULL
         else
             addReplyError(c, "Error purging dirty pages");
     } else {
-        addReplyErrorFormat(c, "Unknown subcommand or wrong number of arguments for '%s'. Try MEMORY HELP", (char*)c->argv[1]->ptr);
+        addReplySubcommandSyntaxError(c);
     }
 }
diff --git a/src/pubsub.c b/src/pubsub.c
index 9e41464bf..7355e10b9 100644
--- a/src/pubsub.c
+++ b/src/pubsub.c
@@ -455,9 +455,13 @@ void publishCommand(client *c) {
 void pubsubCommand(client *c) {
     if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
         const char *help[] = {
-"CHANNELS [<pattern>] -- Return the currently active channels matching a pattern (default: all).",
-"NUMPAT -- Return number of subscriptions to patterns.",
-"NUMSUB [channel-1 .. channel-N] -- Returns the number of subscribers for the specified channels (excluding patterns, default: none).",
+"CHANNELS [<pattern>]",
+"    Return the currently active channels matching a <pattern> (default: '*').",
+"NUMPAT",
+"    Return number of subscriptions to patterns.",
+"NUMSUB [<channel> ...]",
+"    Return the number of subscribers for the specified channels, excluding",
+"    pattern subscriptions(default: no channels).",
 NULL
         };
         addReplyHelp(c, help);
diff --git a/src/scripting.c b/src/scripting.c
index e75892046..6d0bf2349 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -691,11 +691,11 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) {
         if (getNodeByQuery(c,c->cmd,c->argv,c->argc,NULL,&error_code) !=
                            server.cluster->myself)
         {
-            if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) { 
+            if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) {
                 luaPushError(lua,
                     "Lua script attempted to execute a write command while the "
                     "cluster is down and readonly");
-            } else if (error_code == CLUSTER_REDIR_DOWN_STATE) { 
+            } else if (error_code == CLUSTER_REDIR_DOWN_STATE) {
                 luaPushError(lua,
                     "Lua script attempted to execute a command while the "
                     "cluster is down");
@@ -1707,11 +1707,16 @@ void evalShaCommand(client *c) {
 void scriptCommand(client *c) {
     if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
         const char *help[] = {
-"DEBUG (yes|sync|no) -- Set the debug mode for subsequent scripts executed.",
-"EXISTS <sha1> [<sha1> ...] -- Return information about the existence of the scripts in the script cache.",
-"FLUSH -- Flush the Lua scripts cache. Very dangerous on replicas.",
-"KILL -- Kill the currently executing Lua script.",
-"LOAD <script> -- Load a script into the scripts cache, without executing it.",
+"DEBUG (YES|SYNC|NO)",
+"    Set the debug mode for subsequent scripts executed.",
+"EXISTS <sha1> [<sha1> ...]",
+"    Return information about the existence of the scripts in the script cache.",
+"FLUSH",
+"    Flush the Lua scripts cache. Very dangerous on replicas.",
+"KILL",
+"    Kill the currently executing Lua script.",
+"LOAD <script>",
+"    Load a script into the scripts cache without executing it.",
 NULL
         };
         addReplyHelp(c, help);
@@ -1762,7 +1767,7 @@ NULL
             addReply(c,shared.ok);
             c->flags |= CLIENT_LUA_DEBUG_SYNC;
         } else {
-            addReplyError(c,"Use SCRIPT DEBUG yes/sync/no");
+            addReplyError(c,"Use SCRIPT DEBUG YES/SYNC/NO");
             return;
         }
     } else {
diff --git a/src/sentinel.c b/src/sentinel.c
index 1c1b4bf5b..02260feb7 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -3090,25 +3090,45 @@ int sentinelIsQuorumReachable(sentinelRedisInstance *master, int *usableptr) {
 void sentinelCommand(client *c) {
     if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
         const char *help[] = {
-"MASTERS -- Show a list of monitored masters and their state.",
-"MASTER <master-name> -- Show the state and info of the specified master.",
-"REPLICAS <master-name> -- Show a list of replicas for this master and their state.",
-"SENTINELS <master-name> -- Show a list of Sentinel instances for this master and their state.",
-"MYID -- Show Current Sentinel Id",
-"IS-MASTER-DOWN-BY-ADDR <ip> <port> <current-epoch> <runid> -- Check if the master specified by ip:port is down from current Sentinel's point of view.",
-"GET-MASTER-ADDR-BY-NAME <master-name> -- Return the ip and port number of the master with that name.",
-"RESET <pattern> -- Reset masters for specific master name matching this pattern.",
-"FAILOVER <master-name> -- Manually failover a master node without asking for agreement from other Sentinels",
-"PENDING-SCRIPTS -- Get pending scripts information.", 
-"MONITOR <name> <ip> <port> <quorum> -- Start monitoring a new master with the specified name, ip, port and quorum.",
-"FLUSHCONFIG -- Force Sentinel to rewrite its configuration on disk, including the current Sentinel state.",
-"REMOVE <master-name> -- Remove master from Sentinel's monitor list.",
-"CKQUORUM <master-name> -- Check if the current Sentinel configuration is able to reach the quorum needed to failover a master "
-"and the majority needed to authorize the failover.",
-"SET <master-name> <option> <value> -- Set configuration paramters for certain masters.",
-"INFO-CACHE <master-name> -- Return last cached INFO output from masters and all its replicas.",
-"SIMULATE-FAILURE (crash-after-election|crash-after-promotion|help) -- Simulate a Sentinel crash.",
-"HELP -- Prints this help.",
+"CKQUORUM <master-name>",
+"    Check if the current Sentinel configuration is able to reach the quorum",
+"    needed to failover a master and the majority needed to authorize the",
+"    failover.",
+"GET-MASTER-ADDR-BY-NAME <master-name>",
+"    Return the ip and port number of the master with that name.",
+"FAILOVER <master-name>",
+"    Manually failover a master node without asking for agreement from other",
+"    Sentinels",
+"FLUSHCONFIG",
+"    Force Sentinel to rewrite its configuration on disk, including the current",
+"    Sentinel state.",
+"INFO-CACHE <master-name>",
+"    Return last cached INFO output from masters and all its replicas.",
+"IS-MASTER-DOWN-BY-ADDR <ip> <port> <current-epoch> <runid>",
+"    Check if the master specified by ip:port is down from current Sentinel's",
+"    point of view.",
+"MASTER <master-name>",
+"    Show the state and info of the specified master.",
+"MASTERS",
+"    Show a list of monitored masters and their state.",
+"MONITOR <name> <ip> <port> <quorum>",
+"    Start monitoring a new master with the specified name, ip, port and quorum.",
+"MYID",
+"    Return the ID of the Sentinel instance.",
+"PENDING-SCRIPTS",
+"    Get pending scripts information.",
+"REMOVE <master-name>",
+"    Remove master from Sentinel's monitor list.",
+"REPLICAS <master-name>",
+"    Show a list of replicas for this master and their state.",
+"RESET <pattern>",
+"    Reset masters for specific master name matching this pattern.",
+"SENTINELS <master-name>",
+"    Show a list of Sentinel instances for this master and their state.",
+"SET <master-name> <option> <value>",
+"    Set configuration paramters for certain masters.",
+"SIMULATE-FAILURE (CRASH-AFTER-ELECTION|CRASH-AFTER-PROMOTION|HELP)",
+"    Simulate a Sentinel crash.",
 NULL
         };
         addReplyHelp(c, help);
diff --git a/src/server.c b/src/server.c
index 1fee70789..56f7484b1 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2233,6 +2233,7 @@ void whileBlockedCron() {
         activeDefragCycle();
 
         server.blocked_last_cron += hz_ms;
+
         /* Increment cronloop so that run_with_period works. */
         server.cronloops++;
     }
@@ -4248,10 +4249,14 @@ void commandCommand(client *c) {
 
     if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
         const char *help[] = {
-"(no subcommand) -- Return details about all Redis commands.",
-"COUNT -- Return the total number of commands in this Redis server.",
-"GETKEYS <full-command> -- Return the keys from a full Redis command.",
-"INFO [command-name ...] -- Return details about multiple Redis commands.",
+"(no subcommand)",
+"    Return details about all Redis commands.",
+"COUNT",
+"    Return the total number of commands in this Redis server.",
+"GETKEYS <full-command>",
+"    Return the keys from a full Redis command.",
+"INFO [<command-name> ...]",
+"    Return details about multiple Redis commands.",
 NULL
         };
         addReplyHelp(c, help);
@@ -5365,7 +5370,7 @@ void loadDataFromDisk(void) {
 void redisOutOfMemoryHandler(size_t allocation_size) {
     serverLog(LL_WARNING,"Out Of Memory allocating %zu bytes!",
         allocation_size);
-    serverPanic("Redis aborting for OUT OF MEMORY. Allocating %zu bytes!", 
+    serverPanic("Redis aborting for OUT OF MEMORY. Allocating %zu bytes!",
         allocation_size);
 }
 
diff --git a/src/slowlog.c b/src/slowlog.c
index 408456b14..b8c13f1cb 100644
--- a/src/slowlog.c
+++ b/src/slowlog.c
@@ -142,11 +142,15 @@ void slowlogReset(void) {
 void slowlogCommand(client *c) {
     if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
         const char *help[] = {
-"GET [count] -- Return top entries from the slowlog (default: 10)."
-"    Entries are made of:",
-"    id, timestamp, time in microseconds, arguments array, client IP and port, client name",
-"LEN -- Return the length of the slowlog.",
-"RESET -- Reset the slowlog.",
+"GET [<count>]",
+"    Return top <count> entries from the slowlog (default: 10). Entries are",
+"    made of:",
+"    id, timestamp, time in microseconds, arguments array, client IP and port,",
+"    client name",
+"LEN",
+"    Return the length of the slowlog.",
+"RESET",
+"    Reset the slowlog.",
 NULL
         };
         addReplyHelp(c, help);
diff --git a/src/t_stream.c b/src/t_stream.c
index 3ed9618fc..46918bae8 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1981,19 +1981,9 @@ uint64_t streamDelConsumer(streamCG *cg, sds name) {
 /* XGROUP CREATE <key> <groupname> <id or $> [MKSTREAM]
  * XGROUP SETID <key> <groupname> <id or $>
  * XGROUP DESTROY <key> <groupname>
- * CREATECONSUMER <key> <groupname> <consumer>
+ * XGROUP CREATECONSUMER <key> <groupname> <consumer>
  * XGROUP DELCONSUMER <key> <groupname> <consumername> */
 void xgroupCommand(client *c) {
-    const char *help[] = {
-"CREATE         <key> <groupname> <id or $> [opt] -- Create a new consumer group.",
-"               option MKSTREAM: create the empty stream if it does not exist.",
-"SETID          <key> <groupname> <id or $>  -- Set the current group ID.",
-"DESTROY        <key> <groupname>            -- Remove the specified group.",
-"CREATECONSUMER <key> <groupname> <consumer> -- Create new consumer in the specified group.",
-"DELCONSUMER    <key> <groupname> <consumer> -- Remove the specified consumer.",
-"HELP                                        -- Prints this help.",
-NULL
-    };
     stream *s = NULL;
     sds grpname = NULL;
     streamCG *cg = NULL;
@@ -2047,7 +2037,24 @@ NULL
     }
 
     /* Dispatch the different subcommands. */
-    if (!strcasecmp(opt,"CREATE") && (c->argc == 5 || c->argc == 6)) {
+    if (c->argc == 2 && !strcasecmp(opt,"HELP")) {
+        const char *help[] = {
+"CREATE <key> <groupname> <id|$> [option]",
+"    Create a new consumer group. Options are:",
+"    * MKSTREAM",
+"      Create the empty stream if it does not exist.",
+"CREATECONSUMER <key> <groupname> <consumer>",
+"    Create a new consumer in the specified group.",
+"DELCONSUMER <key> <groupname> <consumer>",
+"    Remove the specified consumer.",
+"DESTROY <key> <groupname>"
+"    Remove the specified group.",
+"SETID <key> <groupname> <id|$>",
+"    Set the current group ID.",
+NULL
+        };
+        addReplyHelp(c, help);
+    } else if (!strcasecmp(opt,"CREATE") && (c->argc == 5 || c->argc == 6)) {
         streamID id;
         if (!strcmp(c->argv[4]->ptr,"$")) {
             if (s) {
@@ -2119,8 +2126,6 @@ NULL
         server.dirty++;
         notifyKeyspaceEvent(NOTIFY_STREAM,"xgroup-delconsumer",
                             c->argv[2],c->db->id);
-    } else if (c->argc == 2 && !strcasecmp(opt,"HELP")) {
-        addReplyHelp(c, help);
     } else {
         addReplySubcommandSyntaxError(c);
     }
@@ -2971,27 +2976,25 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) {
  * XINFO STREAM <key> [FULL [COUNT <count>]]
  * XINFO HELP. */
 void xinfoCommand(client *c) {
-    const char *help[] = {
-"CONSUMERS <key> <groupname>         -- Show consumer groups of group <groupname>.",
-"GROUPS <key>                        -- Show the stream consumer groups.",
-"STREAM <key> [FULL [COUNT <count>]] -- Show information about the stream.",
-"                                       FULL will return the full state of the stream,",
-"                                            including all entries, groups, consumers and PELs.",
-"                                            It's possible to show only the first stream/PEL entries",
-"                                            by using the COUNT modifier (Default is 10)",
-"HELP                                -- Print this help.",
-NULL
-    };
     stream *s = NULL;
     char *opt;
     robj *key;
 
     /* HELP is special. Handle it ASAP. */
     if (!strcasecmp(c->argv[1]->ptr,"HELP")) {
+        const char *help[] = {
+"CONSUMERS <key> <groupname>",
+"    Show consumers of <groupname>.",
+"GROUPS <key>",
+"    Show the stream consumer groups.",
+"STREAM <key> [FULL [COUNT <count>]",
+"    Show information about the stream.",
+NULL
+        };
         addReplyHelp(c, help);
         return;
     } else if (c->argc < 3) {
-        addReplyError(c,"syntax error, try 'XINFO HELP'");
+        addReplySubcommandSyntaxError(c);
         return;
     }
 

From df9c213bb0b90c57867d25d272729d8492244582 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 4 Jan 2021 23:16:19 +0200
Subject: [PATCH 0942/1098] remove unused call to zmalloc_size in defrag.c
 (#8285)

---
 src/defrag.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/defrag.c b/src/defrag.c
index fc327c506..e189deddd 100644
--- a/src/defrag.c
+++ b/src/defrag.c
@@ -58,7 +58,6 @@ void* activeDefragAlloc(void *ptr) {
     void *newptr;
     if(!je_get_defrag_hint(ptr)) {
         server.stat_active_defrag_misses++;
-        size = zmalloc_size(ptr);
         return NULL;
     }
     /* move this allocation to a new allocation.

From 2017407b4d1d19a91af1e7c0b199f2c1775dbaf9 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 5 Jan 2021 08:29:20 +0200
Subject: [PATCH 0943/1098] Fix wrong order of key/value in Lua map response
 (#8266)

When a Lua script returns a map to redis (a feature which was added in
redis 6 together with RESP3), it would have returned the value first and
the key second.

If the client was using RESP2, it was getting them out of order, and if
the client was in RESP3, it was getting a map of value => key.
This was happening regardless of the Lua script using redis.setresp(3)
or not.

This also affects a case where the script was returning a map which it got
from from redis by doing something like: redis.setresp(3); return redis.call()

This fix is a breaking change for redis 6.0 users who happened to rely
on the wrong order (either ones that used redis.setresp(3), or ones that
returned a map explicitly).

This commit also includes other two changes in the tests:
1. The test suite now handles RESP3 maps as dicts rather than nested
   lists
2. Remove some redundant (duplicate) tests from tracking.tcl
---
 src/scripting.c          |  4 ++--
 tests/support/redis.tcl  | 11 +++++------
 tests/unit/scripting.tcl | 24 ++++++++++++++++++++++++
 tests/unit/tracking.tcl  | 19 +++++--------------
 4 files changed, 36 insertions(+), 22 deletions(-)

diff --git a/src/scripting.c b/src/scripting.c
index 6d0bf2349..75604e4d8 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -407,9 +407,9 @@ void luaReplyToRedisReply(client *c, lua_State *lua) {
             lua_pushnil(lua); /* Use nil to start iteration. */
             while (lua_next(lua,-2)) {
                 /* Stack now: table, key, value */
-                luaReplyToRedisReply(c, lua); /* Return value. */
-                lua_pushvalue(lua,-1);        /* Dup key before consuming. */
+                lua_pushvalue(lua,-2);        /* Dup key before consuming. */
                 luaReplyToRedisReply(c, lua); /* Return key. */
+                luaReplyToRedisReply(c, lua); /* Return value. */
                 /* Stack now: table, key. */
                 maplen++;
             }
diff --git a/tests/support/redis.tcl b/tests/support/redis.tcl
index 9eb5b94e2..8eca2ac32 100644
--- a/tests/support/redis.tcl
+++ b/tests/support/redis.tcl
@@ -214,20 +214,19 @@ proc ::redis::redis_multi_bulk_read {id fd} {
 proc ::redis::redis_read_map {id fd} {
     set count [redis_read_line $fd]
     if {$count == -1} return {}
-    set l {}
+    set d {}
     set err {}
     for {set i 0} {$i < $count} {incr i} {
         if {[catch {
-            set t {}
-            lappend t [redis_read_reply $id $fd] ; # key
-            lappend t [redis_read_reply $id $fd] ; # value
-            lappend l $t
+            set k [redis_read_reply $id $fd] ; # key
+            set v [redis_read_reply $id $fd] ; # value
+            dict set d $k $v
         } e] && $err eq {}} {
             set err $e
         }
     }
     if {$err ne {}} {return -code error $err}
-    return $l
+    return $d
 }
 
 proc ::redis::redis_read_line fd {
diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl
index 0da1eda9f..6fd152594 100644
--- a/tests/unit/scripting.tcl
+++ b/tests/unit/scripting.tcl
@@ -563,6 +563,30 @@ start_server {tags {"scripting"}} {
         } e
         set e
     } {*wrong number*}
+
+    test {Script with RESP3 map} {
+        set expected_dict [dict create field value]
+        set expected_list [list field value]
+
+        # Sanity test for RESP3 without scripts
+        r HELLO 3
+        r hset hash field value
+        set res [r hgetall hash]
+        assert_equal $res $expected_dict
+
+        # Test RESP3 client with script in both RESP2 and RESP3 modes
+        set res [r eval {redis.setresp(3); return redis.call('hgetall', KEYS[1])} 1 hash]
+        assert_equal $res $expected_dict
+        set res [r eval {redis.setresp(2); return redis.call('hgetall', KEYS[1])} 1 hash]
+        assert_equal $res $expected_list
+
+        # Test RESP2 client with script in both RESP2 and RESP3 modes
+        r HELLO 2
+        set res [r eval {redis.setresp(3); return redis.call('hgetall', KEYS[1])} 1 hash]
+        assert_equal $res $expected_list
+        set res [r eval {redis.setresp(2); return redis.call('hgetall', KEYS[1])} 1 hash]
+        assert_equal $res $expected_list
+    }
 }
 
 # Start a new server since the last test in this stanza will kill the
diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index 6f8b46c84..8ed2801e7 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -132,31 +132,22 @@ start_server {tags {"tracking"}} {
 
     test {HELLO 3 reply is correct} {
         set reply [r HELLO 3]
-        assert {[lindex $reply 2] eq {proto 3}}
+        assert_equal [dict get $reply proto] 3
     }
 
     test {HELLO without protover} {
         set reply [r HELLO 3]
-        assert {[lindex $reply 2] eq {proto 3}}
+        assert_equal [dict get $reply proto] 3
 
         set reply [r HELLO]
-        assert {[lindex $reply 2] eq {proto 3}}
-
-        set reply [r HELLO]
-        assert {[lindex $reply 2] eq {proto 3}}
+        assert_equal [dict get $reply proto] 3
 
         set reply [r HELLO 2]
-        assert {[lindex $reply 4] eq "proto"}
-        assert {[lindex $reply 5] eq 2}
+        assert_equal [dict get $reply proto] 2
 
         set reply [r HELLO]
-        assert {[lindex $reply 4] eq "proto"}
-        assert {[lindex $reply 5] eq 2}
+        assert_equal [dict get $reply proto] 2
 
-        set reply [r HELLO]
-        assert {[lindex $reply 4] eq "proto"}
-        assert {[lindex $reply 5] eq 2}
-        
         # restore RESP3 for next test
         r HELLO 3
     }

From 324070c8f6f63240629893c2ce9bcbeecf6e77fb Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 5 Jan 2021 09:15:10 +0200
Subject: [PATCH 0944/1098] Fix rdb checksum / crc64 on bigendian (#8270)

Turns out the RDB checksum in Redis 6.0 on bigendian is broken.
It always returned 0, so the RDB files are generated as if checksum is
disabled, and will be loaded ok on littleendian, and on bigendian.
But it'll not be able to load RDB files generated on littleendian or older versions.

Similarly DUMP and RESTORE will work on the same version (0==0),
but will be unable to exchange dump payloads with littleendian or old versions.
---
 src/crcspeed.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/crcspeed.c b/src/crcspeed.c
index d4955bfc9..67cb8fd9f 100644
--- a/src/crcspeed.c
+++ b/src/crcspeed.c
@@ -35,7 +35,8 @@ void crcspeed64little_init(crcfn64 crcfn, uint64_t table[8][256]) {
 
     /* generate CRCs for all single byte sequences */
     for (int n = 0; n < 256; n++) {
-        table[0][n] = crcfn(0, &n, 1);
+        unsigned char v = n;
+        table[0][n] = crcfn(0, &v, 1);
     }
 
     /* generate nested CRC table for future slice-by-8 lookup */

From 595ecd5f4be39eeec71fb07f687b2d6b7cf5c20c Mon Sep 17 00:00:00 2001
From: huangzhw <huang_zhw@126.com>
Date: Wed, 6 Jan 2021 00:41:53 +0800
Subject: [PATCH 0945/1098] sdscatfmt call sdsMakeRoomFor, asked for more space
 than intended (#8286)

instead of asking for the extra new space it wanted, it asked to grow the
string by the size it already has too.
i.e. a string of 1000 bytes, needing to grow by 10 bytes, would have been
asking for an **additional** 1010 bytes.
---
 src/sds.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sds.c b/src/sds.c
index 5efb36de6..f16114471 100644
--- a/src/sds.c
+++ b/src/sds.c
@@ -642,7 +642,7 @@ sds sdscatfmt(sds s, char const *fmt, ...) {
     /* To avoid continuous reallocations, let's start with a buffer that
      * can hold at least two times the format string itself. It's not the
      * best heuristic but seems to work in practice. */
-    s = sdsMakeRoomFor(s, initlen + strlen(fmt)*2);
+    s = sdsMakeRoomFor(s, strlen(fmt)*2);
     va_start(ap,fmt);
     f = fmt;    /* Next format specifier byte to process. */
     i = initlen; /* Position of the next byte to write to dest str. */

From 714e103ac317bfa179b0a132c0f78d4ddc84a435 Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Wed, 6 Jan 2021 10:34:27 +0200
Subject: [PATCH 0946/1098] Add XAUTOCLAIM (#7973)

New command: XAUTOCLAIM <key> <group> <consumer> <min-idle-time> <start> [COUNT <count>] [JUSTID]

The purpose is to claim entries from a stale consumer without the usual
XPENDING+XCLAIM combo which takes two round trips.

The syntax for XAUTOCLAIM is similar to scan: A cursor is returned (streamID)
by each call and should be used as start for the next call. 0-0 means the scan is complete.

This PR extends the deferred reply mechanism for any bulk string (not just counts)

This PR carries some unrelated test code changes:
- Renames the term "client" into "consumer" in the stream-cgroups test
- And also changes DEBUG SLEEP into "after"

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/networking.c                   |  44 +++++--
 src/server.c                       |   4 +
 src/server.h                       |   2 +
 src/t_stream.c                     | 159 +++++++++++++++++++++++
 tests/unit/type/stream-cgroups.tcl | 197 ++++++++++++++++++++++-------
 5 files changed, 347 insertions(+), 59 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 77b25d0b7..29e5605d9 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -559,13 +559,9 @@ void *addReplyDeferredLen(client *c) {
     return listLast(c->reply);
 }
 
-/* Populate the length object and try gluing it to the next chunk. */
-void setDeferredAggregateLen(client *c, void *node, long length, char prefix) {
-    serverAssert(length >= 0);
+void setDeferredReply(client *c, void *node, const char *s, size_t length) {
     listNode *ln = (listNode*)node;
     clientReplyBlock *next;
-    char lenstr[128];
-    size_t lenstr_len = sprintf(lenstr, "%c%ld\r\n", prefix, length);
 
     /* Abort when *node is NULL: when the client should not accept writes
      * we return NULL in addReplyDeferredLen() */
@@ -583,25 +579,39 @@ void setDeferredAggregateLen(client *c, void *node, long length, char prefix) {
      * - It has enough room already allocated
      * - And not too large (avoid large memmove) */
     if (ln->next != NULL && (next = listNodeValue(ln->next)) &&
-        next->size - next->used >= lenstr_len &&
-        next->used < PROTO_REPLY_CHUNK_BYTES * 4) {
-        memmove(next->buf + lenstr_len, next->buf, next->used);
-        memcpy(next->buf, lenstr, lenstr_len);
-        next->used += lenstr_len;
+        next->size - next->used >= length &&
+        next->used < PROTO_REPLY_CHUNK_BYTES * 4)
+    {
+        memmove(next->buf + length, next->buf, next->used);
+        memcpy(next->buf, s, length);
+        next->used += length;
         listDelNode(c->reply,ln);
     } else {
         /* Create a new node */
-        clientReplyBlock *buf = zmalloc(lenstr_len + sizeof(clientReplyBlock));
+        clientReplyBlock *buf = zmalloc(length + sizeof(clientReplyBlock));
         /* Take over the allocation's internal fragmentation */
         buf->size = zmalloc_usable_size(buf) - sizeof(clientReplyBlock);
-        buf->used = lenstr_len;
-        memcpy(buf->buf, lenstr, lenstr_len);
+        buf->used = length;
+        memcpy(buf->buf, s, length);
         listNodeValue(ln) = buf;
         c->reply_bytes += buf->size;
     }
     asyncCloseClientOnOutputBufferLimitReached(c);
 }
 
+/* Populate the length object and try gluing it to the next chunk. */
+void setDeferredAggregateLen(client *c, void *node, long length, char prefix) {
+    serverAssert(length >= 0);
+
+    /* Abort when *node is NULL: when the client should not accept writes
+     * we return NULL in addReplyDeferredLen() */
+    if (node == NULL) return;
+
+    char lenstr[128];
+    size_t lenstr_len = sprintf(lenstr, "%c%ld\r\n", prefix, length);
+    setDeferredReply(c, node, lenstr, lenstr_len);
+}
+
 void setDeferredArrayLen(client *c, void *node, long length) {
     setDeferredAggregateLen(c,node,length,'*');
 }
@@ -797,6 +807,14 @@ void addReplyBulkSds(client *c, sds s)  {
     addReply(c,shared.crlf);
 }
 
+/* Set sds to a deferred reply (for symmetry with addReplyBulkSds it also frees the sds) */
+void setDeferredReplyBulkSds(client *c, void *node, sds s) {
+    sds reply = sdscatprintf(sdsempty(), "$%d\r\n%s\r\n", (unsigned)sdslen(s), s);
+    setDeferredReply(c, node, reply, sdslen(reply));
+    sdsfree(reply);
+    sdsfree(s);
+}
+
 /* Add a C null term string as bulk reply */
 void addReplyBulkCString(client *c, const char *s) {
     if (s == NULL) {
diff --git a/src/server.c b/src/server.c
index 56f7484b1..dc5d52af8 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1027,6 +1027,10 @@ struct redisCommand redisCommandTable[] = {
      "write random fast @stream",
      0,NULL,1,1,1,0,0,0},
 
+    {"xautoclaim",xautoclaimCommand,-6,
+     "write random fast @stream",
+     0,NULL,1,1,1,0,0,0},
+
     {"xinfo",xinfoCommand,-2,
      "read-only random @stream",
      0,NULL,2,2,1,0,0,0},
diff --git a/src/server.h b/src/server.h
index d79bf5b23..8b98f803d 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1759,6 +1759,7 @@ void addReplyBulkLongLong(client *c, long long ll);
 void addReply(client *c, robj *obj);
 void addReplySds(client *c, sds s);
 void addReplyBulkSds(client *c, sds s);
+void setDeferredReplyBulkSds(client *c, void *node, sds s);
 void addReplyErrorObject(client *c, robj *err);
 void addReplyErrorSds(client *c, sds err);
 void addReplyError(client *c, const char *err);
@@ -2584,6 +2585,7 @@ void xsetidCommand(client *c);
 void xackCommand(client *c);
 void xpendingCommand(client *c);
 void xclaimCommand(client *c);
+void xautoclaimCommand(client *c);
 void xinfoCommand(client *c);
 void xdelCommand(client *c);
 void xtrimCommand(client *c);
diff --git a/src/t_stream.c b/src/t_stream.c
index 46918bae8..d085a8948 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -960,6 +960,11 @@ void addReplyStreamID(client *c, streamID *id) {
     addReplyBulkSds(c,replyid);
 }
 
+void setDeferredReplyStreamID(client *c, void *dr, streamID *id) {
+    sds replyid = sdscatfmt(sdsempty(),"%U-%U",id->ms,id->seq);
+    setDeferredReplyBulkSds(c, dr, replyid);
+}
+
 /* Similar to the above function, but just creates an object, usually useful
  * for replication purposes to create arguments. */
 robj *createObjectFromStreamID(streamID *id) {
@@ -2666,6 +2671,160 @@ cleanup:
     if (ids != static_ids) zfree(ids);
 }
 
+/* XAUTOCLAIM <key> <group> <consumer> <min-idle-time> <start> [COUNT <count>] [JUSTID]
+ *
+ * Gets ownership of one or multiple messages in the Pending Entries List
+ * of a given stream consumer group.
+ *
+ * For each PEL entry, if its idle time greater or equal to <min-idle-time>,
+ * then the message new owner becomes the specified <consumer>.
+ * If the minimum idle time specified is zero, messages are claimed
+ * regardless of their idle time.
+ *
+ * This command creates the consumer as side effect if it does not yet
+ * exists. Moreover the command reset the idle time of the message to 0.
+ *
+ * The command returns an array of messages that the user
+ * successfully claimed, so that the caller is able to understand
+ * what messages it is now in charge of. */
+void xautoclaimCommand(client *c) {
+    streamCG *group = NULL;
+    robj *o = lookupKeyRead(c->db,c->argv[1]);
+    long long minidle; /* Minimum idle time argument, in milliseconds. */
+    long count = 100; /* Maximum entries to claim. */
+    streamID startid;
+    int startex;
+    int justid = 0;
+
+    /* Parse idle/start/end/count arguments ASAP if needed, in order to report
+     * syntax errors before any other error. */
+    if (getLongLongFromObjectOrReply(c,c->argv[4],&minidle,"Invalid min-idle-time argument for XAUTOCLAIM") != C_OK)
+        return;
+    if (minidle < 0) minidle = 0;
+
+    if (streamParseIntervalIDOrReply(c,c->argv[5],&startid,&startex,0) != C_OK)
+        return;
+    if (startex && streamIncrID(&startid) != C_OK) {
+        addReplyError(c,"invalid start ID for the interval");
+        return;
+    }
+
+    int j = 6; /* options start at argv[6] */
+    while(j < c->argc) {
+        int moreargs = (c->argc-1) - j; /* Number of additional arguments. */
+        char *opt = c->argv[j]->ptr;
+        if (!strcasecmp(opt,"COUNT") && moreargs) {
+            if (getPositiveLongFromObjectOrReply(c,c->argv[j+1],&count,NULL) != C_OK)
+                return;
+            if (count == 0) {
+                addReplyError(c,"COUNT must be > 0");
+                return;
+            }
+            j++;
+        } else if (!strcasecmp(opt,"JUSTID")) {
+            justid = 1;
+        } else {
+            addReplyErrorObject(c,shared.syntaxerr);
+            return;
+        }
+        j++;
+    }
+
+    if (o) {
+        if (checkType(c,o,OBJ_STREAM))
+            return; /* Type error. */
+        group = streamLookupCG(o->ptr,c->argv[2]->ptr);
+    }
+
+    /* No key or group? Send an error given that the group creation
+     * is mandatory. */
+    if (o == NULL || group == NULL) {
+        addReplyErrorFormat(c,"-NOGROUP No such key '%s' or consumer group '%s'",
+                            (char*)c->argv[1]->ptr,
+                            (char*)c->argv[2]->ptr);
+        return;
+    }
+
+    /* Do the actual claiming. */
+    streamConsumer *consumer = NULL;
+    long long attempts = count*10;
+
+    addReplyArrayLen(c, 2);
+    void *endidptr = addReplyDeferredLen(c);
+    void *arraylenptr = addReplyDeferredLen(c);
+
+    unsigned char startkey[sizeof(streamID)];
+    streamEncodeID(startkey,&startid);
+    raxIterator ri;
+    raxStart(&ri,group->pel);
+    raxSeek(&ri,">=",startkey,sizeof(startkey));
+    size_t arraylen = 0;
+    mstime_t now = mstime();
+    while (attempts-- && count && raxNext(&ri)) {
+        streamNACK *nack = ri.data;
+
+        if (minidle) {
+            mstime_t this_idle = now - nack->delivery_time;
+            if (this_idle < minidle)
+                continue;
+        }
+
+        streamID id;
+        streamDecodeID(ri.key, &id);
+
+        if (consumer == NULL)
+            consumer = streamLookupConsumer(group,c->argv[3]->ptr,SLC_NONE,NULL);
+        if (nack->consumer != consumer) {
+            /* Remove the entry from the old consumer.
+             * Note that nack->consumer is NULL if we created the
+             * NACK above because of the FORCE option. */
+            if (nack->consumer)
+                raxRemove(nack->consumer->pel,ri.key,ri.key_len,NULL);
+        }
+
+        /* Update the consumer and idle time. */
+        nack->delivery_time = now;
+        nack->delivery_count++;
+
+        if (nack->consumer != consumer) {
+            /* Add the entry in the new consumer local PEL. */
+            raxInsert(consumer->pel,ri.key,ri.key_len,nack,NULL);
+            nack->consumer = consumer;
+        }
+
+        /* Send the reply for this entry. */
+        if (justid) {
+            addReplyStreamID(c,&id);
+        } else {
+            size_t emitted =
+                streamReplyWithRange(c,o->ptr,&id,&id,1,0,NULL,NULL,
+                                     STREAM_RWR_RAWENTRIES,NULL);
+            if (!emitted)
+                addReplyNull(c);
+        }
+        arraylen++;
+        count--;
+
+        /* Propagate this change. */
+        robj *idstr = createObjectFromStreamID(&id);
+        streamPropagateXCLAIM(c,c->argv[1],group,c->argv[2],idstr,nack);
+        decrRefCount(idstr);
+        server.dirty++;
+    }
+
+    streamID endid;
+    if (raxEOF(&ri)) {
+        endid.ms = endid.seq = 0;
+    } else {
+        streamDecodeID(ri.key, &endid);
+    }
+    raxStop(&ri);
+
+    setDeferredArrayLen(c,arraylenptr,arraylen);
+    setDeferredReplyStreamID(c,endidptr,&endid);
+
+    preventCommandPropagation(c);
+}
 
 /* XDEL <key> [<ID1> <ID2> ... <IDN>]
  *
diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl
index 91dc2245e..f8de0741d 100644
--- a/tests/unit/type/stream-cgroups.tcl
+++ b/tests/unit/type/stream-cgroups.tcl
@@ -27,7 +27,7 @@ start_server {
         # and not the element "foo bar" which was pre existing in the
         # stream (see previous test)
         set reply [
-            r XREADGROUP GROUP mygroup client-1 STREAMS mystream ">"
+            r XREADGROUP GROUP mygroup consumer-1 STREAMS mystream ">"
         ]
         assert {[llength [lindex $reply 0 1]] == 2}
         lindex $reply 0 1 0 1
@@ -39,13 +39,13 @@ start_server {
         r XADD mystream * d 4
         # Read a few elements using a different consumer name
         set reply [
-            r XREADGROUP GROUP mygroup client-2 STREAMS mystream ">"
+            r XREADGROUP GROUP mygroup consumer-2 STREAMS mystream ">"
         ]
         assert {[llength [lindex $reply 0 1]] == 2}
         assert {[lindex $reply 0 1 0 1] eq {c 3}}
 
-        set r1 [r XREADGROUP GROUP mygroup client-1 COUNT 10 STREAMS mystream 0]
-        set r2 [r XREADGROUP GROUP mygroup client-2 COUNT 10 STREAMS mystream 0]
+        set r1 [r XREADGROUP GROUP mygroup consumer-1 COUNT 10 STREAMS mystream 0]
+        set r2 [r XREADGROUP GROUP mygroup consumer-2 COUNT 10 STREAMS mystream 0]
         assert {[lindex $r1 0 1 0 1] eq {a 1}}
         assert {[lindex $r2 0 1 0 1] eq {c 3}}
     }
@@ -56,9 +56,9 @@ start_server {
         for {set j 0} {$j < 4} {incr j} {
             set item [lindex $pending $j]
             if {$j < 2} {
-                set owner client-1
+                set owner consumer-1
             } else {
-                set owner client-2
+                set owner consumer-2
             }
             assert {[lindex $item 1] eq $owner}
             assert {[lindex $item 1] eq $owner}
@@ -66,7 +66,7 @@ start_server {
     }
 
     test {XPENDING can return single consumer items} {
-        set pending [r XPENDING mystream mygroup - + 10 client-1]
+        set pending [r XPENDING mystream mygroup - + 10 consumer-1]
         assert {[llength $pending] == 2}
     }
 
@@ -77,9 +77,9 @@ start_server {
 
     test {XPENDING with IDLE} {
         after 20
-        set pending [r XPENDING mystream mygroup IDLE 99999999 - + 10 client-1]
+        set pending [r XPENDING mystream mygroup IDLE 99999999 - + 10 consumer-1]
         assert {[llength $pending] == 0}
-        set pending [r XPENDING mystream mygroup IDLE 1 - + 10 client-1]
+        set pending [r XPENDING mystream mygroup IDLE 1 - + 10 consumer-1]
         assert {[llength $pending] == 2}
         set pending [r XPENDING mystream mygroup IDLE 99999999 - + 10]
         assert {[llength $pending] == 0}
@@ -101,12 +101,12 @@ start_server {
         }
     }
 
-    test {XACK is able to remove items from the client/group PEL} {
-        set pending [r XPENDING mystream mygroup - + 10 client-1]
+    test {XACK is able to remove items from the consumer/group PEL} {
+        set pending [r XPENDING mystream mygroup - + 10 consumer-1]
         set id1 [lindex $pending 0 0]
         set id2 [lindex $pending 1 0]
         assert {[r XACK mystream mygroup $id1] eq 1}
-        set pending [r XPENDING mystream mygroup - + 10 client-1]
+        set pending [r XPENDING mystream mygroup - + 10 consumer-1]
         assert {[llength $pending] == 1}
         set id [lindex $pending 0 0]
         assert {$id eq $id2}
@@ -242,52 +242,52 @@ start_server {
         set id3 [r XADD mystream * c 3]
         r XGROUP CREATE mystream mygroup 0
 
-        # Client 1 reads item 1 from the stream without acknowledgements.
-        # Client 2 then claims pending item 1 from the PEL of client 1
+        # Consumer 1 reads item 1 from the stream without acknowledgements.
+        # Consumer 2 then claims pending item 1 from the PEL of consumer 1
         set reply [
-            r XREADGROUP GROUP mygroup client1 count 1 STREAMS mystream >
+            r XREADGROUP GROUP mygroup consumer1 count 1 STREAMS mystream >
         ]
         assert {[llength [lindex $reply 0 1 0 1]] == 2}
         assert {[lindex $reply 0 1 0 1] eq {a 1}}
 
         # make sure the entry is present in both the gorup, and the right consumer
         assert {[llength [r XPENDING mystream mygroup - + 10]] == 1}
-        assert {[llength [r XPENDING mystream mygroup - + 10 client1]] == 1}
-        assert {[llength [r XPENDING mystream mygroup - + 10 client2]] == 0}
+        assert {[llength [r XPENDING mystream mygroup - + 10 consumer1]] == 1}
+        assert {[llength [r XPENDING mystream mygroup - + 10 consumer2]] == 0}
 
-        r debug sleep 0.2
+        after 200
         set reply [
-            r XCLAIM mystream mygroup client2 10 $id1
+            r XCLAIM mystream mygroup consumer2 10 $id1
         ]
         assert {[llength [lindex $reply 0 1]] == 2}
         assert {[lindex $reply 0 1] eq {a 1}}
 
         # make sure the entry is present in both the gorup, and the right consumer
         assert {[llength [r XPENDING mystream mygroup - + 10]] == 1}
-        assert {[llength [r XPENDING mystream mygroup - + 10 client1]] == 0}
-        assert {[llength [r XPENDING mystream mygroup - + 10 client2]] == 1}
+        assert {[llength [r XPENDING mystream mygroup - + 10 consumer1]] == 0}
+        assert {[llength [r XPENDING mystream mygroup - + 10 consumer2]] == 1}
 
-        # Client 1 reads another 2 items from stream
-        r XREADGROUP GROUP mygroup client1 count 2 STREAMS mystream >
-        r debug sleep 0.2
+        # Consumer 1 reads another 2 items from stream
+        r XREADGROUP GROUP mygroup consumer1 count 2 STREAMS mystream >
+        after 200
 
-        # Delete item 2 from the stream. Now client 1 has PEL that contains
-        # only item 3. Try to use client 2 to claim the deleted item 2
-        # from the PEL of client 1, this should return nil
+        # Delete item 2 from the stream. Now consumer 1 has PEL that contains
+        # only item 3. Try to use consumer 2 to claim the deleted item 2
+        # from the PEL of consumer 1, this should return nil
         r XDEL mystream $id2
         set reply [
-            r XCLAIM mystream mygroup client2 10 $id2
+            r XCLAIM mystream mygroup consumer2 10 $id2
         ]
         assert {[llength $reply] == 1}
         assert_equal "" [lindex $reply 0]
 
-        # Delete item 3 from the stream. Now client 1 has PEL that is empty.
-        # Try to use client 2 to claim the deleted item 3 from the PEL
-        # of client 1, this should return nil
-        r debug sleep 0.2
+        # Delete item 3 from the stream. Now consumer 1 has PEL that is empty.
+        # Try to use consumer 2 to claim the deleted item 3 from the PEL
+        # of consumer 1, this should return nil
+        after 200
         r XDEL mystream $id3
         set reply [
-            r XCLAIM mystream mygroup client2 10 $id3
+            r XCLAIM mystream mygroup consumer2 10 $id3
         ]
         assert {[llength $reply] == 1}
         assert_equal "" [lindex $reply 0]
@@ -301,16 +301,16 @@ start_server {
         set id3 [r XADD mystream * c 3]
         r XGROUP CREATE mystream mygroup 0
 
-        # Client 1 reads item 1 from the stream without acknowledgements.
-        # Client 2 then claims pending item 1 from the PEL of client 1
+        # Consumer 1 reads item 1 from the stream without acknowledgements.
+        # Consumer 2 then claims pending item 1 from the PEL of consumer 1
         set reply [
-            r XREADGROUP GROUP mygroup client1 count 1 STREAMS mystream >
+            r XREADGROUP GROUP mygroup consumer1 count 1 STREAMS mystream >
         ]
         assert {[llength [lindex $reply 0 1 0 1]] == 2}
         assert {[lindex $reply 0 1 0 1] eq {a 1}}
-        r debug sleep 0.2
+        after 200
         set reply [
-            r XCLAIM mystream mygroup client2 10 $id1
+            r XCLAIM mystream mygroup consumer2 10 $id1
         ]
         assert {[llength [lindex $reply 0 1]] == 2}
         assert {[lindex $reply 0 1] eq {a 1}}
@@ -321,10 +321,10 @@ start_server {
         assert {[llength [lindex $reply 0]] == 4}
         assert {[lindex $reply 0 3] == 2}
 
-        # Client 3 then claims pending item 1 from the PEL of client 2 using JUSTID
-        r debug sleep 0.2
+        # Consumer 3 then claims pending item 1 from the PEL of consumer 2 using JUSTID
+        after 200
         set reply [
-            r XCLAIM mystream mygroup client3 10 $id1 JUSTID
+            r XCLAIM mystream mygroup consumer3 10 $id1 JUSTID
         ]
         assert {[llength $reply] == 1}
         assert {[lindex $reply 0] eq $id1}
@@ -344,17 +344,122 @@ start_server {
         set id3 [r XADD mystream * c 3]
         r XGROUP CREATE mystream mygroup 0
 
-        set reply [r XREADGROUP GROUP mygroup client1 count 1 STREAMS mystream >]
+        set reply [r XREADGROUP GROUP mygroup consumer1 count 1 STREAMS mystream >]
         assert {[llength [lindex $reply 0 1 0 1]] == 2}
         assert {[lindex $reply 0 1 0 1] eq {a 1}}
-        r debug sleep 0.2
+        after 200
         # re-claim with the same consumer that already has it
-        assert {[llength [r XCLAIM mystream mygroup client1 10 $id1]] == 1}
+        assert {[llength [r XCLAIM mystream mygroup consumer1 10 $id1]] == 1}
 
         # make sure the entry is still in the PEL
         set reply [r XPENDING mystream mygroup - + 10]
         assert {[llength $reply] == 1}
-        assert {[lindex $reply 0 1] eq {client1}}
+        assert {[lindex $reply 0 1] eq {consumer1}}
+    }
+
+    test {XAUTOCLAIM can claim PEL items from another consumer} {
+        # Add 3 items into the stream, and create a consumer group
+        r del mystream
+        set id1 [r XADD mystream * a 1]
+        set id2 [r XADD mystream * b 2]
+        set id3 [r XADD mystream * c 3]
+        r XGROUP CREATE mystream mygroup 0
+
+        # Consumer 1 reads item 1 from the stream without acknowledgements.
+        # Consumer 2 then claims pending item 1 from the PEL of consumer 1
+        set reply [r XREADGROUP GROUP mygroup consumer1 count 1 STREAMS mystream >]
+        assert_equal [llength [lindex $reply 0 1 0 1]] 2
+        assert_equal [lindex $reply 0 1 0 1] {a 1}
+        after 200
+        set reply [r XAUTOCLAIM mystream mygroup consumer2 10 - COUNT 1]
+        assert_equal [llength $reply] 2
+        assert_equal [lindex $reply 0] $id1
+        assert_equal [llength [lindex $reply 1]] 1
+        assert_equal [llength [lindex $reply 1 0]] 2
+        assert_equal [llength [lindex $reply 1 0 1]] 2
+        assert_equal [lindex $reply 1 0 1] {a 1}
+
+        # Consumer 1 reads another 2 items from stream
+        r XREADGROUP GROUP mygroup consumer1 count 2 STREAMS mystream >
+
+        # For min-idle-time
+        after 200
+
+        # Delete item 2 from the stream. Now consumer 1 has PEL that contains
+        # only item 3. Try to use consumer 2 to claim the deleted item 2
+        # from the PEL of consumer 1, this should return nil
+        r XDEL mystream $id2
+        set reply [r XAUTOCLAIM mystream mygroup consumer2 10 - COUNT 2]
+        # id1 is self-claimed here but not id2 ('count' was set to 2)
+        assert_equal [llength $reply] 2
+        assert_equal [lindex $reply 0] $id2
+        assert_equal [llength [lindex $reply 1]] 2
+        assert_equal [llength [lindex $reply 1 0]] 2
+        assert_equal [llength [lindex $reply 1 0 1]] 2
+        assert_equal [lindex $reply 1 0 1] {a 1}
+        assert_equal [lindex $reply 1 1] ""
+
+        # Delete item 3 from the stream. Now consumer 1 has PEL that is empty.
+        # Try to use consumer 2 to claim the deleted item 3 from the PEL
+        # of consumer 1, this should return nil
+        after 200
+        r XDEL mystream $id3
+        set reply [r XAUTOCLAIM mystream mygroup consumer2 10 - JUSTID]
+        # id1 is self-claimed here but not id2 and id3 ('count' is default 100)
+
+        # we also test the JUSTID modifier here. note that, when using JUSTID,
+        # deleted entries are returned in reply (consistent with XCLAIM).
+
+        assert_equal [llength $reply] 2
+        assert_equal [lindex $reply 0] "0-0"
+        assert_equal [llength [lindex $reply 1]] 3
+        assert_equal [lindex $reply 1 0] $id1
+        assert_equal [lindex $reply 1 1] $id2
+        assert_equal [lindex $reply 1 2] $id3
+    }
+
+    test {XAUTOCLAIM as an iterator} {
+        # Add 5 items into the stream, and create a consumer group
+        r del mystream
+        set id1 [r XADD mystream * a 1]
+        set id2 [r XADD mystream * b 2]
+        set id3 [r XADD mystream * c 3]
+        set id4 [r XADD mystream * d 4]
+        set id5 [r XADD mystream * e 5]
+        r XGROUP CREATE mystream mygroup 0
+
+        # Read 5 messages into consumer1
+        r XREADGROUP GROUP mygroup consumer1 count 90 STREAMS mystream >
+
+        # For min-idle-time
+        after 200
+
+        # Claim 2 entries
+        set reply [r XAUTOCLAIM mystream mygroup consumer2 10 - COUNT 2]
+        assert_equal [llength $reply] 2
+        set cursor [lindex $reply 0]
+        assert_equal $cursor $id2
+        assert_equal [llength [lindex $reply 1]] 2
+        assert_equal [llength [lindex $reply 1 0 1]] 2
+        assert_equal [lindex $reply 1 0 1] {a 1}
+
+        # Claim 2 more entries
+        set reply [r XAUTOCLAIM mystream mygroup consumer2 10 ($cursor COUNT 2]
+        assert_equal [llength $reply] 2
+        set cursor [lindex $reply 0]
+        assert_equal $cursor $id4
+        assert_equal [llength [lindex $reply 1]] 2
+        assert_equal [llength [lindex $reply 1 0 1]] 2
+        assert_equal [lindex $reply 1 0 1] {c 3}
+
+        # Claim last entry
+        set reply [r XAUTOCLAIM mystream mygroup consumer2 10 ($cursor COUNT 2]
+        assert_equal [llength $reply] 2
+        set cursor [lindex $reply 0]
+        assert_equal $cursor {0-0}
+        assert_equal [llength [lindex $reply 1]] 1
+        assert_equal [llength [lindex $reply 1 0 1]] 2
+        assert_equal [lindex $reply 1 0 1] {e 5}
     }
 
     test {XINFO FULL output} {
@@ -477,7 +582,7 @@ start_server {
             assert {$curr_grpinfo == $grpinfo}
             set n_consumers [lindex $grpinfo 3]
 
-            # Bob should be created only when there will be new data for this client
+            # Bob should be created only when there will be new data for this consumer
             assert_equal $n_consumers 2
             set reply [r xinfo consumers mystream mygroup]
             set consumer_info [lindex $reply 0]

From cfcd0fa6f71af4e224c1d86be55b6b01f2288be2 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Wed, 6 Jan 2021 14:20:53 -0500
Subject: [PATCH 0947/1098] fix memory leak in processInlineBuffer error
 handling code (#8295)

This code path is normally executed only when v6.0 and above replicates from v2.4
---
 src/networking.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/networking.c b/src/networking.c
index 29e5605d9..acb1b27cb 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1734,6 +1734,7 @@ int processInlineBuffer(client *c) {
      * However the is an exception: masters may send us just a newline
      * to keep the connection active. */
     if (querylen != 0 && c->flags & CLIENT_MASTER) {
+        sdsfreesplitres(argv,argc);
         serverLog(LL_WARNING,"WARNING: Receiving inline protocol from master, master stream corruption? Closing the master connection and discarding the cached master.");
         setProtocolError("Master using the inline protocol. Desync?",c);
         return C_ERR;

From b5029dfdadb387ba8a3b8517ec4a7117bf4070f4 Mon Sep 17 00:00:00 2001
From: "Jonah H. Harris" <jonah.harris@gmail.com>
Date: Thu, 7 Jan 2021 03:58:53 -0500
Subject: [PATCH 0948/1098] Add ZRANGESTORE command, and improve ZSTORE command
 (#7844)

Add ZRANGESTORE command, and improve ZSTORE command to deprecated Z[REV]RANGE[BYSCORE|BYLEX].

Syntax for the new ZRANGESTORE command:
ZRANGESTORE [BYSCORE | BYLEX] [REV] [LIMIT offset count]

New syntax for ZRANGE:
ZRANGE [BYSCORE | BYLEX] [REV] [WITHSCORES] [LIMIT offset count]

Old syntax for ZRANGE:
ZRANGE [WITHSCORES]

Other ZRANGE commands remain unchanged.

The implementation uses common code for all of these, by utilizing a consumer interface that in one
command response to the client, and in the other command stores a zset key.

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/server.c             |   4 +
 src/server.h             |   1 +
 src/t_zset.c             | 667 +++++++++++++++++++++++++--------------
 tests/unit/type/zset.tcl |  82 +++++
 4 files changed, 521 insertions(+), 233 deletions(-)

diff --git a/src/server.c b/src/server.c
index dc5d52af8..6140b21ac 100644
--- a/src/server.c
+++ b/src/server.c
@@ -464,6 +464,10 @@ struct redisCommand redisCommandTable[] = {
      "read-only @sortedset",
      0,NULL,1,1,1,0,0,0},
 
+    {"zrangestore",zrangestoreCommand,-5,
+     "write use-memory @sortedset",
+     0,NULL,1,2,1,0,0,0},
+
     {"zrangebyscore",zrangebyscoreCommand,-4,
      "read-only @sortedset",
      0,NULL,1,1,1,0,0,0},
diff --git a/src/server.h b/src/server.h
index 8b98f803d..b5610432c 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2517,6 +2517,7 @@ void zinterstoreCommand(client *c);
 void zdiffstoreCommand(client *c);
 void zunionCommand(client *c);
 void zinterCommand(client *c);
+void zrangestoreCommand(client *c);
 void zdiffCommand(client *c);
 void zscanCommand(client *c);
 void hkeysCommand(client *c);
diff --git a/src/t_zset.c b/src/t_zset.c
index 25c7dda4c..3d63c41c6 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -1822,11 +1822,15 @@ void zremCommand(client *c) {
     addReplyLongLong(c,deleted);
 }
 
+typedef enum {
+    ZRANGE_AUTO = 0,
+    ZRANGE_RANK,
+    ZRANGE_SCORE,
+    ZRANGE_LEX,
+} zrange_type;
+
 /* Implements ZREMRANGEBYRANK, ZREMRANGEBYSCORE, ZREMRANGEBYLEX commands. */
-#define ZRANGE_RANK 0
-#define ZRANGE_SCORE 1
-#define ZRANGE_LEX 2
-void zremrangeGenericCommand(client *c, int rangetype) {
+void zremrangeGenericCommand(client *c, zrange_type rangetype) {
     robj *key = c->argv[1];
     robj *zobj;
     int keyremoved = 0;
@@ -1834,22 +1838,28 @@ void zremrangeGenericCommand(client *c, int rangetype) {
     zrangespec range;
     zlexrangespec lexrange;
     long start, end, llen;
+    char *notify_type = NULL;
 
     /* Step 1: Parse the range. */
     if (rangetype == ZRANGE_RANK) {
+        notify_type = "zremrangebyrank";
         if ((getLongFromObjectOrReply(c,c->argv[2],&start,NULL) != C_OK) ||
             (getLongFromObjectOrReply(c,c->argv[3],&end,NULL) != C_OK))
             return;
     } else if (rangetype == ZRANGE_SCORE) {
+        notify_type = "zremrangebyscore";
         if (zslParseRange(c->argv[2],c->argv[3],&range) != C_OK) {
             addReplyError(c,"min or max is not a float");
             return;
         }
     } else if (rangetype == ZRANGE_LEX) {
+        notify_type = "zremrangebylex";
         if (zslParseLexRange(c->argv[2],c->argv[3],&lexrange) != C_OK) {
             addReplyError(c,"min or max not valid string range item");
             return;
         }
+    } else {
+        serverPanic("unknown rangetype %d", (int)rangetype);
     }
 
     /* Step 2: Lookup & range sanity checks if needed. */
@@ -1875,6 +1885,7 @@ void zremrangeGenericCommand(client *c, int rangetype) {
     /* Step 3: Perform the range deletion operation. */
     if (zobj->encoding == OBJ_ENCODING_ZIPLIST) {
         switch(rangetype) {
+        case ZRANGE_AUTO:
         case ZRANGE_RANK:
             zobj->ptr = zzlDeleteRangeByRank(zobj->ptr,start+1,end+1,&deleted);
             break;
@@ -1892,6 +1903,7 @@ void zremrangeGenericCommand(client *c, int rangetype) {
     } else if (zobj->encoding == OBJ_ENCODING_SKIPLIST) {
         zset *zs = zobj->ptr;
         switch(rangetype) {
+        case ZRANGE_AUTO:
         case ZRANGE_RANK:
             deleted = zslDeleteRangeByRank(zs->zsl,start+1,end+1,zs->dict);
             break;
@@ -1913,9 +1925,8 @@ void zremrangeGenericCommand(client *c, int rangetype) {
 
     /* Step 4: Notifications and reply. */
     if (deleted) {
-        char *event[3] = {"zremrangebyrank","zremrangebyscore","zremrangebylex"};
         signalModifiedKey(c,c->db,key);
-        notifyKeyspaceEvent(NOTIFY_ZSET,event[rangetype],key,c->db->id);
+        notifyKeyspaceEvent(NOTIFY_ZSET,notify_type,key,c->db->id);
         if (keyremoved)
             notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
     }
@@ -2778,27 +2789,180 @@ void zdiffCommand(client *c) {
     zunionInterDiffGenericCommand(c, NULL, 1, SET_OP_DIFF);
 }
 
-void zrangeGenericCommand(client *c, int reverse) {
-    robj *key = c->argv[1];
-    robj *zobj;
-    int withscores = 0;
-    long start;
-    long end;
-    long llen;
-    long rangelen;
+typedef enum {
+    ZRANGE_DIRECTION_AUTO = 0,
+    ZRANGE_DIRECTION_FORWARD,
+    ZRANGE_DIRECTION_REVERSE
+} zrange_direction;
 
-    if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != C_OK) ||
-        (getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != C_OK)) return;
+typedef enum {
+    ZRANGE_CONSUMER_TYPE_CLIENT = 0,
+    ZRANGE_CONSUMER_TYPE_INTERNAL
+} zrange_consumer_type;
 
-    if (c->argc == 5 && !strcasecmp(c->argv[4]->ptr,"withscores")) {
-        withscores = 1;
-    } else if (c->argc >= 5) {
-        addReplyErrorObject(c,shared.syntaxerr);
-        return;
+typedef struct zrange_result_handler zrange_result_handler;
+
+typedef void (*zrangeResultBeginFunction)(zrange_result_handler *c);
+typedef void (*zrangeResultFinalizeFunction)(
+    zrange_result_handler *c, size_t result_count);
+typedef void (*zrangeResultEmitCBufferFunction)(
+    zrange_result_handler *c, const void *p, size_t len, double score);
+typedef void (*zrangeResultEmitLongLongFunction)(
+    zrange_result_handler *c, long long ll, double score);
+
+void zrangeGenericCommand (zrange_result_handler *handler, int argc_start, int store,
+                           zrange_type rangetype, zrange_direction direction);
+
+/* Interface struct for ZRANGE/ZRANGESTORE generic implementation.
+ * There is one implementation of this interface that sends a RESP reply to clients.
+ * and one implementation that stores the range result into a zset object. */
+struct zrange_result_handler {
+    zrange_consumer_type                 type;
+    client                              *client;
+    robj                                *dstkey;
+    robj                                *dstobj;
+    void                                *userdata;
+    int                                  withscores;
+    int                                  should_emit_array_length;
+    zrangeResultBeginFunction            beginResultEmission;
+    zrangeResultFinalizeFunction         finalizeResultEmission;
+    zrangeResultEmitCBufferFunction      emitResultFromCBuffer;
+    zrangeResultEmitLongLongFunction     emitResultFromLongLong;
+};
+
+/* Result handler methods for responding the ZRANGE to clients. */
+static void zrangeResultBeginClient(zrange_result_handler *handler) {
+    handler->userdata = addReplyDeferredLen(handler->client);
+}
+
+static void zrangeResultEmitCBufferToClient(zrange_result_handler *handler,
+    const void *value, size_t value_length_in_bytes, double score)
+{
+    if (handler->should_emit_array_length) {
+        addReplyArrayLen(handler->client, 2);
     }
 
-    if ((zobj = lookupKeyReadOrReply(c,key,shared.emptyarray)) == NULL
-         || checkType(c,zobj,OBJ_ZSET)) return;
+    addReplyBulkCBuffer(handler->client, value, value_length_in_bytes);
+
+    if (handler->withscores) {
+        addReplyDouble(handler->client, score);
+    }
+}
+
+static void zrangeResultEmitLongLongToClient(zrange_result_handler *handler,
+    long long value, double score)
+{
+    if (handler->should_emit_array_length) {
+        addReplyArrayLen(handler->client, 2);
+    }
+
+    addReplyBulkLongLong(handler->client, value);
+
+    if (handler->withscores) {
+        addReplyDouble(handler->client, score);
+    }
+}
+
+static void zrangeResultFinalizeClient(zrange_result_handler *handler,
+    size_t result_count)
+{
+    if (handler->withscores && (handler->client->resp == 2)) {
+        result_count *= 2;
+    }
+
+    setDeferredArrayLen(handler->client, handler->userdata, result_count);
+}
+
+/* Result handler methods for storing the ZRANGESTORE to a zset. */
+static void zrangeResultBeginStore(zrange_result_handler *handler)
+{
+    handler->dstobj = createZsetZiplistObject();
+}
+
+static void zrangeResultEmitCBufferForStore(zrange_result_handler *handler,
+    const void *value, size_t value_length_in_bytes, double score)
+{
+    double newscore;
+    int retflags = 0;
+    sds ele = sdsnewlen(value, value_length_in_bytes);
+    int retval = zsetAdd(handler->dstobj, score, ele, &retflags, &newscore);
+    sdsfree(ele);
+    serverAssert(retval);
+}
+
+static void zrangeResultEmitLongLongForStore(zrange_result_handler *handler,
+    long long value, double score)
+{
+    double newscore;
+    int retflags = 0;
+    sds ele = sdsfromlonglong(value);
+    int retval = zsetAdd(handler->dstobj, score, ele, &retflags, &newscore);
+    sdsfree(ele);
+    serverAssert(retval);
+}
+
+static void zrangeResultFinalizeStore(zrange_result_handler *handler, size_t result_count)
+{
+    if (result_count) {
+        setKey(handler->client, handler->client->db, handler->dstkey, handler->dstobj);
+        addReplyLongLong(handler->client, result_count);
+        notifyKeyspaceEvent(NOTIFY_ZSET, "zrangestore", handler->dstkey, handler->client->db->id);
+        server.dirty++;
+    } else {
+        addReply(handler->client, shared.czero);
+        if (dbDelete(handler->client->db, handler->dstkey)) {
+            signalModifiedKey(handler->client, handler->client->db, handler->dstkey);
+            notifyKeyspaceEvent(NOTIFY_GENERIC, "del", handler->dstkey, handler->client->db->id);
+            server.dirty++;
+        }
+    }
+    decrRefCount(handler->dstobj);
+}
+
+/* Initialize the consumer interface type with the requested type. */
+static void zrangeResultHandlerInit(zrange_result_handler *handler,
+    client *client, zrange_consumer_type type)
+{
+    memset(handler, 0, sizeof(*handler));
+
+    handler->client = client;
+
+    switch (type) {
+    case ZRANGE_CONSUMER_TYPE_CLIENT:
+        handler->beginResultEmission = zrangeResultBeginClient;
+        handler->finalizeResultEmission = zrangeResultFinalizeClient;
+        handler->emitResultFromCBuffer = zrangeResultEmitCBufferToClient;
+        handler->emitResultFromLongLong = zrangeResultEmitLongLongToClient;
+        break;
+
+    case ZRANGE_CONSUMER_TYPE_INTERNAL:
+        handler->beginResultEmission = zrangeResultBeginStore;
+        handler->finalizeResultEmission = zrangeResultFinalizeStore;
+        handler->emitResultFromCBuffer = zrangeResultEmitCBufferForStore;
+        handler->emitResultFromLongLong = zrangeResultEmitLongLongForStore;
+        break;
+    }
+}
+
+static void zrangeResultHandlerScoreEmissionEnable(zrange_result_handler *handler) {
+    handler->withscores = 1;
+    handler->should_emit_array_length = (handler->client->resp > 2);
+}
+
+static void zrangeResultHandlerDestinationKeySet (zrange_result_handler *handler,
+    robj *dstkey)
+{
+    handler->dstkey = dstkey;
+}
+
+/* This command implements ZRANGE, ZREVRANGE. */
+void genericZrangebyrankCommand(zrange_result_handler *handler,
+    robj *zobj, long start, long end, int withscores, int reverse) {
+
+    client *c = handler->client;
+    long llen;
+    long rangelen;
+    size_t result_cardinality;
 
     /* Sanitize indexes. */
     llen = zsetLength(zobj);
@@ -2806,22 +2970,17 @@ void zrangeGenericCommand(client *c, int reverse) {
     if (end < 0) end = llen+end;
     if (start < 0) start = 0;
 
+    handler->beginResultEmission(handler);
+
     /* Invariant: start >= 0, so this test will be true when end < 0.
      * The range is empty when start > end or start >= length. */
     if (start > end || start >= llen) {
-        addReply(c,shared.emptyarray);
+        handler->finalizeResultEmission(handler, 0);
         return;
     }
     if (end >= llen) end = llen-1;
     rangelen = (end-start)+1;
-
-    /* Return the result in form of a multi-bulk reply. RESP3 clients
-     * will receive sub arrays with score->element, while RESP2 returned
-     * a flat array. */
-    if (withscores && c->resp == 2)
-        addReplyArrayLen(c, rangelen*2);
-    else
-        addReplyArrayLen(c, rangelen);
+    result_cardinality = rangelen;
 
     if (zobj->encoding == OBJ_ENCODING_ZIPLIST) {
         unsigned char *zl = zobj->ptr;
@@ -2829,6 +2988,7 @@ void zrangeGenericCommand(client *c, int reverse) {
         unsigned char *vstr;
         unsigned int vlen;
         long long vlong;
+        double score = 0.0;
 
         if (reverse)
             eptr = ziplistIndex(zl,-2-(2*start));
@@ -2842,12 +3002,14 @@ void zrangeGenericCommand(client *c, int reverse) {
             serverAssertWithInfo(c,zobj,eptr != NULL && sptr != NULL);
             serverAssertWithInfo(c,zobj,ziplistGet(eptr,&vstr,&vlen,&vlong));
 
-            if (withscores && c->resp > 2) addReplyArrayLen(c,2);
-            if (vstr == NULL)
-                addReplyBulkLongLong(c,vlong);
-            else
-                addReplyBulkCBuffer(c,vstr,vlen);
-            if (withscores) addReplyDouble(c,zzlGetScore(sptr));
+            if (withscores) /* don't bother to extract the score if it's gonna be ignored. */
+                score = zzlGetScore(sptr);
+
+            if (vstr == NULL) {
+                handler->emitResultFromLongLong(handler, vlong, score);
+            } else {
+                handler->emitResultFromCBuffer(handler, vstr, vlen, score);
+            }
 
             if (reverse)
                 zzlPrev(zl,&eptr,&sptr);
@@ -2859,7 +3021,6 @@ void zrangeGenericCommand(client *c, int reverse) {
         zset *zs = zobj->ptr;
         zskiplist *zsl = zs->zsl;
         zskiplistNode *ln;
-        sds ele;
 
         /* Check if starting point is trivial, before doing log(N) lookup. */
         if (reverse) {
@@ -2874,83 +3035,53 @@ void zrangeGenericCommand(client *c, int reverse) {
 
         while(rangelen--) {
             serverAssertWithInfo(c,zobj,ln != NULL);
-            ele = ln->ele;
-            if (withscores && c->resp > 2) addReplyArrayLen(c,2);
-            addReplyBulkCBuffer(c,ele,sdslen(ele));
-            if (withscores) addReplyDouble(c,ln->score);
+            sds ele = ln->ele;
+            handler->emitResultFromCBuffer(handler, ele, sdslen(ele), ln->score);
             ln = reverse ? ln->backward : ln->level[0].forward;
         }
     } else {
         serverPanic("Unknown sorted set encoding");
     }
+
+    handler->finalizeResultEmission(handler, result_cardinality);
 }
 
+/* ZRANGESTORE <dst> <src> <min> <max> [BYSCORE | BYLEX] [REV] [LIMIT offset count] */
+void zrangestoreCommand (client *c) {
+    robj *dstkey = c->argv[1];
+    zrange_result_handler handler;
+    zrangeResultHandlerInit(&handler, c, ZRANGE_CONSUMER_TYPE_INTERNAL);
+    zrangeResultHandlerDestinationKeySet(&handler, dstkey);
+    zrangeGenericCommand(&handler, 2, 1, ZRANGE_AUTO, ZRANGE_DIRECTION_AUTO);
+}
+
+/* ZRANGE <key> <min> <max> [BYSCORE | BYLEX] [REV] [WITHSCORES] [LIMIT offset count] */
 void zrangeCommand(client *c) {
-    zrangeGenericCommand(c,0);
+    zrange_result_handler handler;
+    zrangeResultHandlerInit(&handler, c, ZRANGE_CONSUMER_TYPE_CLIENT);
+    zrangeGenericCommand(&handler, 1, 0, ZRANGE_AUTO, ZRANGE_DIRECTION_AUTO);
 }
 
+/* ZREVRANGE <key> <min> <max> [WITHSCORES] */
 void zrevrangeCommand(client *c) {
-    zrangeGenericCommand(c,1);
+    zrange_result_handler handler;
+    zrangeResultHandlerInit(&handler, c, ZRANGE_CONSUMER_TYPE_CLIENT);
+    zrangeGenericCommand(&handler, 1, 0, ZRANGE_RANK, ZRANGE_DIRECTION_REVERSE);
 }
 
 /* This command implements ZRANGEBYSCORE, ZREVRANGEBYSCORE. */
-void genericZrangebyscoreCommand(client *c, int reverse) {
-    zrangespec range;
-    robj *key = c->argv[1];
-    robj *zobj;
-    long offset = 0, limit = -1;
-    int withscores = 0;
+void genericZrangebyscoreCommand(zrange_result_handler *handler,
+    zrangespec *range, robj *zobj, int withscores, long offset,
+    long limit, int reverse) {
+
+    client *c = handler->client;
     unsigned long rangelen = 0;
-    void *replylen = NULL;
-    int minidx, maxidx;
 
-    /* Parse the range arguments. */
-    if (reverse) {
-        /* Range is given as [max,min] */
-        maxidx = 2; minidx = 3;
-    } else {
-        /* Range is given as [min,max] */
-        minidx = 2; maxidx = 3;
-    }
-
-    if (zslParseRange(c->argv[minidx],c->argv[maxidx],&range) != C_OK) {
-        addReplyError(c,"min or max is not a float");
-        return;
-    }
-
-    /* Parse optional extra arguments. Note that ZCOUNT will exactly have
-     * 4 arguments, so we'll never enter the following code path. */
-    if (c->argc > 4) {
-        int remaining = c->argc - 4;
-        int pos = 4;
-
-        while (remaining) {
-            if (remaining >= 1 && !strcasecmp(c->argv[pos]->ptr,"withscores")) {
-                pos++; remaining--;
-                withscores = 1;
-            } else if (remaining >= 3 && !strcasecmp(c->argv[pos]->ptr,"limit")) {
-                if ((getLongFromObjectOrReply(c, c->argv[pos+1], &offset, NULL)
-                        != C_OK) ||
-                    (getLongFromObjectOrReply(c, c->argv[pos+2], &limit, NULL)
-                        != C_OK))
-                {
-                    return;
-                }
-                pos += 3; remaining -= 3;
-            } else {
-                addReplyErrorObject(c,shared.syntaxerr);
-                return;
-            }
-        }
-    }
-
-    /* Ok, lookup the key and get the range */
-    if ((zobj = lookupKeyReadOrReply(c,key,shared.emptyarray)) == NULL ||
-        checkType(c,zobj,OBJ_ZSET)) return;
+    handler->beginResultEmission(handler);
 
     /* For invalid offset, return directly. */
     if (offset > 0 && offset >= (long)zsetLength(zobj)) {
-        addReply(c,shared.emptyarray);
+        handler->finalizeResultEmission(handler, 0);
         return;
     }
 
@@ -2960,29 +3091,17 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
         unsigned char *vstr;
         unsigned int vlen;
         long long vlong;
-        double score;
 
         /* If reversed, get the last node in range as starting point. */
         if (reverse) {
-            eptr = zzlLastInRange(zl,&range);
+            eptr = zzlLastInRange(zl,range);
         } else {
-            eptr = zzlFirstInRange(zl,&range);
-        }
-
-        /* No "first" element in the specified interval. */
-        if (eptr == NULL) {
-            addReply(c,shared.emptyarray);
-            return;
+            eptr = zzlFirstInRange(zl,range);
         }
 
         /* Get score pointer for the first element. */
-        serverAssertWithInfo(c,zobj,eptr != NULL);
-        sptr = ziplistNext(zl,eptr);
-
-        /* We don't know in advance how many matching elements there are in the
-         * list, so we push this object that will represent the multi-bulk
-         * length in the output buffer, and will "fix" it later */
-        replylen = addReplyDeferredLen(c);
+        if (eptr)
+            sptr = ziplistNext(zl,eptr);
 
         /* If there is an offset, just traverse the number of elements without
          * checking the score because that is done in the next loop. */
@@ -2995,13 +3114,13 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
         }
 
         while (eptr && limit--) {
-            score = zzlGetScore(sptr);
+            double score = zzlGetScore(sptr);
 
             /* Abort when the node is no longer in range. */
             if (reverse) {
-                if (!zslValueGteMin(score,&range)) break;
+                if (!zslValueGteMin(score,range)) break;
             } else {
-                if (!zslValueLteMax(score,&range)) break;
+                if (!zslValueLteMax(score,range)) break;
             }
 
             /* We know the element exists, so ziplistGet should always
@@ -3009,13 +3128,11 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
             serverAssertWithInfo(c,zobj,ziplistGet(eptr,&vstr,&vlen,&vlong));
 
             rangelen++;
-            if (withscores && c->resp > 2) addReplyArrayLen(c,2);
             if (vstr == NULL) {
-                addReplyBulkLongLong(c,vlong);
+                handler->emitResultFromLongLong(handler, vlong, score);
             } else {
-                addReplyBulkCBuffer(c,vstr,vlen);
+                handler->emitResultFromCBuffer(handler, vstr, vlen, score);
             }
-            if (withscores) addReplyDouble(c,score);
 
             /* Move to next node */
             if (reverse) {
@@ -3031,22 +3148,11 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
 
         /* If reversed, get the last node in range as starting point. */
         if (reverse) {
-            ln = zslLastInRange(zsl,&range);
+            ln = zslLastInRange(zsl,range);
         } else {
-            ln = zslFirstInRange(zsl,&range);
+            ln = zslFirstInRange(zsl,range);
         }
 
-        /* No "first" element in the specified interval. */
-        if (ln == NULL) {
-            addReply(c,shared.emptyarray);
-            return;
-        }
-
-        /* We don't know in advance how many matching elements there are in the
-         * list, so we push this object that will represent the multi-bulk
-         * length in the output buffer, and will "fix" it later */
-        replylen = addReplyDeferredLen(c);
-
         /* If there is an offset, just traverse the number of elements without
          * checking the score because that is done in the next loop. */
         while (ln && offset--) {
@@ -3060,15 +3166,14 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
         while (ln && limit--) {
             /* Abort when the node is no longer in range. */
             if (reverse) {
-                if (!zslValueGteMin(ln->score,&range)) break;
+                if (!zslValueGteMin(ln->score,range)) break;
             } else {
-                if (!zslValueLteMax(ln->score,&range)) break;
+                if (!zslValueLteMax(ln->score,range)) break;
             }
 
             rangelen++;
-            if (withscores && c->resp > 2) addReplyArrayLen(c,2);
-            addReplyBulkCBuffer(c,ln->ele,sdslen(ln->ele));
-            if (withscores) addReplyDouble(c,ln->score);
+            handler->emitResultFromCBuffer(handler, ln->ele, sdslen(ln->ele),
+              ((withscores) ? ln->score : ln->score));
 
             /* Move to next node */
             if (reverse) {
@@ -3081,16 +3186,21 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
         serverPanic("Unknown sorted set encoding");
     }
 
-    if (withscores && c->resp == 2) rangelen *= 2;
-    setDeferredArrayLen(c, replylen, rangelen);
+    handler->finalizeResultEmission(handler, rangelen);
 }
 
+/* ZRANGEBYSCORE <key> <min> <max> [WITHSCORES] [LIMIT offset count] */
 void zrangebyscoreCommand(client *c) {
-    genericZrangebyscoreCommand(c,0);
+    zrange_result_handler handler;
+    zrangeResultHandlerInit(&handler, c, ZRANGE_CONSUMER_TYPE_CLIENT);
+    zrangeGenericCommand(&handler, 1, 0, ZRANGE_SCORE, ZRANGE_DIRECTION_FORWARD);
 }
 
+/* ZREVRANGEBYSCORE <key> <min> <max> [WITHSCORES] [LIMIT offset count] */
 void zrevrangebyscoreCommand(client *c) {
-    genericZrangebyscoreCommand(c,1);
+    zrange_result_handler handler;
+    zrangeResultHandlerInit(&handler, c, ZRANGE_CONSUMER_TYPE_CLIENT);
+    zrangeGenericCommand(&handler, 1, 0, ZRANGE_SCORE, ZRANGE_DIRECTION_REVERSE);
 }
 
 void zcountCommand(client *c) {
@@ -3250,58 +3360,14 @@ void zlexcountCommand(client *c) {
 }
 
 /* This command implements ZRANGEBYLEX, ZREVRANGEBYLEX. */
-void genericZrangebylexCommand(client *c, int reverse) {
-    zlexrangespec range;
-    robj *key = c->argv[1];
-    robj *zobj;
-    long offset = 0, limit = -1;
+void genericZrangebylexCommand(zrange_result_handler *handler,
+    zlexrangespec *range, robj *zobj, int withscores, long offset, long limit,
+    int reverse)
+{
+    client *c = handler->client;
     unsigned long rangelen = 0;
-    void *replylen = NULL;
-    int minidx, maxidx;
 
-    /* Parse the range arguments. */
-    if (reverse) {
-        /* Range is given as [max,min] */
-        maxidx = 2; minidx = 3;
-    } else {
-        /* Range is given as [min,max] */
-        minidx = 2; maxidx = 3;
-    }
-
-    if (zslParseLexRange(c->argv[minidx],c->argv[maxidx],&range) != C_OK) {
-        addReplyError(c,"min or max not valid string range item");
-        return;
-    }
-
-    /* Parse optional extra arguments. Note that ZCOUNT will exactly have
-     * 4 arguments, so we'll never enter the following code path. */
-    if (c->argc > 4) {
-        int remaining = c->argc - 4;
-        int pos = 4;
-
-        while (remaining) {
-            if (remaining >= 3 && !strcasecmp(c->argv[pos]->ptr,"limit")) {
-                if ((getLongFromObjectOrReply(c, c->argv[pos+1], &offset, NULL) != C_OK) ||
-                    (getLongFromObjectOrReply(c, c->argv[pos+2], &limit, NULL) != C_OK)) {
-                    zslFreeLexRange(&range);
-                    return;
-                }
-                pos += 3; remaining -= 3;
-            } else {
-                zslFreeLexRange(&range);
-                addReplyErrorObject(c,shared.syntaxerr);
-                return;
-            }
-        }
-    }
-
-    /* Ok, lookup the key and get the range */
-    if ((zobj = lookupKeyReadOrReply(c,key,shared.emptyarray)) == NULL ||
-        checkType(c,zobj,OBJ_ZSET))
-    {
-        zslFreeLexRange(&range);
-        return;
-    }
+    handler->beginResultEmission(handler);
 
     if (zobj->encoding == OBJ_ENCODING_ZIPLIST) {
         unsigned char *zl = zobj->ptr;
@@ -3312,26 +3378,14 @@ void genericZrangebylexCommand(client *c, int reverse) {
 
         /* If reversed, get the last node in range as starting point. */
         if (reverse) {
-            eptr = zzlLastInLexRange(zl,&range);
+            eptr = zzlLastInLexRange(zl,range);
         } else {
-            eptr = zzlFirstInLexRange(zl,&range);
-        }
-
-        /* No "first" element in the specified interval. */
-        if (eptr == NULL) {
-            addReply(c,shared.emptyarray);
-            zslFreeLexRange(&range);
-            return;
+            eptr = zzlFirstInLexRange(zl,range);
         }
 
         /* Get score pointer for the first element. */
-        serverAssertWithInfo(c,zobj,eptr != NULL);
-        sptr = ziplistNext(zl,eptr);
-
-        /* We don't know in advance how many matching elements there are in the
-         * list, so we push this object that will represent the multi-bulk
-         * length in the output buffer, and will "fix" it later */
-        replylen = addReplyDeferredLen(c);
+        if (eptr)
+            sptr = ziplistNext(zl,eptr);
 
         /* If there is an offset, just traverse the number of elements without
          * checking the score because that is done in the next loop. */
@@ -3344,11 +3398,15 @@ void genericZrangebylexCommand(client *c, int reverse) {
         }
 
         while (eptr && limit--) {
+            double score = 0;
+            if (withscores) /* don't bother to extract the score if it's gonna be ignored. */
+                score = zzlGetScore(sptr);
+
             /* Abort when the node is no longer in range. */
             if (reverse) {
-                if (!zzlLexValueGteMin(eptr,&range)) break;
+                if (!zzlLexValueGteMin(eptr,range)) break;
             } else {
-                if (!zzlLexValueLteMax(eptr,&range)) break;
+                if (!zzlLexValueLteMax(eptr,range)) break;
             }
 
             /* We know the element exists, so ziplistGet should always
@@ -3357,9 +3415,9 @@ void genericZrangebylexCommand(client *c, int reverse) {
 
             rangelen++;
             if (vstr == NULL) {
-                addReplyBulkLongLong(c,vlong);
+                handler->emitResultFromLongLong(handler, vlong, score);
             } else {
-                addReplyBulkCBuffer(c,vstr,vlen);
+                handler->emitResultFromCBuffer(handler, vstr, vlen, score);
             }
 
             /* Move to next node */
@@ -3376,23 +3434,11 @@ void genericZrangebylexCommand(client *c, int reverse) {
 
         /* If reversed, get the last node in range as starting point. */
         if (reverse) {
-            ln = zslLastInLexRange(zsl,&range);
+            ln = zslLastInLexRange(zsl,range);
         } else {
-            ln = zslFirstInLexRange(zsl,&range);
+            ln = zslFirstInLexRange(zsl,range);
         }
 
-        /* No "first" element in the specified interval. */
-        if (ln == NULL) {
-            addReply(c,shared.emptyarray);
-            zslFreeLexRange(&range);
-            return;
-        }
-
-        /* We don't know in advance how many matching elements there are in the
-         * list, so we push this object that will represent the multi-bulk
-         * length in the output buffer, and will "fix" it later */
-        replylen = addReplyDeferredLen(c);
-
         /* If there is an offset, just traverse the number of elements without
          * checking the score because that is done in the next loop. */
         while (ln && offset--) {
@@ -3406,13 +3452,13 @@ void genericZrangebylexCommand(client *c, int reverse) {
         while (ln && limit--) {
             /* Abort when the node is no longer in range. */
             if (reverse) {
-                if (!zslLexValueGteMin(ln->ele,&range)) break;
+                if (!zslLexValueGteMin(ln->ele,range)) break;
             } else {
-                if (!zslLexValueLteMax(ln->ele,&range)) break;
+                if (!zslLexValueLteMax(ln->ele,range)) break;
             }
 
             rangelen++;
-            addReplyBulkCBuffer(c,ln->ele,sdslen(ln->ele));
+            handler->emitResultFromCBuffer(handler, ln->ele, sdslen(ln->ele), ln->score);
 
             /* Move to next node */
             if (reverse) {
@@ -3425,16 +3471,171 @@ void genericZrangebylexCommand(client *c, int reverse) {
         serverPanic("Unknown sorted set encoding");
     }
 
-    zslFreeLexRange(&range);
-    setDeferredArrayLen(c, replylen, rangelen);
+    handler->finalizeResultEmission(handler, rangelen);
 }
 
+/* ZRANGEBYLEX <key> <min> <max> [LIMIT offset count] */
 void zrangebylexCommand(client *c) {
-    genericZrangebylexCommand(c,0);
+    zrange_result_handler handler;
+    zrangeResultHandlerInit(&handler, c, ZRANGE_CONSUMER_TYPE_CLIENT);
+    zrangeGenericCommand(&handler, 1, 0, ZRANGE_LEX, ZRANGE_DIRECTION_FORWARD);
 }
 
+/* ZREVRANGEBYLEX <key> <min> <max> [LIMIT offset count] */
 void zrevrangebylexCommand(client *c) {
-    genericZrangebylexCommand(c,1);
+    zrange_result_handler handler;
+    zrangeResultHandlerInit(&handler, c, ZRANGE_CONSUMER_TYPE_CLIENT);
+    zrangeGenericCommand(&handler, 1, 0, ZRANGE_LEX, ZRANGE_DIRECTION_REVERSE);
+}
+
+/**
+ * This function handles ZRANGE and ZRANGESTORE, and also the deprecated
+ * Z[REV]RANGE[BYPOS|BYLEX] commands.
+ *
+ * The simple ZRANGE and ZRANGESTORE can take _AUTO in rangetype and direction,
+ * other command pass explicit value.
+ *
+ * The argc_start points to the src key argument, so following syntax is like:
+ * <src> <min> <max> [BYSCORE | BYLEX] [REV] [WITHSCORES] [LIMIT offset count]
+ */
+void zrangeGenericCommand(zrange_result_handler *handler, int argc_start, int store,
+                          zrange_type rangetype, zrange_direction direction)
+{
+    client *c = handler->client;
+    robj *key = c->argv[argc_start];
+    robj *zobj;
+    zrangespec range;
+    zlexrangespec lexrange;
+    int minidx = argc_start + 1;
+    int maxidx = argc_start + 2;
+
+    /* Options common to all */
+    long opt_start = 0;
+    long opt_end = 0;
+    int opt_withscores = 0;
+    long opt_offset = 0;
+    long opt_limit = -1;
+
+    /* Step 1: Skip the <src> <min> <max> args and parse remaining optional arguments. */
+    for (int j=argc_start + 3; j < c->argc; j++) {
+        int leftargs = c->argc-j-1;
+        if (!store && !strcasecmp(c->argv[j]->ptr,"withscores")) {
+            opt_withscores = 1;
+        } else if (!strcasecmp(c->argv[j]->ptr,"limit") && leftargs >= 2) {
+            if ((getLongFromObjectOrReply(c, c->argv[j+1], &opt_offset, NULL) != C_OK) ||
+                (getLongFromObjectOrReply(c, c->argv[j+2], &opt_limit, NULL) != C_OK))
+            {
+                return;
+            }
+            j += 2;
+        } else if (direction == ZRANGE_DIRECTION_AUTO &&
+                   !strcasecmp(c->argv[j]->ptr,"rev"))
+        {
+            direction = ZRANGE_DIRECTION_REVERSE;
+        } else if (rangetype == ZRANGE_AUTO &&
+                   !strcasecmp(c->argv[j]->ptr,"bylex"))
+        {
+            rangetype = ZRANGE_LEX;
+        } else if (rangetype == ZRANGE_AUTO &&
+                   !strcasecmp(c->argv[j]->ptr,"byscore"))
+        {
+            rangetype = ZRANGE_SCORE;
+        } else {
+            addReplyErrorObject(c,shared.syntaxerr);
+            return;
+        }
+    }
+
+    /* Use defaults if not overriden by arguments. */
+    if (direction == ZRANGE_DIRECTION_AUTO)
+        direction = ZRANGE_DIRECTION_FORWARD;
+    if (rangetype == ZRANGE_AUTO)
+        rangetype = ZRANGE_RANK;
+
+    /* Check for conflicting arguments. */
+    if (opt_limit != -1 && rangetype == ZRANGE_RANK) {
+        addReplyError(c,"syntax error, LIMIT is only supported in combination with either BYSCORE or BYLEX");
+        return;
+    }
+    if (opt_withscores && rangetype == ZRANGE_LEX) {
+        addReplyError(c,"syntax error, WITHSCORES not supported in combination with BYLEX");
+        return;
+    }
+
+    if (direction == ZRANGE_DIRECTION_REVERSE &&
+        ((ZRANGE_SCORE == rangetype) || (ZRANGE_LEX == rangetype)))
+    {
+        /* Range is given as [max,min] */
+        int tmp = maxidx;
+        maxidx = minidx;
+        minidx = tmp;
+    }
+
+    /* Step 2: Parse the range. */
+    switch (rangetype) {
+    case ZRANGE_AUTO:
+    case ZRANGE_RANK:
+        /* Z[REV]RANGE, ZRANGESTORE [REV]RANGE */
+        if ((getLongFromObjectOrReply(c, c->argv[minidx], &opt_start,NULL) != C_OK) ||
+            (getLongFromObjectOrReply(c, c->argv[maxidx], &opt_end,NULL) != C_OK))
+        {
+            return;
+        }
+        break;
+
+    case ZRANGE_SCORE:
+        /* Z[REV]RANGEBYSCORE, ZRANGESTORE [REV]RANGEBYSCORE */
+        if (zslParseRange(c->argv[minidx], c->argv[maxidx], &range) != C_OK) {
+            addReplyError(c, "min or max is not a float");
+            return;
+        }
+        break;
+
+    case ZRANGE_LEX:
+        /* Z[REV]RANGEBYLEX, ZRANGESTORE [REV]RANGEBYLEX */
+        if (zslParseLexRange(c->argv[minidx], c->argv[maxidx], &lexrange) != C_OK) {
+            addReplyError(c, "min or max not valid string range item");
+            return;
+        }
+        break;
+    }
+
+    if (opt_withscores || store) {
+        zrangeResultHandlerScoreEmissionEnable(handler);
+    }
+
+    /* Step 3: Lookup the key and get the range. */
+    if (((zobj = lookupKeyReadOrReply(c, key, shared.emptyarray)) == NULL)
+        || checkType(c, zobj, OBJ_ZSET)) {
+        goto cleanup;
+    }
+
+    /* Step 4: Pass this to the command-specific handler. */
+    switch (rangetype) {
+    case ZRANGE_AUTO:
+    case ZRANGE_RANK:
+        genericZrangebyrankCommand(handler, zobj, opt_start, opt_end,
+            opt_withscores || store, direction == ZRANGE_DIRECTION_REVERSE);
+        break;
+
+    case ZRANGE_SCORE:
+        genericZrangebyscoreCommand(handler, &range, zobj, opt_withscores || store,
+            opt_offset, opt_limit, direction == ZRANGE_DIRECTION_REVERSE);
+        break;
+
+    case ZRANGE_LEX:
+        genericZrangebylexCommand(handler, &lexrange, zobj, opt_withscores || store,
+            opt_offset, opt_limit, direction == ZRANGE_DIRECTION_REVERSE);
+        break;
+    }
+
+    /* Instead of returning here, we'll just fall-through the clean-up. */
+
+cleanup:
+
+    if (rangetype == ZRANGE_LEX) {
+        zslFreeLexRange(&lexrange);
+    }
 }
 
 void zcardCommand(client *c) {
diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl
index c1406797b..8318ebb63 100644
--- a/tests/unit/type/zset.tcl
+++ b/tests/unit/type/zset.tcl
@@ -1472,4 +1472,86 @@ start_server {tags {"zset"}} {
         }
         r config set zset-max-ziplist-entries $original_max
     }
+
+    test {ZRANGESTORE basic} {
+        r flushall
+        r zadd z1 1 a 2 b 3 c 4 d
+        set res [r zrangestore z2 z1 0 -1]
+        assert_equal $res 4
+        r zrange z2 0 -1 withscores
+    } {a 1 b 2 c 3 d 4}
+
+    test {ZRANGESTORE range} {
+        set res [r zrangestore z2 z1 1 2]
+        assert_equal $res 2
+        r zrange z2 0 -1 withscores
+    } {b 2 c 3}
+
+    test {ZRANGESTORE BYLEX} {
+        set res [r zrangestore z2 z1 \[b \[c BYLEX]
+        assert_equal $res 2
+        r zrange z2 0 -1 withscores
+    } {b 2 c 3}
+
+    test {ZRANGESTORE BYSCORE} {
+        set res [r zrangestore z2 z1 1 2 BYSCORE]
+        assert_equal $res 2
+        r zrange z2 0 -1 withscores
+    } {a 1 b 2}
+
+    test {ZRANGESTORE BYSCORE LIMIT} {
+        set res [r zrangestore z2 z1 0 5 BYSCORE LIMIT 0 2]
+        assert_equal $res 2
+        r zrange z2 0 -1 withscores
+    } {a 1 b 2}
+
+    test {ZRANGESTORE BYSCORE REV LIMIT} {
+        set res [r zrangestore z2 z1 5 0 BYSCORE REV LIMIT 0 2]
+        assert_equal $res 2
+        r zrange z2 0 -1 withscores
+    } {c 3 d 4}
+
+    test {ZRANGE BYSCORE REV LIMIT} {
+        r zrange z1 5 0 BYSCORE REV LIMIT 0 2 WITHSCORES
+    } {d 4 c 3}
+
+    test {ZRANGESTORE - empty range} {
+        set res [r zrangestore z2 z1 5 6]
+        assert_equal $res 0
+        r exists z2
+    } {0}
+
+    test {ZRANGESTORE BYLEX - empty range} {
+        set res [r zrangestore z2 z1 \[f \[g BYLEX]
+        assert_equal $res 0
+        r exists z2
+    } {0}
+
+    test {ZRANGESTORE BYSCORE - empty range} {
+        set res [r zrangestore z2 z1 5 6 BYSCORE]
+        assert_equal $res 0
+        r exists z2
+    } {0}
+
+    test {ZRANGE BYLEX} {
+        r zrange z1 \[b \[c BYLEX
+    } {b c}
+
+    test {ZRANGESTORE invalid syntax} {
+        catch {r zrangestore z2 z1 0 -1 limit 1 2} err
+        assert_match "*syntax*" $err
+        catch {r zrangestore z2 z1 0 -1 WITHSCORES} err
+        assert_match "*syntax*" $err
+    }
+
+    test {ZRANGE invalid syntax} {
+        catch {r zrange z1 0 -1 limit 1 2} err
+        assert_match "*syntax*" $err
+        catch {r zrange z1 0 -1 BYLEX WITHSCORES} err
+        assert_match "*syntax*" $err
+        catch {r zrevrange z1 0 -1 BYSCORE} err
+        assert_match "*syntax*" $err
+        catch {r zrangebyscore z1 0 -1 REV} err
+        assert_match "*syntax*" $err
+    }
 }

From f9dacf8aacc8af7fdcb41bc9f927b42390557f7d Mon Sep 17 00:00:00 2001
From: YaacovHazan <yaacov.hazan@redislabs.com>
Date: Wed, 16 Dec 2020 15:14:04 +0200
Subject: [PATCH 0949/1098] Refactory fork child related infra, Unify child pid

This is a refactory commit, isn't suppose to have any actual impact.
it does the following:
- keep just one server struct fork child pid variable instead of 3
- have one server struct variable indicating the purpose of the current fork
  child.
- redisFork is now responsible of updating the server struct with the pid,
  which means it can be the one that calls updateDictResizePolicy
- move child info pipe handling into redisFork instead of having them
  repeated outside
- there are two classes of fork purposes, mutually exclusive group (AOF, RDB,
  Module), and one that can create several forks to coexist in parallel (LDB,
  but maybe Modules some day too, Module API allows for that).
- minor fix to killRDBChild:
  unlike killAppendOnlyChild and TerminateModuleForkChild, the killRDBChild
  doesn't clear the pid variable or call wait4, so checkChildrenDone does
  the cleanup for it.
  This commit removes the explicit calls to rdbRemoveTempFile, closeChildInfoPipe,
  updateDictResizePolicy, which didn't do any harm, but where unnecessary.
---
 src/aof.c         |  32 ++++++-------
 src/db.c          |   2 +-
 src/module.c      |  27 ++++-------
 src/networking.c  |   2 +-
 src/rdb.c         |  26 ++++-------
 src/replication.c |  10 ++--
 src/server.c      | 113 ++++++++++++++++++++++++++++++++--------------
 src/server.h      |   7 +--
 8 files changed, 122 insertions(+), 97 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index 2accc781e..928e1c4d1 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -213,22 +213,20 @@ void aof_background_fsync(int fd) {
 void killAppendOnlyChild(void) {
     int statloc;
     /* No AOFRW child? return. */
-    if (server.aof_child_pid == -1) return;
+    if (server.child_type != CHILD_TYPE_AOF) return;
     /* Kill AOFRW child, wait for child exit. */
     serverLog(LL_NOTICE,"Killing running AOF rewrite child: %ld",
-        (long) server.aof_child_pid);
-    if (kill(server.aof_child_pid,SIGUSR1) != -1) {
-        while(wait3(&statloc,0,NULL) != server.aof_child_pid);
+        (long) server.child_pid);
+    if (kill(server.child_pid,SIGUSR1) != -1) {
+        while(wait3(&statloc,0,NULL) != server.child_pid);
     }
     /* Reset the buffer accumulating changes while the child saves. */
     aofRewriteBufferReset();
-    aofRemoveTempFile(server.aof_child_pid);
-    server.aof_child_pid = -1;
+    aofRemoveTempFile(server.child_pid);
+    resetChildState();
     server.aof_rewrite_time_start = -1;
     /* Close pipes used for IPC between the two processes. */
     aofClosePipes();
-    closeChildInfoPipe();
-    updateDictResizePolicy();
 }
 
 /* Called when the user switches from "appendonly yes" to "appendonly no"
@@ -265,14 +263,14 @@ int startAppendOnly(void) {
             strerror(errno));
         return C_ERR;
     }
-    if (hasActiveChildProcess() && server.aof_child_pid == -1) {
+    if (hasActiveChildProcess() && server.child_type != CHILD_TYPE_AOF) {
         server.aof_rewrite_scheduled = 1;
         serverLog(LL_WARNING,"AOF was enabled but there is already another background operation. An AOF background was scheduled to start when possible.");
     } else {
         /* If there is a pending AOF rewrite, we need to switch it off and
          * start a new one: the old one cannot be reused because it is not
          * accumulating the AOF buffer. */
-        if (server.aof_child_pid != -1) {
+        if (server.child_type == CHILD_TYPE_AOF) {
             serverLog(LL_WARNING,"AOF was enabled but there is already an AOF rewriting in background. Stopping background AOF and starting a rewrite now.");
             killAppendOnlyChild();
         }
@@ -646,7 +644,7 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a
      * accumulate the differences between the child DB and the current one
      * in a buffer, so that when the child process will do its work we
      * can append the differences to the new append only file. */
-    if (server.aof_child_pid != -1)
+    if (server.child_type == CHILD_TYPE_AOF)
         aofRewriteBufferAppend((unsigned char*)buf,sdslen(buf));
 
     sdsfree(buf);
@@ -1703,7 +1701,6 @@ int rewriteAppendOnlyFileBackground(void) {
 
     if (hasActiveChildProcess()) return C_ERR;
     if (aofCreatePipes() != C_OK) return C_ERR;
-    openChildInfoPipe();
     if ((childpid = redisFork(CHILD_TYPE_AOF)) == 0) {
         char tmpfile[256];
 
@@ -1720,7 +1717,6 @@ int rewriteAppendOnlyFileBackground(void) {
     } else {
         /* Parent */
         if (childpid == -1) {
-            closeChildInfoPipe();
             serverLog(LL_WARNING,
                 "Can't rewrite append only file in background: fork: %s",
                 strerror(errno));
@@ -1731,8 +1727,7 @@ int rewriteAppendOnlyFileBackground(void) {
             "Background append only file rewriting started by pid %ld",(long) childpid);
         server.aof_rewrite_scheduled = 0;
         server.aof_rewrite_time_start = time(NULL);
-        server.aof_child_pid = childpid;
-        updateDictResizePolicy();
+
         /* We set appendseldb to -1 in order to force the next call to the
          * feedAppendOnlyFile() to issue a SELECT command, so the differences
          * accumulated by the parent into server.aof_rewrite_buf will start
@@ -1745,7 +1740,7 @@ int rewriteAppendOnlyFileBackground(void) {
 }
 
 void bgrewriteaofCommand(client *c) {
-    if (server.aof_child_pid != -1) {
+    if (server.child_type == CHILD_TYPE_AOF) {
         addReplyError(c,"Background append only file rewriting already in progress");
     } else if (hasActiveChildProcess()) {
         server.aof_rewrite_scheduled = 1;
@@ -1803,7 +1798,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
          * rewritten AOF. */
         latencyStartMonitor(latency);
         snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof",
-            (int)server.aof_child_pid);
+            (int)server.child_pid);
         newfd = open(tmpfile,O_WRONLY|O_APPEND);
         if (newfd == -1) {
             serverLog(LL_WARNING,
@@ -1931,8 +1926,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
 cleanup:
     aofClosePipes();
     aofRewriteBufferReset();
-    aofRemoveTempFile(server.aof_child_pid);
-    server.aof_child_pid = -1;
+    aofRemoveTempFile(server.child_pid);
     server.aof_rewrite_time_last = time(NULL)-server.aof_rewrite_time_start;
     server.aof_rewrite_time_start = -1;
     /* Schedule a new rewrite if we are waiting for it to switch the AOF ON. */
diff --git a/src/db.c b/src/db.c
index 62ffda101..c4b48fb0e 100644
--- a/src/db.c
+++ b/src/db.c
@@ -617,7 +617,7 @@ int getFlushCommandFlags(client *c, int *flags) {
 /* Flushes the whole server data set. */
 void flushAllDataAndResetRDB(int flags) {
     server.dirty += emptyDb(-1,flags,NULL);
-    if (server.rdb_child_pid != -1) killRDBChild();
+    if (server.child_type == CHILD_TYPE_RDB) killRDBChild();
     if (server.saveparamslen > 0) {
         /* Normally rdbSave() will reset dirty, but we don't want this here
          * as otherwise FLUSHALL will not be replicated nor put into the AOF. */
diff --git a/src/module.c b/src/module.c
index 56ff63869..2de6bee2f 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7065,23 +7065,16 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc
  */
 int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) {
     pid_t childpid;
-    if (hasActiveChildProcess()) {
-        return -1;
-    }
 
-    openChildInfoPipe();
     if ((childpid = redisFork(CHILD_TYPE_MODULE)) == 0) {
         /* Child */
         redisSetProcTitle("redis-module-fork");
     } else if (childpid == -1) {
-        closeChildInfoPipe();
         serverLog(LL_WARNING,"Can't fork for module: %s", strerror(errno));
     } else {
         /* Parent */
-        server.module_child_pid = childpid;
         moduleForkInfo.done_handler = cb;
         moduleForkInfo.done_handler_user_data = user_data;
-        updateDictResizePolicy();
         serverLog(LL_VERBOSE, "Module fork started pid: %ld ", (long) childpid);
     }
     return childpid;
@@ -7101,22 +7094,20 @@ int RM_ExitFromChild(int retcode) {
  * child or the pid does not match, return C_ERR without doing anything. */
 int TerminateModuleForkChild(int child_pid, int wait) {
     /* Module child should be active and pid should match. */
-    if (server.module_child_pid == -1 ||
-        server.module_child_pid != child_pid) return C_ERR;
+    if (server.child_type != CHILD_TYPE_MODULE ||
+        server.child_pid != child_pid) return C_ERR;
 
     int statloc;
     serverLog(LL_VERBOSE,"Killing running module fork child: %ld",
-        (long) server.module_child_pid);
-    if (kill(server.module_child_pid,SIGUSR1) != -1 && wait) {
-        while(wait4(server.module_child_pid,&statloc,0,NULL) !=
-              server.module_child_pid);
+        (long) server.child_pid);
+    if (kill(server.child_pid,SIGUSR1) != -1 && wait) {
+        while(wait4(server.child_pid,&statloc,0,NULL) !=
+              server.child_pid);
     }
     /* Reset the buffer accumulating changes while the child saves. */
-    server.module_child_pid = -1;
+    resetChildState();
     moduleForkInfo.done_handler = NULL;
     moduleForkInfo.done_handler_user_data = NULL;
-    closeChildInfoPipe();
-    updateDictResizePolicy();
     return C_OK;
 }
 
@@ -7133,12 +7124,12 @@ int RM_KillForkChild(int child_pid) {
 void ModuleForkDoneHandler(int exitcode, int bysignal) {
     serverLog(LL_NOTICE,
         "Module fork exited pid: %ld, retcode: %d, bysignal: %d",
-        (long) server.module_child_pid, exitcode, bysignal);
+        (long) server.child_pid, exitcode, bysignal);
     if (moduleForkInfo.done_handler) {
         moduleForkInfo.done_handler(exitcode, bysignal,
             moduleForkInfo.done_handler_user_data);
     }
-    server.module_child_pid = -1;
+
     moduleForkInfo.done_handler = NULL;
     moduleForkInfo.done_handler_user_data = NULL;
 }
diff --git a/src/networking.c b/src/networking.c
index acb1b27cb..e71e39624 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1358,7 +1358,7 @@ void freeClient(client *c) {
          * to keep data safe and we may delay configured 'save' for full sync. */
         if (server.saveparamslen == 0 &&
             c->replstate == SLAVE_STATE_WAIT_BGSAVE_END &&
-            server.rdb_child_pid != -1 &&
+            server.child_type == CHILD_TYPE_RDB &&
             server.rdb_child_type == RDB_CHILD_TYPE_DISK &&
             anyOtherSlaveWaitRdb(c) == 0)
         {
diff --git a/src/rdb.c b/src/rdb.c
index abcfeadd7..1281948d1 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1410,7 +1410,6 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
 
     server.dirty_before_bgsave = server.dirty;
     server.lastbgsave_try = time(NULL);
-    openChildInfoPipe();
 
     if ((childpid = redisFork(CHILD_TYPE_RDB)) == 0) {
         int retval;
@@ -1426,7 +1425,6 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
     } else {
         /* Parent */
         if (childpid == -1) {
-            closeChildInfoPipe();
             server.lastbgsave_status = C_ERR;
             serverLog(LL_WARNING,"Can't save in background: fork: %s",
                 strerror(errno));
@@ -1434,9 +1432,7 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
         }
         serverLog(LL_NOTICE,"Background saving started by pid %ld",(long) childpid);
         server.rdb_save_time_start = time(NULL);
-        server.rdb_child_pid = childpid;
         server.rdb_child_type = RDB_CHILD_TYPE_DISK;
-        updateDictResizePolicy();
         return C_OK;
     }
     return C_OK; /* unreached */
@@ -2654,7 +2650,7 @@ static void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal) {
         serverLog(LL_WARNING,
             "Background saving terminated by signal %d", bysignal);
         latencyStartMonitor(latency);
-        rdbRemoveTempFile(server.rdb_child_pid, 0);
+        rdbRemoveTempFile(server.child_pid, 0);
         latencyEndMonitor(latency);
         latencyAddSampleIfNeeded("rdb-unlink-temp-file",latency);
         /* SIGUSR1 is whitelisted, so we have a way to kill a child without
@@ -2706,7 +2702,6 @@ void backgroundSaveDoneHandler(int exitcode, int bysignal) {
         break;
     }
 
-    server.rdb_child_pid = -1;
     server.rdb_child_type = RDB_CHILD_TYPE_NONE;
     server.rdb_save_time_last = time(NULL)-server.rdb_save_time_start;
     server.rdb_save_time_start = -1;
@@ -2719,10 +2714,13 @@ void backgroundSaveDoneHandler(int exitcode, int bysignal) {
  * the child did not exit for an error, but because we wanted), and performs
  * the cleanup needed. */
 void killRDBChild(void) {
-    kill(server.rdb_child_pid,SIGUSR1);
-    rdbRemoveTempFile(server.rdb_child_pid, 0);
-    closeChildInfoPipe();
-    updateDictResizePolicy();
+    kill(server.child_pid, SIGUSR1);
+    /* Because we are not using here wait4 (like we have in killAppendOnlyChild
+     * and TerminateModuleForkChild), all the cleanup operations is done by
+     * checkChildrenDone, that later will find that the process killed.
+     * This includes:
+     * - resetChildState
+     * - rdbRemoveTempFile */
 }
 
 /* Spawn an RDB child that writes the RDB to the sockets of the slaves
@@ -2773,7 +2771,6 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
     }
 
     /* Create the child process. */
-    openChildInfoPipe();
     if ((childpid = redisFork(CHILD_TYPE_RDB)) == 0) {
         /* Child */
         int retval, dummy;
@@ -2824,14 +2821,11 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
             server.rdb_pipe_conns = NULL;
             server.rdb_pipe_numconns = 0;
             server.rdb_pipe_numconns_writing = 0;
-            closeChildInfoPipe();
         } else {
             serverLog(LL_NOTICE,"Background RDB transfer started by pid %ld",
                 (long) childpid);
             server.rdb_save_time_start = time(NULL);
-            server.rdb_child_pid = childpid;
             server.rdb_child_type = RDB_CHILD_TYPE_SOCKET;
-            updateDictResizePolicy();
             close(rdb_pipe_write); /* close write in parent so that it can detect the close on the child. */
             if (aeCreateFileEvent(server.el, server.rdb_pipe_read, AE_READABLE, rdbPipeReadHandler,NULL) == AE_ERR) {
                 serverPanic("Unrecoverable error creating server.rdb_pipe_read file event.");
@@ -2843,7 +2837,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
 }
 
 void saveCommand(client *c) {
-    if (server.rdb_child_pid != -1) {
+    if (server.child_type == CHILD_TYPE_RDB) {
         addReplyError(c,"Background save already in progress");
         return;
     }
@@ -2874,7 +2868,7 @@ void bgsaveCommand(client *c) {
     rdbSaveInfo rsi, *rsiptr;
     rsiptr = rdbPopulateSaveInfo(&rsi);
 
-    if (server.rdb_child_pid != -1) {
+    if (server.child_type == CHILD_TYPE_RDB) {
         addReplyError(c,"Background save already in progress");
     } else if (hasActiveChildProcess()) {
         if (schedule) {
diff --git a/src/replication.c b/src/replication.c
index 20eb83e72..361bdb626 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -786,7 +786,7 @@ void syncCommand(client *c) {
     }
 
     /* CASE 1: BGSAVE is in progress, with disk target. */
-    if (server.rdb_child_pid != -1 &&
+    if (server.child_type == CHILD_TYPE_RDB &&
         server.rdb_child_type == RDB_CHILD_TYPE_DISK)
     {
         /* Ok a background save is in progress. Let's check if it is a good
@@ -816,7 +816,7 @@ void syncCommand(client *c) {
         }
 
     /* CASE 2: BGSAVE is in progress, with socket target. */
-    } else if (server.rdb_child_pid != -1 &&
+    } else if (server.child_type == CHILD_TYPE_RDB &&
                server.rdb_child_type == RDB_CHILD_TYPE_SOCKET)
     {
         /* There is an RDB child process but it is writing directly to
@@ -914,7 +914,7 @@ void replconfCommand(client *c) {
              * There's a chance the ACK got to us before we detected that the
              * bgsave is done (since that depends on cron ticks), so run a
              * quick check first (instead of waiting for the next ACK. */
-            if (server.rdb_child_pid != -1 && c->replstate == SLAVE_STATE_WAIT_BGSAVE_END)
+            if (server.child_type == CHILD_TYPE_RDB && c->replstate == SLAVE_STATE_WAIT_BGSAVE_END)
                 checkChildrenDone();
             if (c->repl_put_online_on_ack && c->replstate == SLAVE_STATE_ONLINE)
                 putSlaveOnline(c);
@@ -1721,13 +1721,13 @@ void readSyncBulkPayload(connection *conn) {
         connRecvTimeout(conn,0);
     } else {
         /* Ensure background save doesn't overwrite synced data */
-        if (server.rdb_child_pid != -1) {
+        if (server.child_type == CHILD_TYPE_RDB) {
             serverLog(LL_NOTICE,
                 "Replica is about to load the RDB file received from the "
                 "master, but there is a pending RDB child running. "
                 "Killing process %ld and removing its temp file to avoid "
                 "any race",
-                    (long) server.rdb_child_pid);
+                (long) server.child_pid);
             killRDBChild();
         }
 
diff --git a/src/server.c b/src/server.c
index 6140b21ac..ee2a00986 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1559,12 +1559,32 @@ void updateDictResizePolicy(void) {
         dictDisableResize();
 }
 
+const char *strChildType(int type) {
+    switch(type) {
+        case CHILD_TYPE_RDB: return "RDB";
+        case CHILD_TYPE_AOF: return "AOF";
+        case CHILD_TYPE_LDB: return "LDB";
+        case CHILD_TYPE_MODULE: return "MODULE";
+        default: return "Unknown";
+    }
+}
+
 /* Return true if there are active children processes doing RDB saving,
  * AOF rewriting, or some side process spawned by a loaded module. */
 int hasActiveChildProcess() {
-    return server.rdb_child_pid != -1 ||
-           server.aof_child_pid != -1 ||
-           server.module_child_pid != -1;
+    return server.child_pid != -1;
+}
+
+void resetChildState() {
+    server.child_type = CHILD_TYPE_NONE;
+    server.child_pid = -1;
+    updateDictResizePolicy();
+    closeChildInfoPipe();
+}
+
+/* Return if child type is mutual exclusive with other fork children */
+int isMutuallyExclusiveChildType(int type) {
+    return type == CHILD_TYPE_RDB || type == CHILD_TYPE_AOF || type == CHILD_TYPE_MODULE;
 }
 
 /* Return true if this instance has persistence completely turned off:
@@ -1886,29 +1906,30 @@ void checkChildrenDone(void) {
 
         if (pid == -1) {
             serverLog(LL_WARNING,"wait3() returned an error: %s. "
-                "rdb_child_pid = %d, aof_child_pid = %d, module_child_pid = %d",
+                "child_type: %s, child_pid = %d",
                 strerror(errno),
-                (int) server.rdb_child_pid,
-                (int) server.aof_child_pid,
-                (int) server.module_child_pid);
-        } else if (pid == server.rdb_child_pid) {
-            backgroundSaveDoneHandler(exitcode,bysignal);
-            if (!bysignal && exitcode == 0) receiveChildInfo();
-        } else if (pid == server.aof_child_pid) {
-            backgroundRewriteDoneHandler(exitcode,bysignal);
-            if (!bysignal && exitcode == 0) receiveChildInfo();
-        } else if (pid == server.module_child_pid) {
-            ModuleForkDoneHandler(exitcode,bysignal);
+                strChildType(server.child_type),
+                (int) server.child_pid);
+        } else if (pid == server.child_pid) {
+            if (server.child_type == CHILD_TYPE_RDB) {
+                backgroundSaveDoneHandler(exitcode, bysignal);
+            } else if (server.child_type == CHILD_TYPE_AOF) {
+                backgroundRewriteDoneHandler(exitcode, bysignal);
+            } else if (server.child_type == CHILD_TYPE_MODULE) {
+                ModuleForkDoneHandler(exitcode, bysignal);
+            } else {
+                serverPanic("Unknown child type %d for child pid %d", server.child_type, server.child_pid);
+                exit(1);
+            }
             if (!bysignal && exitcode == 0) receiveChildInfo();
+            resetChildState();
         } else {
             if (!ldbRemoveChild(pid)) {
                 serverLog(LL_WARNING,
-                    "Warning, detected child with unmatched pid: %ld",
-                    (long)pid);
+                          "Warning, detected child with unmatched pid: %ld",
+                          (long) pid);
             }
         }
-        updateDictResizePolicy();
-        closeChildInfoPipe();
 
         /* start any pending forks immediately. */
         replicationStartPendingFork();
@@ -3087,9 +3108,8 @@ void initServer(void) {
     server.in_eval = 0;
     server.in_exec = 0;
     server.propagate_in_transaction = 0;
-    server.rdb_child_pid = -1;
-    server.aof_child_pid = -1;
-    server.module_child_pid = -1;
+    server.child_pid = -1;
+    server.child_type = CHILD_TYPE_NONE;
     server.rdb_child_type = RDB_CHILD_TYPE_NONE;
     server.rdb_pipe_conns = NULL;
     server.rdb_pipe_numconns = 0;
@@ -4056,25 +4076,28 @@ int prepareForShutdown(int flags) {
     /* Kill the saving child if there is a background saving in progress.
        We want to avoid race conditions, for instance our saving child may
        overwrite the synchronous saving did by SHUTDOWN. */
-    if (server.rdb_child_pid != -1) {
+    if (server.child_type == CHILD_TYPE_RDB) {
         serverLog(LL_WARNING,"There is a child saving an .rdb. Killing it!");
-        /* Note that, in killRDBChild, we call rdbRemoveTempFile that will
-         * do close fd(in order to unlink file actully) in background thread.
+        killRDBChild();
+        /* Note that, in killRDBChild normally has backgroundSaveDoneHandler
+         * doing it's cleanup, but in this case this code will not be reached,
+         * so we need to call rdbRemoveTempFile which will close fd(in order
+         * to unlink file actully) in background thread.
          * The temp rdb file fd may won't be closed when redis exits quickly,
          * but OS will close this fd when process exits. */
-        killRDBChild();
+        rdbRemoveTempFile(server.child_pid, 0);
     }
 
     /* Kill module child if there is one. */
-    if (server.module_child_pid != -1) {
+    if (server.child_type == CHILD_TYPE_MODULE) {
         serverLog(LL_WARNING,"There is a module fork child. Killing it!");
-        TerminateModuleForkChild(server.module_child_pid,0);
+        TerminateModuleForkChild(server.child_pid,0);
     }
 
     if (server.aof_state != AOF_OFF) {
         /* Kill the AOF saving child as the AOF we already have may be longer
          * but contains the full dataset anyway. */
-        if (server.aof_child_pid != -1) {
+        if (server.child_type == CHILD_TYPE_AOF) {
             /* If we have AOF enabled but haven't written the AOF yet, don't
              * shutdown or else the dataset will be lost. */
             if (server.aof_state == AOF_WAIT_REWRITE) {
@@ -4613,23 +4636,23 @@ sds genRedisInfoString(const char *section) {
             "module_fork_last_cow_size:%zu\r\n",
             server.loading,
             server.dirty,
-            server.rdb_child_pid != -1,
+            server.child_type == CHILD_TYPE_RDB,
             (intmax_t)server.lastsave,
             (server.lastbgsave_status == C_OK) ? "ok" : "err",
             (intmax_t)server.rdb_save_time_last,
-            (intmax_t)((server.rdb_child_pid == -1) ?
+            (intmax_t)((server.child_type != CHILD_TYPE_RDB) ?
                 -1 : time(NULL)-server.rdb_save_time_start),
             server.stat_rdb_cow_bytes,
             server.aof_state != AOF_OFF,
-            server.aof_child_pid != -1,
+            server.child_type == CHILD_TYPE_AOF,
             server.aof_rewrite_scheduled,
             (intmax_t)server.aof_rewrite_time_last,
-            (intmax_t)((server.aof_child_pid == -1) ?
+            (intmax_t)((server.child_type != CHILD_TYPE_AOF) ?
                 -1 : time(NULL)-server.aof_rewrite_time_start),
             (server.aof_lastbgrewrite_status == C_OK) ? "ok" : "err",
             (server.aof_last_write_status == C_OK) ? "ok" : "err",
             server.stat_aof_cow_bytes,
-            server.module_child_pid != -1,
+            server.child_type == CHILD_TYPE_MODULE,
             server.stat_module_cow_bytes);
 
         if (server.aof_enabled) {
@@ -5289,6 +5312,13 @@ void closeClildUnusedResourceAfterFork() {
 
 /* purpose is one of CHILD_TYPE_ types */
 int redisFork(int purpose) {
+    if (isMutuallyExclusiveChildType(purpose)) {
+        if (hasActiveChildProcess())
+            return -1;
+
+        openChildInfoPipe();
+    }
+
     int childpid;
     long long start = ustime();
     if ((childpid = fork()) == 0) {
@@ -5304,8 +5334,23 @@ int redisFork(int purpose) {
         server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */
         latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000);
         if (childpid == -1) {
+            if (isMutuallyExclusiveChildType(purpose)) closeChildInfoPipe();
             return -1;
         }
+
+        /* The child_pid and child_type are only for mutual exclusive children.
+         * other child types should handle and store their pid's in dedicated variables.
+         *
+         * Today, we allows CHILD_TYPE_LDB to run in parallel with the other fork types:
+         * - it isn't used for production, so it will not make the server to be less efficient
+         * - used for debugging, and we don't want to block it from running while other
+         *   forks are running (like RDB and AOF) */
+        if (isMutuallyExclusiveChildType(purpose)) {
+            server.child_pid = childpid;
+            server.child_type = purpose;
+        }
+
+        updateDictResizePolicy();
     }
     return childpid;
 }
diff --git a/src/server.h b/src/server.h
index b5610432c..9d383cc37 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1145,7 +1145,8 @@ struct redisServer {
     int module_blocked_pipe[2]; /* Pipe used to awake the event loop if a
                                    client blocked on a module command needs
                                    to be processed. */
-    pid_t module_child_pid;     /* PID of module child */
+    pid_t child_pid;            /* PID of current child */
+    int child_type;             /* Type of current child */
     /* Networking */
     int port;                   /* TCP listening port */
     int tls_port;               /* TLS listening port */
@@ -1281,7 +1282,6 @@ struct redisServer {
     off_t aof_fsync_offset;         /* AOF offset which is already synced to disk. */
     int aof_flush_sleep;            /* Micros to sleep before flush. (used by tests) */
     int aof_rewrite_scheduled;      /* Rewrite once BGSAVE terminates. */
-    pid_t aof_child_pid;            /* PID if rewriting process */
     list *aof_rewrite_buf_blocks;   /* Hold changes during an AOF rewrite. */
     sds aof_buf;      /* AOF buffer, written before entering the event loop */
     int aof_fd;       /* File descriptor of currently selected AOF file */
@@ -1311,7 +1311,6 @@ struct redisServer {
     /* RDB persistence */
     long long dirty;                /* Changes to DB from the last save */
     long long dirty_before_bgsave;  /* Used to restore dirty on failed BGSAVE */
-    pid_t rdb_child_pid;            /* PID of RDB saving child */
     struct saveparam *saveparams;   /* Save points array for RDB */
     int saveparamslen;              /* Number of saving points */
     char *rdb_filename;             /* Name of RDB file */
@@ -2007,6 +2006,8 @@ void receiveChildInfo(void);
 /* Fork helpers */
 int redisFork(int type);
 int hasActiveChildProcess();
+void resetChildState();
+int isMutuallyExclusiveChildType(int type);
 void sendChildCOWInfo(int ptype, char *pname);
 
 /* acl.c -- Authentication related prototypes. */

From ea930a352ca748c0fa49a1b2ee894ea92bc83b0e Mon Sep 17 00:00:00 2001
From: YaacovHazan <yaacov.hazan@redislabs.com>
Date: Sun, 20 Dec 2020 20:23:20 +0200
Subject: [PATCH 0950/1098] Report child copy-on-write info continuously

Add INFO field, rdb_active_cow_size, to report COW of a live fork child while
it's active.
- once in 1024 keys check the time, and if there's more than one second since
  the last report send a report to the parent via the pipe.
- refactor the child_info_data struct, it's an implementation detail that
  shouldn't be in the server struct, and not used to communicate data between
  caller and callee
- remove the magic value from that struct (not sure what it was good for), and
  instead add handling of short reads.
- add another value to the structure, cow_type, to indicate if the report is
  for the new rdb_active_cow_size field, or it's the last report of a
  successful operation
- add new Module API to report the active COW
- add more asserts variants to test.tcl
---
 src/aof.c                 |  44 ++++++++++++++--
 src/childinfo.c           | 104 ++++++++++++++++++++++++++++++--------
 src/module.c              |   9 +++-
 src/rdb.c                 |  25 +++++++--
 src/redismodule.h         |   2 +
 src/server.c              |  19 ++++---
 src/server.h              |  13 ++---
 tests/integration/rdb.tcl |  75 +++++++++++++++++++++++++++
 tests/support/test.tcl    |  48 +++++++++++-------
 9 files changed, 277 insertions(+), 62 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index 928e1c4d1..d3191277f 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1425,6 +1425,8 @@ int rewriteAppendOnlyFileRio(rio *aof) {
     dictEntry *de;
     size_t processed = 0;
     int j;
+    long key_count = 0;
+    long long cow_updated_time = 0;
 
     for (j = 0; j < server.dbnum; j++) {
         char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n";
@@ -1484,6 +1486,19 @@ int rewriteAppendOnlyFileRio(rio *aof) {
                 processed = aof->processed_bytes;
                 aofReadDiffFromParent();
             }
+
+            /* Update COW info every 1 second (approximately).
+             * in order to avoid calling mstime() on each iteration, we will
+             * check the diff every 1024 keys */
+            if ((key_count & 1023) == 0) {
+                key_count = 0;
+                long long now = mstime();
+                if (now - cow_updated_time >= 1000) {
+                    sendChildCOWInfo(CHILD_TYPE_AOF, 0, "AOF rewrite");
+                    cow_updated_time = now;
+                }
+            }
+            key_count++;
         }
         dictReleaseIterator(di);
         di = NULL;
@@ -1577,8 +1592,31 @@ int rewriteAppendOnlyFile(char *filename) {
     serverLog(LL_NOTICE,
         "Concatenating %.2f MB of AOF diff received from parent.",
         (double) sdslen(server.aof_child_diff) / (1024*1024));
-    if (rioWrite(&aof,server.aof_child_diff,sdslen(server.aof_child_diff)) == 0)
-        goto werr;
+
+    /* Now we write the entire AOF buffer we received from the parent
+     * via the pipe during the life of this fork child.
+     * once a second, we'll take a break and send updated COW info to the parent */
+    size_t bytes_to_write = sdslen(server.aof_child_diff);
+    const char *buf = server.aof_child_diff;
+    long long cow_updated_time = mstime();
+
+    while (bytes_to_write) {
+        /* We write the AOF buffer in chunk of 8MB so that we can check the time in between them */
+        size_t chunk_size = bytes_to_write < (8<<20) ? bytes_to_write : (8<<20);
+
+        if (rioWrite(&aof,buf,chunk_size) == 0)
+            goto werr;
+
+        bytes_to_write -= chunk_size;
+        buf += chunk_size;
+
+        /* Update COW info */
+        long long now = mstime();
+        if (now - cow_updated_time >= 1000) {
+            sendChildCOWInfo(CHILD_TYPE_AOF, 0, "AOF rewrite");
+            cow_updated_time = now;
+        }
+    }
 
     /* Make sure data will not remain on the OS's output buffers */
     if (fflush(fp)) goto werr;
@@ -1709,7 +1747,7 @@ int rewriteAppendOnlyFileBackground(void) {
         redisSetCpuAffinity(server.aof_rewrite_cpulist);
         snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
         if (rewriteAppendOnlyFile(tmpfile) == C_OK) {
-            sendChildCOWInfo(CHILD_TYPE_AOF, "AOF rewrite");
+            sendChildCOWInfo(CHILD_TYPE_AOF, 1, "AOF rewrite");
             exitFromChild(0);
         } else {
             exitFromChild(1);
diff --git a/src/childinfo.c b/src/childinfo.c
index d11aa7bcf..85adfe2f6 100644
--- a/src/childinfo.c
+++ b/src/childinfo.c
@@ -30,6 +30,12 @@
 #include "server.h"
 #include <unistd.h>
 
+typedef struct {
+    int process_type;           /* AOF or RDB child? */
+    int on_exit;                /* COW size of active or exited child */
+    size_t cow_size;            /* Copy on write size. */
+} child_info_data;
+
 /* Open a child-parent channel used in order to move information about the
  * RDB / AOF saving process from the child to the parent (for instance
  * the amount of copy on write memory used) */
@@ -41,7 +47,7 @@ void openChildInfoPipe(void) {
     } else if (anetNonBlock(NULL,server.child_info_pipe[0]) != ANET_OK) {
         closeChildInfoPipe();
     } else {
-        memset(&server.child_info_data,0,sizeof(server.child_info_data));
+        server.child_info_nread = 0;
     }
 }
 
@@ -54,34 +60,88 @@ void closeChildInfoPipe(void) {
         close(server.child_info_pipe[1]);
         server.child_info_pipe[0] = -1;
         server.child_info_pipe[1] = -1;
+        server.child_info_nread = 0;
     }
 }
 
-/* Send COW data to parent. The child should call this function after populating
- * the corresponding fields it want to sent (according to the process type). */
-void sendChildInfo(int ptype) {
+/* Send COW data to parent. */
+void sendChildInfo(int process_type, int on_exit, size_t cow_size) {
     if (server.child_info_pipe[1] == -1) return;
-    server.child_info_data.magic = CHILD_INFO_MAGIC;
-    server.child_info_data.process_type = ptype;
-    ssize_t wlen = sizeof(server.child_info_data);
-    if (write(server.child_info_pipe[1],&server.child_info_data,wlen) != wlen) {
+
+    child_info_data buffer = {.process_type = process_type, .on_exit = on_exit, .cow_size = cow_size};
+    ssize_t wlen = sizeof(buffer);
+
+    if (write(server.child_info_pipe[1],&buffer,wlen) != wlen) {
         /* Nothing to do on error, this will be detected by the other side. */
     }
 }
 
-/* Receive COW data from parent. */
-void receiveChildInfo(void) {
-    if (server.child_info_pipe[0] == -1) return;
-    ssize_t wlen = sizeof(server.child_info_data);
-    if (read(server.child_info_pipe[0],&server.child_info_data,wlen) == wlen &&
-        server.child_info_data.magic == CHILD_INFO_MAGIC)
-    {
-        if (server.child_info_data.process_type == CHILD_TYPE_RDB) {
-            server.stat_rdb_cow_bytes = server.child_info_data.cow_size;
-        } else if (server.child_info_data.process_type == CHILD_TYPE_AOF) {
-            server.stat_aof_cow_bytes = server.child_info_data.cow_size;
-        } else if (server.child_info_data.process_type == CHILD_TYPE_MODULE) {
-            server.stat_module_cow_bytes = server.child_info_data.cow_size;
-        }
+/* Update COW data. */
+void updateChildInfo(int process_type, int on_exit, size_t cow_size) {
+    if (!on_exit) {
+        server.stat_current_cow_bytes = cow_size;
+        return;
+    }
+
+    if (process_type == CHILD_TYPE_RDB) {
+        server.stat_rdb_cow_bytes = cow_size;
+    } else if (process_type == CHILD_TYPE_AOF) {
+        server.stat_aof_cow_bytes = cow_size;
+    } else if (process_type == CHILD_TYPE_MODULE) {
+        server.stat_module_cow_bytes = cow_size;
+    }
+}
+
+/* Read COW info data from the pipe.
+ * if complete data read into the buffer, process type, copy-on-write type and copy-on-write size
+ * are stored into *process_type, *on_exit and *cow_size respectively and returns 1.
+ * otherwise, the partial data is left in the buffer, waiting for the next read, and returns 0. */
+int readChildInfo(int *process_type, int *on_exit, size_t *cow_size) {
+    /* We are using here a static buffer in combination with the server.child_info_nread to handle short reads */
+    static child_info_data buffer;
+    ssize_t wlen = sizeof(buffer);
+
+    /* Do not overlap */
+    if (server.child_info_nread == wlen) server.child_info_nread = 0;
+
+    int nread = read(server.child_info_pipe[0], (char *)&buffer + server.child_info_nread, wlen - server.child_info_nread);
+    if (nread > 0) {
+        server.child_info_nread += nread;
+    }
+
+    /* We have complete child info */
+    if (server.child_info_nread == wlen) {
+        *process_type = buffer.process_type;
+        *on_exit = buffer.on_exit;
+        *cow_size = buffer.cow_size;
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+/* Receive COW data from child. */
+void receiveChildInfo(void) {
+    if (server.child_info_pipe[0] == -1) return;
+
+    int process_type;
+    int on_exit;
+    size_t cow_size;
+    if (readChildInfo(&process_type, &on_exit, &cow_size)) {
+        updateChildInfo(process_type, on_exit, cow_size);
+    }
+}
+
+/* Receive last COW data from child. */
+void receiveLastChildInfo(void) {
+    if (server.child_info_pipe[0] == -1) return;
+
+    /* Drain the pipe and update child info */
+    int process_type;
+    int on_exit;
+    size_t cow_size;
+
+    while (readChildInfo(&process_type, &on_exit, &cow_size) > 0) {
+        updateChildInfo(process_type, on_exit, cow_size);
     }
 }
diff --git a/src/module.c b/src/module.c
index 2de6bee2f..c9e0b8812 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7080,11 +7080,17 @@ int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) {
     return childpid;
 }
 
+/* The module is advised to call this function from the fork child once in a while,
+ * so that it can report COW memory to the parent which will be reported in INFO */
+void RM_SendChildCOWInfo(void) {
+    sendChildCOWInfo(CHILD_TYPE_MODULE, 0, "Module fork");
+}
+
 /* Call from the child process when you want to terminate it.
  * retcode will be provided to the done handler executed on the parent process.
  */
 int RM_ExitFromChild(int retcode) {
-    sendChildCOWInfo(CHILD_TYPE_MODULE, "Module fork");
+    sendChildCOWInfo(CHILD_TYPE_MODULE, 1, "Module fork");
     exitFromChild(retcode);
     return REDISMODULE_OK;
 }
@@ -8590,6 +8596,7 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(CommandFilterArgReplace);
     REGISTER_API(CommandFilterArgDelete);
     REGISTER_API(Fork);
+    REGISTER_API(SendChildCOWInfo);
     REGISTER_API(ExitFromChild);
     REGISTER_API(KillForkChild);
     REGISTER_API(RegisterInfoFunc);
diff --git a/src/rdb.c b/src/rdb.c
index 1281948d1..25e900689 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1219,9 +1219,11 @@ int rdbSaveRio(rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi) {
     dictIterator *di = NULL;
     dictEntry *de;
     char magic[10];
-    int j;
     uint64_t cksum;
     size_t processed = 0;
+    int j;
+    long key_count = 0;
+    long long cow_updated_time = 0;
 
     if (server.rdb_checksum)
         rdb->update_cksum = rioGenericUpdateChecksum;
@@ -1267,6 +1269,23 @@ int rdbSaveRio(rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi) {
                 processed = rdb->processed_bytes;
                 aofReadDiffFromParent();
             }
+
+            /* Update COW info every 1 second (approximately).
+             * in order to avoid calling mstime() on each iteration, we will
+             * check the diff every 1024 keys */
+            if ((key_count & 1023) == 0) {
+                key_count = 0;
+                long long now = mstime();
+                if (now - cow_updated_time >= 1000) {
+                    if (rdbflags & RDBFLAGS_AOF_PREAMBLE) {
+                        sendChildCOWInfo(CHILD_TYPE_AOF, 0, "AOF rewrite");
+                    } else {
+                        sendChildCOWInfo(CHILD_TYPE_RDB, 0, "RDB");
+                    }
+                    cow_updated_time = now;
+                }
+            }
+            key_count++;
         }
         dictReleaseIterator(di);
         di = NULL; /* So that we don't release it again on error. */
@@ -1419,7 +1438,7 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
         redisSetCpuAffinity(server.bgsave_cpulist);
         retval = rdbSave(filename,rsi);
         if (retval == C_OK) {
-            sendChildCOWInfo(CHILD_TYPE_RDB, "RDB");
+            sendChildCOWInfo(CHILD_TYPE_RDB, 1, "RDB");
         }
         exitFromChild((retval == C_OK) ? 0 : 1);
     } else {
@@ -2786,7 +2805,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
             retval = C_ERR;
 
         if (retval == C_OK) {
-            sendChildCOWInfo(CHILD_TYPE_RDB, "RDB");
+            sendChildCOWInfo(CHILD_TYPE_RDB, 1, "RDB");
         }
 
         rioFreeFd(&rdb);
diff --git a/src/redismodule.h b/src/redismodule.h
index d2afa1f21..36c566bb3 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -778,6 +778,7 @@ REDISMODULE_API int (*RedisModule_CommandFilterArgInsert)(RedisModuleCommandFilt
 REDISMODULE_API int (*RedisModule_CommandFilterArgReplace)(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_CommandFilterArgDelete)(RedisModuleCommandFilterCtx *fctx, int pos) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_Fork)(RedisModuleForkDoneHandler cb, void *user_data) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_SendChildCOWInfo)(void) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_ExitFromChild)(int retcode) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_KillForkChild)(int child_pid) REDISMODULE_ATTR;
 REDISMODULE_API float (*RedisModule_GetUsedMemoryRatio)() REDISMODULE_ATTR;
@@ -1033,6 +1034,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(CommandFilterArgReplace);
     REDISMODULE_GET_API(CommandFilterArgDelete);
     REDISMODULE_GET_API(Fork);
+    REDISMODULE_GET_API(SendChildCOWInfo);
     REDISMODULE_GET_API(ExitFromChild);
     REDISMODULE_GET_API(KillForkChild);
     REDISMODULE_GET_API(GetUsedMemoryRatio);
diff --git a/src/server.c b/src/server.c
index ee2a00986..7d2688b66 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1578,6 +1578,7 @@ int hasActiveChildProcess() {
 void resetChildState() {
     server.child_type = CHILD_TYPE_NONE;
     server.child_pid = -1;
+    server.stat_current_cow_bytes = 0;
     updateDictResizePolicy();
     closeChildInfoPipe();
 }
@@ -2098,6 +2099,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
     /* Check if a background saving or AOF rewrite in progress terminated. */
     if (hasActiveChildProcess() || ldbPendingChildren())
     {
+        run_with_period(1000) receiveChildInfo();
         checkChildrenDone();
     } else {
         /* If there is not a background saving/rewrite in progress check if
@@ -3119,7 +3121,7 @@ void initServer(void) {
     server.rdb_bgsave_scheduled = 0;
     server.child_info_pipe[0] = -1;
     server.child_info_pipe[1] = -1;
-    server.child_info_data.magic = 0;
+    server.child_info_nread = 0;
     aofRewriteBufferReset();
     server.aof_buf = sdsempty();
     server.lastsave = time(NULL); /* At startup we consider the DB saved. */
@@ -3131,6 +3133,7 @@ void initServer(void) {
     /* A few stats we don't want to reset: server startup time, and peak mem. */
     server.stat_starttime = time(NULL);
     server.stat_peak_memory = 0;
+    server.stat_current_cow_bytes = 0;
     server.stat_rdb_cow_bytes = 0;
     server.stat_aof_cow_bytes = 0;
     server.stat_module_cow_bytes = 0;
@@ -4617,6 +4620,7 @@ sds genRedisInfoString(const char *section) {
         info = sdscatprintf(info,
             "# Persistence\r\n"
             "loading:%d\r\n"
+            "current_cow_size:%zu\r\n"
             "rdb_changes_since_last_save:%lld\r\n"
             "rdb_bgsave_in_progress:%d\r\n"
             "rdb_last_save_time:%jd\r\n"
@@ -4635,6 +4639,7 @@ sds genRedisInfoString(const char *section) {
             "module_fork_in_progress:%d\r\n"
             "module_fork_last_cow_size:%zu\r\n",
             server.loading,
+            server.stat_current_cow_bytes,
             server.dirty,
             server.child_type == CHILD_TYPE_RDB,
             (intmax_t)server.lastsave,
@@ -5342,12 +5347,13 @@ int redisFork(int purpose) {
          * other child types should handle and store their pid's in dedicated variables.
          *
          * Today, we allows CHILD_TYPE_LDB to run in parallel with the other fork types:
-         * - it isn't used for production, so it will not make the server to be less efficient
+         * - it isn't used for production, so it will not make the server be less efficient
          * - used for debugging, and we don't want to block it from running while other
          *   forks are running (like RDB and AOF) */
         if (isMutuallyExclusiveChildType(purpose)) {
             server.child_pid = childpid;
             server.child_type = purpose;
+            server.stat_current_cow_bytes = 0;
         }
 
         updateDictResizePolicy();
@@ -5355,17 +5361,16 @@ int redisFork(int purpose) {
     return childpid;
 }
 
-void sendChildCOWInfo(int ptype, char *pname) {
+void sendChildCOWInfo(int ptype, int on_exit, char *pname) {
     size_t private_dirty = zmalloc_get_private_dirty(-1);
 
     if (private_dirty) {
-        serverLog(LL_NOTICE,
+        serverLog(on_exit ? LL_NOTICE : LL_VERBOSE,
             "%s: %zu MB of memory used by copy-on-write",
-            pname, private_dirty/(1024*1024));
+            pname, private_dirty);
     }
 
-    server.child_info_data.cow_size = private_dirty;
-    sendChildInfo(ptype);
+    sendChildInfo(ptype, on_exit, private_dirty);
 }
 
 void memtest(size_t megabytes, int passes);
diff --git a/src/server.h b/src/server.h
index 9d383cc37..62281a63c 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1098,7 +1098,6 @@ struct clusterState;
 #undef hz
 #endif
 
-#define CHILD_INFO_MAGIC 0xC17DDA7A12345678LL
 #define CHILD_TYPE_NONE 0
 #define CHILD_TYPE_RDB 1
 #define CHILD_TYPE_AOF 2
@@ -1228,6 +1227,7 @@ struct redisServer {
     struct malloc_stats cron_malloc_stats; /* sampled in serverCron(). */
     redisAtomic long long stat_net_input_bytes; /* Bytes read from network. */
     redisAtomic long long stat_net_output_bytes; /* Bytes written to network. */
+    size_t stat_current_cow_bytes;  /* Copy on write bytes while child is active. */
     size_t stat_rdb_cow_bytes;      /* Copy on write bytes during RDB saving. */
     size_t stat_aof_cow_bytes;      /* Copy on write bytes during AOF rewrite. */
     size_t stat_module_cow_bytes;   /* Copy on write bytes during module fork. */
@@ -1342,11 +1342,7 @@ struct redisServer {
                                      * value means fractions of microsecons (on average). */
     /* Pipe and data structures for child -> parent info sharing. */
     int child_info_pipe[2];         /* Pipe used to write the child_info_data. */
-    struct {
-        int process_type;           /* AOF or RDB child? */
-        size_t cow_size;            /* Copy on write size. */
-        unsigned long long magic;   /* Magic value to make sure data is valid. */
-    } child_info_data;
+    int child_info_nread;           /* Num of bytes of the last read from pipe */
     /* Propagation of commands in AOF / replication */
     redisOpArray also_propagate;    /* Additional command to propagate. */
     /* Logging */
@@ -2000,15 +1996,16 @@ void restartAOFAfterSYNC();
 /* Child info */
 void openChildInfoPipe(void);
 void closeChildInfoPipe(void);
-void sendChildInfo(int process_type);
+void sendChildInfo(int process_type, int on_exit, size_t cow_size);
 void receiveChildInfo(void);
+void receiveLastChildInfo(void);
 
 /* Fork helpers */
 int redisFork(int type);
 int hasActiveChildProcess();
 void resetChildState();
 int isMutuallyExclusiveChildType(int type);
-void sendChildCOWInfo(int ptype, char *pname);
+void sendChildCOWInfo(int ptype, int on_exit, char *pname);
 
 /* acl.c -- Authentication related prototypes. */
 extern rax *Users;
diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl
index aadfe281f..cd4dfbadb 100644
--- a/tests/integration/rdb.tcl
+++ b/tests/integration/rdb.tcl
@@ -198,3 +198,78 @@ test {client freed during loading} {
         exec kill [srv 0 pid]
     }
 }
+
+start_server {overrides {save ""}} {
+    test {Test child sending COW info} {
+        # make sure that rdb_last_cow_size and current_cow_size are zero (the test using new server),
+        # so that the comparisons during the test will be valid
+        assert {[s current_cow_size] == 0}
+        assert {[s rdb_last_cow_size] == 0}
+
+        # using a 200us delay, the bgsave is empirically taking about 10 seconds.
+        # we need it to take more than some 5 seconds, since redis only report COW once a second.
+        r config set rdb-key-save-delay 200
+
+        # populate the db with 10k keys of 4k each
+        set rd [redis_deferring_client 0]
+        set size 4096
+        set cmd_count 10000
+        for {set k 0} {$k < $cmd_count} {incr k} {
+            $rd set key$k [string repeat A $size]
+        }
+
+        for {set k 0} {$k < $cmd_count} {incr k} {
+            catch { $rd read }
+        }
+
+        $rd close
+
+        # start background rdb save
+        r bgsave
+
+        # on each iteration, we will write some key to the server to trigger copy-on-write, and
+        # wait to see that it reflected in INFO.
+        set iteration 1
+        while 1 {
+            # take a sample before writing new data to the server
+            set cow_size [s current_cow_size]
+            if {$::verbose} {
+                puts "COW info before copy-on-write: $cow_size"
+            }
+
+            # trigger copy-on-write
+            r setrange key$iteration 0 [string repeat B $size]
+
+            # wait to see that current_cow_size value updated (as long as the child is in progress)
+            wait_for_condition 80 100 {
+                [s rdb_bgsave_in_progress] == 0 ||
+                [s current_cow_size] >= $cow_size + $size
+            } else {
+                if {$::verbose} {
+                    puts "COW info on fail: [s current_cow_size]"
+                }
+                fail "COW info didn't report"
+            }
+
+            # for no accurate, stop after 2 iterations
+            if {!$::accurate && $iteration == 2} {
+                break
+            }
+
+            # stop iterating if the bgsave completed
+            if { [s rdb_bgsave_in_progress] == 0 } {
+                break
+            }
+
+            incr iteration 1
+        }
+
+        # make sure we saw report of current_cow_size
+        assert_morethan_equal $iteration 2
+
+        # if bgsave completed, check that rdb_last_cow_size value is at least as last rdb_active_cow_size.
+        if { [s rdb_bgsave_in_progress] == 0 } {
+            assert_morethan_equal [s rdb_last_cow_size] $cow_size
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/support/test.tcl b/tests/support/test.tcl
index 23015b3a7..39aebe156 100644
--- a/tests/support/test.tcl
+++ b/tests/support/test.tcl
@@ -31,36 +31,48 @@ proc assert_match {pattern value} {
     }
 }
 
+proc assert_failed {expected_err detail} {
+     if {$detail ne ""} {
+        set detail "(detail: $detail)"
+     } else {
+        set detail "(context: [info frame -2])"
+     }
+     error "assertion:$expected_err $detail"
+}
+
 proc assert_equal {value expected {detail ""}} {
     if {$expected ne $value} {
-        if {$detail ne ""} {
-            set detail "(detail: $detail)"
-        } else {
-            set detail "(context: [info frame -1])"
-        }
-        error "assertion:Expected '$value' to be equal to '$expected' $detail"
+        assert_failed "Expected '$value' to be equal to '$expected'" $detail
     }
 }
 
 proc assert_lessthan {value expected {detail ""}} {
     if {!($value < $expected)} {
-        if {$detail ne ""} {
-            set detail "(detail: $detail)"
-        } else {
-            set detail "(context: [info frame -1])"
-        }
-        error "assertion:Expected '$value' to be lessthan to '$expected' $detail"
+        assert_failed "Expected '$value' to be less than '$expected'" $detail
+    }
+}
+
+proc assert_lessthan_equal {value expected {detail ""}} {
+    if {!($value <= $expected)} {
+        assert_failed "Expected '$value' to be less than or equal to '$expected'" $detail
+    }
+}
+
+proc assert_morethan {value expected {detail ""}} {
+    if {!($value > $expected)} {
+        assert_failed "Expected '$value' to be more than '$expected'" $detail
+    }
+}
+
+proc assert_morethan_equal {value expected {detail ""}} {
+    if {!($value >= $expected)} {
+        assert_failed "Expected '$value' to be more than or equal to '$expected'" $detail
     }
 }
 
 proc assert_range {value min max {detail ""}} {
     if {!($value <= $max && $value >= $min)} {
-        if {$detail ne ""} {
-            set detail "(detail: $detail)"
-        } else {
-            set detail "(context: [info frame -1])"
-        }
-        error "assertion:Expected '$value' to be between to '$min' and '$max' $detail"
+        assert_failed "Expected '$value' to be between to '$min' and '$max'" $detail
     }
 }
 

From b02780c41dbc5b28d265b5cf141c03c1a7383ef9 Mon Sep 17 00:00:00 2001
From: George Prekas <prekgeo@yahoo.com>
Date: Thu, 7 Jan 2021 09:06:05 -0600
Subject: [PATCH 0951/1098] Add check for the MADV_FREE/fork arm64 Linux kernel
 bug (#8224)

Older arm64 Linux kernels have a bug that could lead to data corruption during
background save under the following scenario:

1) jemalloc uses MADV_FREE on a page,
2) jemalloc reuses and writes the page,
3) Redis forks the background save process, and
4) Linux performs page reclamation.

Under these conditions, Linux will reclaim the page wrongfully and the
background save process will read zeros when it tries to read the page.

The bug has been fixed in Linux with commit:
ff1712f953e27f0b0718762ec17d0adb15c9fd0b ("arm64: pgtable: Ensure dirty bit is
preserved across pte_wrprotect()")

This Commit adds an ignore-warnings config, when not found, redis will
print a warning and exit on startup (default behavior).

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 redis.conf   |   7 +++
 src/config.c |   1 +
 src/server.c | 137 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 src/server.h |   1 +
 4 files changed, 145 insertions(+), 1 deletion(-)

diff --git a/redis.conf b/redis.conf
index 891bb1663..a5062fda9 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1976,3 +1976,10 @@ jemalloc-bg-thread yes
 #
 # Set bgsave child process to cpu affinity 1,10,11
 # bgsave_cpulist 1,10-11
+
+# In some cases redis will emit warnings and even refuse to start if it detects
+# that the system is in bad state, it is possible to suppress these warnings
+# by setting the following config which takes a space delimited list of warnings
+# to suppress
+#
+# ignore-warnings ARM64-COW-BUG
diff --git a/src/config.c b/src/config.c
index d804ff0ef..2e109dbae 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2424,6 +2424,7 @@ standardConfig configs[] = {
     createStringConfig("bio_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bio_cpulist, NULL, NULL, NULL),
     createStringConfig("aof_rewrite_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.aof_rewrite_cpulist, NULL, NULL, NULL),
     createStringConfig("bgsave_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bgsave_cpulist, NULL, NULL, NULL),
+    createStringConfig("ignore-warnings", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.ignore_warnings, "", NULL, NULL),
 
     /* SDS Configs */
     createSDSConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL, NULL, NULL),
diff --git a/src/server.c b/src/server.c
index 7d2688b66..faa67fccb 100644
--- a/src/server.c
+++ b/src/server.c
@@ -59,6 +59,10 @@
 #include <sys/socket.h>
 #include <sys/resource.h>
 
+#ifdef __linux__
+#include <sys/mman.h>
+#endif
+
 /* Our shared "common" objects */
 
 struct sharedObjectsStruct shared;
@@ -5090,6 +5094,21 @@ void monitorCommand(client *c) {
 
 /* =================================== Main! ================================ */
 
+int checkIgnoreWarning(const char *warning) {
+    int argc, j;
+    sds *argv = sdssplitargs(server.ignore_warnings, &argc);
+    if (argv == NULL)
+        return 0;
+
+    for (j = 0; j < argc; j++) {
+        char *flag = argv[j];
+        if (!strcasecmp(flag, warning))
+            break;
+    }
+    sdsfreesplitres(argv,argc);
+    return j < argc;
+}
+
 #ifdef __linux__
 int linuxOvercommitMemoryValue(void) {
     FILE *fp = fopen("/proc/sys/vm/overcommit_memory","r");
@@ -5113,6 +5132,113 @@ void linuxMemoryWarnings(void) {
         serverLog(LL_WARNING,"WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo madvise > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled (set to 'madvise' or 'never').");
     }
 }
+
+#ifdef __arm64__
+
+/* Get size in kilobytes of the Shared_Dirty pages of the calling process for the
+ * memory map corresponding to the provided address, or -1 on error. */
+static int smapsGetSharedDirty(unsigned long addr) {
+    int ret, in_mapping = 0, val = -1;
+    unsigned long from, to;
+    char buf[64];
+    FILE *f;
+
+    f = fopen("/proc/self/smaps", "r");
+    serverAssert(f);
+
+    while (1) {
+        if (!fgets(buf, sizeof(buf), f))
+            break;
+
+        ret = sscanf(buf, "%lx-%lx", &from, &to);
+        if (ret == 2)
+            in_mapping = from <= addr && addr < to;
+
+        if (in_mapping && !memcmp(buf, "Shared_Dirty:", 13)) {
+            ret = sscanf(buf, "%*s %d", &val);
+            serverAssert(ret == 1);
+            break;
+        }
+    }
+
+    fclose(f);
+    return val;
+}
+
+/* Older arm64 Linux kernels have a bug that could lead to data corruption
+ * during background save in certain scenarios. This function checks if the
+ * kernel is affected.
+ * The bug was fixed in commit ff1712f953e27f0b0718762ec17d0adb15c9fd0b
+ * titled: "arm64: pgtable: Ensure dirty bit is preserved across pte_wrprotect()"
+ * Return 1 if the kernel seems to be affected, and 0 otherwise. */
+int linuxMadvFreeForkBugCheck(void) {
+    int ret, pipefd[2];
+    pid_t pid;
+    char *p, *q, bug_found = 0;
+    const long map_size = 3 * 4096;
+
+    /* Create a memory map that's in our full control (not one used by the allocator). */
+    p = mmap(NULL, map_size, PROT_READ, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+    serverAssert(p != MAP_FAILED);
+
+    q = p + 4096;
+
+    /* Split the memory map in 3 pages by setting their protection as RO|RW|RO to prevent
+     * Linux from merging this memory map with adjacent VMAs. */
+    ret = mprotect(q, 4096, PROT_READ | PROT_WRITE);
+    serverAssert(!ret);
+
+    /* Write to the page once to make it resident */
+    *(volatile char*)q = 0;
+
+    /* Tell the kernel that this page is free to be reclaimed. */
+#ifndef MADV_FREE
+#define MADV_FREE 8
+#endif
+    ret = madvise(q, 4096, MADV_FREE);
+    serverAssert(!ret);
+
+    /* Write to the page after being marked for freeing, this is supposed to take
+     * ownership of that page again. */
+    *(volatile char*)q = 0;
+
+    /* Create a pipe for the child to return the info to the parent. */
+    ret = pipe(pipefd);
+    serverAssert(!ret);
+
+    /* Fork the process. */
+    pid = fork();
+    serverAssert(pid >= 0);
+    if (!pid) {
+        /* Child: check if the page is marked as dirty, expecing 4 (kB).
+         * A value of 0 means the kernel is affected by the bug. */
+        if (!smapsGetSharedDirty((unsigned long)q))
+            bug_found = 1;
+
+        ret = write(pipefd[1], &bug_found, 1);
+        serverAssert(ret == 1);
+
+        exit(0);
+    } else {
+        /* Read the result from the child. */
+        ret = read(pipefd[0], &bug_found, 1);
+        serverAssert(ret == 1);
+
+        /* Reap the child pid. */
+        serverAssert(waitpid(pid, NULL, 0) == pid);
+    }
+
+    /* Cleanup */
+    ret = close(pipefd[0]);
+    serverAssert(!ret);
+    ret = close(pipefd[1]);
+    serverAssert(!ret);
+    ret = munmap(p, map_size);
+    serverAssert(!ret);
+
+    return bug_found;
+}
+#endif /* __arm64__ */
 #endif /* __linux__ */
 
 void createPidFile(void) {
@@ -5711,7 +5837,16 @@ int main(int argc, char **argv) {
         serverLog(LL_WARNING,"Server initialized");
     #ifdef __linux__
         linuxMemoryWarnings();
-    #endif
+    #if defined (__arm64__)
+        if (linuxMadvFreeForkBugCheck()) {
+            serverLog(LL_WARNING,"WARNING Your kernel has a bug that could lead to data corruption during background save. Please upgrade to the latest stable kernel.");
+            if (!checkIgnoreWarning("ARM64-COW-BUG")) {
+                serverLog(LL_WARNING,"Redis will now exit to prevent data corruption. Note that it is possible to suppress this warning by setting the following config: ignore-warnings ARM64-COW-BUG");
+                exit(1);
+            }
+        }
+    #endif /* __arm64__ */
+    #endif /* __linux__ */
         moduleInitModulesSystemLast();
         moduleLoadFromQueue();
         ACLLoadUsersAtStartup();
diff --git a/src/server.h b/src/server.h
index 62281a63c..fb6d71c71 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1136,6 +1136,7 @@ struct redisServer {
     int in_eval;                /* Are we inside EVAL? */
     int in_exec;                /* Are we inside EXEC? */
     int propagate_in_transaction;  /* Make sure we don't propagate nested MULTI/EXEC */
+    char *ignore_warnings;      /* Config: warnings that should be ignored. */
     /* Modules */
     dict *moduleapi;            /* Exported core APIs dictionary for modules. */
     dict *sharedapi;            /* Like moduleapi but containing the APIs that

From 47579bdf5cccfbac1cf2b6e7e52218c5b9faed86 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Thu, 7 Jan 2021 23:36:54 -0800
Subject: [PATCH 0952/1098] Add support for client pause WRITE (#8170)

Implementation of client pause WRITE and client unpause
---
 src/blocked.c        |  14 +++
 src/cluster.c        |   7 +-
 src/db.c             |   6 ++
 src/evict.c          |   2 +-
 src/expire.c         |   2 +-
 src/module.c         |   5 +-
 src/networking.c     | 156 +++++++++++++++++++++------------
 src/replication.c    |   2 +-
 src/server.c         |  60 ++++++++++---
 src/server.h         |  78 ++++++++++-------
 tests/unit/pause.tcl | 200 +++++++++++++++++++++++++++++++++++++++++++
 tests/unit/wait.tcl  |  10 +++
 12 files changed, 435 insertions(+), 107 deletions(-)
 create mode 100644 tests/unit/pause.tcl

diff --git a/src/blocked.c b/src/blocked.c
index e3f7b74d6..250e4bd98 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -89,6 +89,12 @@ void blockClient(client *c, int btype) {
     server.blocked_clients++;
     server.blocked_clients_by_type[btype]++;
     addClientToTimeoutTable(c);
+    if (btype == BLOCKED_PAUSE) {
+        listAddNodeTail(server.paused_clients, c);
+        c->paused_list_node = listLast(server.paused_clients);
+        /* Mark this client to execute its command */
+        c->flags |= CLIENT_PENDING_COMMAND;
+    }
 }
 
 /* This function is called in the beforeSleep() function of the event loop
@@ -110,6 +116,11 @@ void processUnblockedClients(void) {
          * client is not blocked before to proceed, but things may change and
          * the code is conceptually more correct this way. */
         if (!(c->flags & CLIENT_BLOCKED)) {
+            /* If we have a queued command, execute it now. */
+            if (processPendingCommandsAndResetClient(c) == C_ERR) {
+                continue;
+            }
+            /* Then process client if it has more data in it's buffer. */
             if (c->querybuf && sdslen(c->querybuf) > 0) {
                 processInputBuffer(c);
             }
@@ -154,6 +165,9 @@ void unblockClient(client *c) {
     } else if (c->btype == BLOCKED_MODULE) {
         if (moduleClientIsBlockedOnKeys(c)) unblockClientWaitingData(c);
         unblockClientFromModule(c);
+    } else if (c->btype == BLOCKED_PAUSE) {
+        listDelNode(server.paused_clients,c->paused_list_node);
+        c->paused_list_node = NULL;
     } else {
         serverPanic("Unknown btype in unblockClient().");
     }
diff --git a/src/cluster.c b/src/cluster.c
index d9f1a66d7..c327334ee 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -2164,7 +2164,7 @@ int clusterProcessPacket(clusterLink *link) {
         resetManualFailover();
         server.cluster->mf_end = now + CLUSTER_MF_TIMEOUT;
         server.cluster->mf_slave = sender;
-        pauseClients(now+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT));
+        pauseClients(now+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT),CLIENT_PAUSE_WRITE);
         serverLog(LL_WARNING,"Manual failover requested by replica %.40s.",
             sender->name);
         /* We need to send a ping message to the replica, as it would carry
@@ -3421,9 +3421,8 @@ void clusterHandleSlaveMigration(int max_slaves) {
  * The function can be used both to initialize the manual failover state at
  * startup or to abort a manual failover in progress. */
 void resetManualFailover(void) {
-    if (server.cluster->mf_end && clientsArePaused()) {
-        server.clients_pause_end_time = 0;
-        clientsArePaused(); /* Just use the side effect of the function. */
+    if (server.cluster->mf_end) {
+        checkClientPauseTimeoutAndReturnIfPaused();
     }
     server.cluster->mf_end = 0; /* No manual failover in progress. */
     server.cluster->mf_can_start = 0;
diff --git a/src/db.c b/src/db.c
index c4b48fb0e..5d63566a7 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1520,6 +1520,12 @@ int expireIfNeeded(redisDb *db, robj *key) {
      * we think the key is expired at this time. */
     if (server.masterhost != NULL) return 1;
 
+    /* If clients are paused, we keep the current dataset constant,
+     * but return to the client what we believe is the right state. Typically,
+     * at the end of the pause we will properly expire the key OR we will
+     * have failed over and the new primary will send us the expire. */
+    if (checkClientPauseTimeoutAndReturnIfPaused()) return 1;
+
     /* Delete the key */
     server.stat_expiredkeys++;
     propagateExpire(db,key,server.lazyfree_lazy_expire);
diff --git a/src/evict.c b/src/evict.c
index 3642352ad..04513cd1a 100644
--- a/src/evict.c
+++ b/src/evict.c
@@ -462,7 +462,7 @@ static int isSafeToPerformEvictions(void) {
     /* When clients are paused the dataset should be static not just from the
      * POV of clients not being able to write, but also from the POV of
      * expires and evictions of keys not being performed. */
-    if (clientsArePaused()) return 0;
+    if (checkClientPauseTimeoutAndReturnIfPaused()) return 0;
 
     return 1;
 }
diff --git a/src/expire.c b/src/expire.c
index 5433f46ca..275a735a7 100644
--- a/src/expire.c
+++ b/src/expire.c
@@ -148,7 +148,7 @@ void activeExpireCycle(int type) {
     /* When clients are paused the dataset should be static not just from the
      * POV of clients not being able to write, but also from the POV of
      * expires and evictions of keys not being performed. */
-    if (clientsArePaused()) return;
+    if (checkClientPauseTimeoutAndReturnIfPaused()) return;
 
     if (type == ACTIVE_EXPIRE_CYCLE_FAST) {
         /* Don't start a fast cycle if the previous cycle did not exit
diff --git a/src/module.c b/src/module.c
index c9e0b8812..bf186f8b7 100644
--- a/src/module.c
+++ b/src/module.c
@@ -747,6 +747,7 @@ int64_t commandFlagsFromString(char *s) {
         else if (!strcasecmp(t,"no-slowlog")) flags |= CMD_SKIP_SLOWLOG;
         else if (!strcasecmp(t,"fast")) flags |= CMD_FAST;
         else if (!strcasecmp(t,"no-auth")) flags |= CMD_NO_AUTH;
+        else if (!strcasecmp(t,"may-replicate")) flags |= CMD_MAY_REPLICATE;
         else if (!strcasecmp(t,"getkeys-api")) flags |= CMD_MODULE_GETKEYS;
         else if (!strcasecmp(t,"no-cluster")) flags |= CMD_MODULE_NO_CLUSTER;
         else break;
@@ -813,6 +814,8 @@ int64_t commandFlagsFromString(char *s) {
  * * **"no-auth"**:    This command can be run by an un-authenticated client.
  *                     Normally this is used by a command that is used
  *                     to authenticate a client. 
+ * * **"may-replicate"**: This command may generate replication traffic, even
+ *                        though it's not a write command.  
  */
 int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) {
     int64_t flags = strflags ? commandFlagsFromString((char*)strflags) : 0;
@@ -2042,7 +2045,7 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) {
  * periodically in timer callbacks or other periodic callbacks.
  */
 int RM_AvoidReplicaTraffic() {
-    return clientsArePaused();
+    return checkClientPauseTimeoutAndReturnIfPaused();
 }
 
 /* Change the currently selected DB. Returns an error if the id
diff --git a/src/networking.c b/src/networking.c
index e71e39624..9a86e6085 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -178,6 +178,7 @@ client *createClient(connection *conn) {
     c->peerid = NULL;
     c->sockname = NULL;
     c->client_list_node = NULL;
+    c->paused_list_node = NULL;
     c->client_tracking_redirection = 0;
     c->client_tracking_prefixes = NULL;
     c->client_cron_last_memory_usage = 0;
@@ -1960,7 +1961,7 @@ void commandProcessed(client *c) {
      * still be able to access the client argv and argc field.
      * The client will be reset in unblockClientFromModule(). */
     if (!(c->flags & CLIENT_BLOCKED) ||
-        c->btype != BLOCKED_MODULE)
+        (c->btype != BLOCKED_MODULE && c->btype != BLOCKED_PAUSE))
     {
         resetClient(c);
     }
@@ -2003,6 +2004,20 @@ int processCommandAndResetClient(client *c) {
     return deadclient ? C_ERR : C_OK;
 }
 
+
+/* This function will execute any fully parsed commands pending on
+ * the client. Returns C_ERR if the client is no longer valid after executing
+ * the command, and C_OK for all other cases. */
+int processPendingCommandsAndResetClient(client *c) {
+    if (c->flags & CLIENT_PENDING_COMMAND) {
+        c->flags &= ~CLIENT_PENDING_COMMAND;
+        if (processCommandAndResetClient(c) == C_ERR) {
+            return C_ERR;
+        }
+    }
+    return C_OK;
+}
+
 /* This function is called every time, in the client structure 'c', there is
  * more query buffer to process, because we read more data from the socket
  * or because a client was blocked and later reactivated, so there could be
@@ -2010,9 +2025,6 @@ int processCommandAndResetClient(client *c) {
 void processInputBuffer(client *c) {
     /* Keep processing while there is something in the input buffer */
     while(c->qb_pos < sdslen(c->querybuf)) {
-        /* Return if clients are paused. */
-        if (!(c->flags & CLIENT_SLAVE) && clientsArePaused()) break;
-
         /* Immediately abort if the client is in the middle of something. */
         if (c->flags & CLIENT_BLOCKED) break;
 
@@ -2436,8 +2448,10 @@ void clientCommand(client *c) {
 "    Return information about client connections. Options:",
 "    * TYPE (NORMAL|MASTER|REPLICA|PUBSUB)",
 "      Return clients of specified type.",
-"PAUSE <timeout>",
-"    Suspend all clients for <timout> milliseconds.",
+"UNPAUSE",
+"    Stop the current client pause, resuming traffic.",
+"PAUSE <timeout> [WRITE|ALL]",
+"    Suspend all, or just write, clients for <timout> milliseconds.",
 "REPLY (ON|OFF|SKIP)",
 "    Control the replies sent to the current connection.",
 "SETNAME <name>",
@@ -2653,13 +2667,31 @@ NULL
             addReplyBulk(c,c->name);
         else
             addReplyNull(c);
-    } else if (!strcasecmp(c->argv[1]->ptr,"pause") && c->argc == 3) {
-        /* CLIENT PAUSE */
+    } else if (!strcasecmp(c->argv[1]->ptr,"unpause") && c->argc == 2) {
+        /* CLIENT UNPAUSE */
+        unpauseClients();
+        addReply(c,shared.ok);
+    } else if (!strcasecmp(c->argv[1]->ptr,"pause") && (c->argc == 3 ||
+                                                        c->argc == 4))
+    {
+        /* CLIENT PAUSE TIMEOUT [WRITE|ALL] */
         long long duration;
+        int type = CLIENT_PAUSE_ALL;
+        if (c->argc == 4) {
+            if (!strcasecmp(c->argv[3]->ptr,"write")) {
+                type = CLIENT_PAUSE_WRITE;
+            } else if (!strcasecmp(c->argv[3]->ptr,"all")) {
+                type = CLIENT_PAUSE_ALL;
+            } else {
+                addReplyError(c,
+                    "CLIENT PAUSE mode must be WRITE or ALL");  
+                return;       
+            }
+        }
 
         if (getTimeoutFromObjectOrReply(c,c->argv[2],&duration,
-                UNIT_MILLISECONDS) != C_OK) return;
-        pauseClients(duration);
+            UNIT_MILLISECONDS) != C_OK) return;
+        pauseClients(duration, type);
         addReply(c,shared.ok);
     } else if (!strcasecmp(c->argv[1]->ptr,"tracking") && c->argc >= 3) {
         /* CLIENT TRACKING (on|off) [REDIRECT <id>] [BCAST] [PREFIX first]
@@ -3209,54 +3241,66 @@ void flushSlavesOutputBuffers(void) {
     }
 }
 
-/* Pause clients up to the specified unixtime (in ms). While clients
- * are paused no command is processed from clients, so the data set can't
- * change during that time.
- *
- * However while this function pauses normal and Pub/Sub clients, slaves are
- * still served, so this function can be used on server upgrades where it is
- * required that slaves process the latest bytes from the replication stream
- * before being turned to masters.
+/* Pause clients up to the specified unixtime (in ms) for a given type of
+ * commands.
  *
+ * A main use case of this function is to allow pausing replication traffic
+ * so that a failover without data loss to occur. Replicas will continue to receive
+ * traffic to faciliate this functionality.
+ * 
  * This function is also internally used by Redis Cluster for the manual
  * failover procedure implemented by CLUSTER FAILOVER.
  *
  * The function always succeed, even if there is already a pause in progress.
- * In such a case, the pause is extended if the duration is more than the
- * time left for the previous duration. However if the duration is smaller
- * than the time left for the previous pause, no change is made to the
- * left duration. */
-void pauseClients(mstime_t end) {
-    if (!server.clients_paused || end > server.clients_pause_end_time)
-        server.clients_pause_end_time = end;
-    server.clients_paused = 1;
+ * In such a case, the duration is set to the maximum and new end time and the
+ * type is set to the more restrictive type of pause. */
+void pauseClients(mstime_t end, pause_type type) {
+    if (type > server.client_pause_type) {
+        server.client_pause_type = type;
+    }
+
+    if (end > server.client_pause_end_time) {
+        server.client_pause_end_time = end;
+    }
+
+    /* We allow write commands that were queued
+     * up before and after to execute. We need
+     * to track this state so that we don't assert
+     * in propagate(). */
+    if (server.in_exec) {
+        server.client_pause_in_transaction = 1;
+    }
 }
 
-/* Return non-zero if clients are currently paused. As a side effect the
- * function checks if the pause time was reached and clear it. */
-int clientsArePaused(void) {
-    if (server.clients_paused &&
-        server.clients_pause_end_time < server.mstime)
-    {
-        listNode *ln;
-        listIter li;
-        client *c;
+/* Unpause clients and queue them for reprocessing. */
+void unpauseClients(void) {
+    listNode *ln;
+    listIter li;
+    client *c;
+    
+    server.client_pause_type = CLIENT_PAUSE_OFF;
 
-        server.clients_paused = 0;
-
-        /* Put all the clients in the unblocked clients queue in order to
-         * force the re-processing of the input buffer if any. */
-        listRewind(server.clients,&li);
-        while ((ln = listNext(&li)) != NULL) {
-            c = listNodeValue(ln);
-
-            /* Don't touch slaves and blocked clients.
-             * The latter pending requests will be processed when unblocked. */
-            if (c->flags & (CLIENT_SLAVE|CLIENT_BLOCKED)) continue;
-            queueClientForReprocessing(c);
-        }
+    /* Unblock all of the clients so they are reprocessed. */
+    listRewind(server.paused_clients,&li);
+    while ((ln = listNext(&li)) != NULL) {
+        c = listNodeValue(ln);
+        unblockClient(c);
     }
-    return server.clients_paused;
+}
+
+/* Returns true if clients are paused and false otherwise. */ 
+int areClientsPaused(void) {
+    return server.client_pause_type != CLIENT_PAUSE_OFF;
+}
+
+/* Checks if the current client pause has elapsed and unpause clients
+ * if it has. Also returns true if clients are now paused and false 
+ * otherwise. */
+int checkClientPauseTimeoutAndReturnIfPaused(void) {
+    if (server.client_pause_end_time < server.mstime) {
+        unpauseClients();
+    }
+    return areClientsPaused();
 }
 
 /* This function is called by Redis in order to process a few events from
@@ -3634,15 +3678,13 @@ int handleClientsWithPendingReadsUsingThreads(void) {
         c->flags &= ~CLIENT_PENDING_READ;
         listDelNode(server.clients_pending_read,ln);
 
-        if (c->flags & CLIENT_PENDING_COMMAND) {
-            c->flags &= ~CLIENT_PENDING_COMMAND;
-            if (processCommandAndResetClient(c) == C_ERR) {
-                /* If the client is no longer valid, we avoid
-                 * processing the client later. So we just go
-                 * to the next. */
-                continue;
-            }
+        if (processPendingCommandsAndResetClient(c) == C_ERR) {
+            /* If the client is no longer valid, we avoid
+             * processing the client later. So we just go
+             * to the next. */
+            continue;
         }
+
         processInputBuffer(c);
 
         /* We may have pending replies if a thread readQueryFromClient() produced
diff --git a/src/replication.c b/src/replication.c
index 361bdb626..9fb19eaca 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -3237,7 +3237,7 @@ void replicationCron(void) {
         int manual_failover_in_progress =
             server.cluster_enabled &&
             server.cluster->mf_end &&
-            clientsArePaused();
+            checkClientPauseTimeoutAndReturnIfPaused();
 
         if (!manual_failover_in_progress) {
             ping_argv[0] = createStringObject("PING",4);
diff --git a/src/server.c b/src/server.c
index faa67fccb..2f9fa4f39 100644
--- a/src/server.c
+++ b/src/server.c
@@ -163,6 +163,13 @@ struct redisServer server; /* Server global state */
  *              delay its execution as long as the kernel scheduler is giving
  *              us time. Note that commands that may trigger a DEL as a side
  *              effect (like SET) are not fast commands.
+ * 
+ * may-replicate: Command may produce replication traffic, but should be 
+ *                allowed under circumstances where write commands are disallowed. 
+ *                Examples include PUBLISH, which replicates pubsub messages,and 
+ *                EVAL, which may execute write commands, which are replicated, 
+ *                or may just execute read commands. A command can not be marked 
+ *                both "write" and "may-replicate"
  *
  * The following additional flags are only used in order to put commands
  * in a specific ACL category. Commands can have multiple ACL categories.
@@ -818,7 +825,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"publish",publishCommand,3,
-     "pub-sub ok-loading ok-stale fast",
+     "pub-sub ok-loading ok-stale fast may-replicate",
      0,NULL,0,0,0,0,0,0},
 
     {"pubsub",pubsubCommand,-2,
@@ -884,11 +891,11 @@ struct redisCommand redisCommandTable[] = {
     /* EVAL can modify the dataset, however it is not flagged as a write
      * command since we do the check while running commands from Lua. */
     {"eval",evalCommand,-3,
-     "no-script @scripting",
+     "no-script may-replicate @scripting",
      0,evalGetKeys,0,0,0,0,0,0},
 
     {"evalsha",evalShaCommand,-3,
-     "no-script @scripting",
+     "no-script may-replicate @scripting",
      0,evalGetKeys,0,0,0,0,0,0},
 
     {"slowlog",slowlogCommand,-2,
@@ -896,7 +903,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"script",scriptCommand,-2,
-     "no-script @scripting",
+     "no-script may-replicate @scripting",
      0,NULL,0,0,0,0,0,0},
 
     {"time",timeCommand,1,
@@ -977,7 +984,7 @@ struct redisCommand redisCommandTable[] = {
      * we claim that the representation, even if accessible, is an internal
      * affair, and the command is semantically read only. */
     {"pfcount",pfcountCommand,-2,
-     "read-only @hyperloglog",
+     "read-only may-replicate @hyperloglog",
      0,NULL,1,-1,1,0,0,0},
 
     {"pfmerge",pfmergeCommand,-2,
@@ -2163,8 +2170,8 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
             flushAppendOnlyFile(0);
     }
 
-    /* Clear the paused clients flag if needed. */
-    clientsArePaused(); /* Don't check return value, just use the side effect.*/
+    /* Clear the paused clients state if needed. */
+    checkClientPauseTimeoutAndReturnIfPaused();
 
     /* Replication cron function -- used to reconnect to master,
      * detect transfer failures, start background RDB transfers and so forth. */
@@ -2361,8 +2368,12 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
      * during the previous event loop iteration. Note that we do this after
      * processUnblockedClients(), so if there are multiple pipelined WAITs
      * and the just unblocked WAIT gets blocked again, we don't have to wait
-     * a server cron cycle in absence of other event loop events. See #6623. */
-    if (server.get_ack_from_slaves) {
+     * a server cron cycle in absence of other event loop events. See #6623.
+     * 
+     * We also don't send the ACKs while clients are paused, since it can
+     * increment the replication backlog, they'll be sent after the pause
+     * if we are still the master. */
+    if (server.get_ack_from_slaves && !checkClientPauseTimeoutAndReturnIfPaused()) {
         robj *argv[3];
 
         argv[0] = createStringObject("REPLCONF",8);
@@ -3040,7 +3051,8 @@ void initServer(void) {
     server.ready_keys = listCreate();
     server.clients_waiting_acks = listCreate();
     server.get_ack_from_slaves = 0;
-    server.clients_paused = 0;
+    server.client_pause_type = 0;
+    server.paused_clients = listCreate();
     server.events_processed_while_blocked = 0;
     server.system_memory_size = zmalloc_get_memory_size();
     server.blocked_last_cron = 0;
@@ -3114,6 +3126,7 @@ void initServer(void) {
     server.in_eval = 0;
     server.in_exec = 0;
     server.propagate_in_transaction = 0;
+    server.client_pause_in_transaction = 0;
     server.child_pid = -1;
     server.child_type = CHILD_TYPE_NONE;
     server.rdb_child_type = RDB_CHILD_TYPE_NONE;
@@ -3280,6 +3293,8 @@ int populateCommandTableParseFlags(struct redisCommand *c, char *strflags) {
             c->flags |= CMD_FAST | CMD_CATEGORY_FAST;
         } else if (!strcasecmp(flag,"no-auth")) {
             c->flags |= CMD_NO_AUTH;
+        } else if (!strcasecmp(flag,"may-replicate")) {
+            c->flags |= CMD_MAY_REPLICATE;
         } else {
             /* Parse ACL categories here if the flag name starts with @. */
             uint64_t catflag;
@@ -3439,6 +3454,10 @@ void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc,
     if (server.in_exec && !server.propagate_in_transaction)
         execCommandPropagateMulti(dbid);
 
+    /* This needs to be unreachable since the dataset should be fixed during 
+     * client pause, otherwise data may be lossed during a failover. */
+    serverAssert(!(areClientsPaused() && !server.client_pause_in_transaction));
+
     if (server.aof_state != AOF_OFF && flags & PROPAGATE_AOF)
         feedAppendOnlyFile(cmd,dbid,argv,argc);
     if (flags & PROPAGATE_REPL)
@@ -3477,6 +3496,7 @@ void alsoPropagate(struct redisCommand *cmd, int dbid, robj **argv, int argc,
  * Redis command implementation in order to to force the propagation of a
  * specific command execution into AOF / Replication. */
 void forceCommandPropagation(client *c, int flags) {
+    serverAssert(c->cmd->flags & (CMD_WRITE | CMD_MAY_REPLICATE));
     if (flags & PROPAGATE_REPL) c->flags |= CLIENT_FORCE_REPL;
     if (flags & PROPAGATE_AOF) c->flags |= CLIENT_FORCE_AOF;
 }
@@ -3701,6 +3721,12 @@ void call(client *c, int flags) {
     }
     server.also_propagate = prev_also_propagate;
 
+    /* Client pause takes effect after a transaction has finished. This needs
+     * to be located after everything is propagated. */
+    if (!server.in_exec && server.client_pause_in_transaction) {
+        server.client_pause_in_transaction = 0;
+    }
+
     /* If the client has keys tracking enabled for client side caching,
      * make sure to remember the keys it fetched via this command. */
     if (c->cmd->flags & CMD_READONLY) {
@@ -3814,6 +3840,8 @@ int processCommand(client *c) {
                                (c->cmd->proc == execCommand && (c->mstate.cmd_inv_flags & CMD_STALE));
     int is_denyloading_command = !(c->cmd->flags & CMD_LOADING) ||
                                  (c->cmd->proc == execCommand && (c->mstate.cmd_inv_flags & CMD_LOADING));
+    int is_may_replicate_command = (c->cmd->flags & (CMD_WRITE | CMD_MAY_REPLICATE)) ||
+                                   (c->cmd->proc == execCommand && (c->mstate.cmd_flags & (CMD_WRITE | CMD_MAY_REPLICATE)));
 
     /* Check if the user is authenticated. This check is skipped in case
      * the default user is flagged as "nopass" and is active. */
@@ -4012,6 +4040,17 @@ int processCommand(client *c) {
         return C_OK;
     }
 
+    /* If the server is paused, block the client until
+     * the pause has ended. Replicas are never paused. */
+    if (!(c->flags & CLIENT_SLAVE) && 
+        ((server.client_pause_type & CLIENT_PAUSE_ALL) ||
+        (server.client_pause_type & CLIENT_PAUSE_WRITE && is_may_replicate_command)))
+    {
+        c->bpop.timeout = 0;
+        blockClient(c,BLOCKED_PAUSE);
+        return C_OK;       
+    }
+
     /* Exec the command */
     if (c->flags & CLIENT_MULTI &&
         c->cmd->proc != execCommand && c->cmd->proc != discardCommand &&
@@ -4266,6 +4305,7 @@ void addReplyCommand(client *c, struct redisCommand *cmd) {
         flagcount += addReplyCommandFlag(c,cmd,CMD_ASKING, "asking");
         flagcount += addReplyCommandFlag(c,cmd,CMD_FAST, "fast");
         flagcount += addReplyCommandFlag(c,cmd,CMD_NO_AUTH, "no_auth");
+        flagcount += addReplyCommandFlag(c,cmd,CMD_MAY_REPLICATE, "may_replicate");
         if (cmdHasMovableKeys(cmd)) {
             addReplyStatus(c, "movablekeys");
             flagcount += 1;
diff --git a/src/server.h b/src/server.h
index fb6d71c71..586e085e9 100644
--- a/src/server.h
+++ b/src/server.h
@@ -181,33 +181,34 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
 #define CMD_ASKING (1ULL<<13)          /* "cluster-asking" flag */
 #define CMD_FAST (1ULL<<14)            /* "fast" flag */
 #define CMD_NO_AUTH (1ULL<<15)         /* "no-auth" flag */
+#define CMD_MAY_REPLICATE (1ULL<<16)   /* "may-replicate" flag */
 
 /* Command flags used by the module system. */
-#define CMD_MODULE_GETKEYS (1ULL<<16)  /* Use the modules getkeys interface. */
-#define CMD_MODULE_NO_CLUSTER (1ULL<<17) /* Deny on Redis Cluster. */
+#define CMD_MODULE_GETKEYS (1ULL<<17)  /* Use the modules getkeys interface. */
+#define CMD_MODULE_NO_CLUSTER (1ULL<<18) /* Deny on Redis Cluster. */
 
 /* Command flags that describe ACLs categories. */
-#define CMD_CATEGORY_KEYSPACE (1ULL<<18)
-#define CMD_CATEGORY_READ (1ULL<<19)
-#define CMD_CATEGORY_WRITE (1ULL<<20)
-#define CMD_CATEGORY_SET (1ULL<<21)
-#define CMD_CATEGORY_SORTEDSET (1ULL<<22)
-#define CMD_CATEGORY_LIST (1ULL<<23)
-#define CMD_CATEGORY_HASH (1ULL<<24)
-#define CMD_CATEGORY_STRING (1ULL<<25)
-#define CMD_CATEGORY_BITMAP (1ULL<<26)
-#define CMD_CATEGORY_HYPERLOGLOG (1ULL<<27)
-#define CMD_CATEGORY_GEO (1ULL<<28)
-#define CMD_CATEGORY_STREAM (1ULL<<29)
-#define CMD_CATEGORY_PUBSUB (1ULL<<30)
-#define CMD_CATEGORY_ADMIN (1ULL<<31)
-#define CMD_CATEGORY_FAST (1ULL<<32)
-#define CMD_CATEGORY_SLOW (1ULL<<33)
-#define CMD_CATEGORY_BLOCKING (1ULL<<34)
-#define CMD_CATEGORY_DANGEROUS (1ULL<<35)
-#define CMD_CATEGORY_CONNECTION (1ULL<<36)
-#define CMD_CATEGORY_TRANSACTION (1ULL<<37)
-#define CMD_CATEGORY_SCRIPTING (1ULL<<38)
+#define CMD_CATEGORY_KEYSPACE (1ULL<<19)
+#define CMD_CATEGORY_READ (1ULL<<20)
+#define CMD_CATEGORY_WRITE (1ULL<<21)
+#define CMD_CATEGORY_SET (1ULL<<22)
+#define CMD_CATEGORY_SORTEDSET (1ULL<<23)
+#define CMD_CATEGORY_LIST (1ULL<<24)
+#define CMD_CATEGORY_HASH (1ULL<<25)
+#define CMD_CATEGORY_STRING (1ULL<<26)
+#define CMD_CATEGORY_BITMAP (1ULL<<27)
+#define CMD_CATEGORY_HYPERLOGLOG (1ULL<<28)
+#define CMD_CATEGORY_GEO (1ULL<<29)
+#define CMD_CATEGORY_STREAM (1ULL<<30)
+#define CMD_CATEGORY_PUBSUB (1ULL<<31)
+#define CMD_CATEGORY_ADMIN (1ULL<<32)
+#define CMD_CATEGORY_FAST (1ULL<<33)
+#define CMD_CATEGORY_SLOW (1ULL<<34)
+#define CMD_CATEGORY_BLOCKING (1ULL<<35)
+#define CMD_CATEGORY_DANGEROUS (1ULL<<36)
+#define CMD_CATEGORY_CONNECTION (1ULL<<37)
+#define CMD_CATEGORY_TRANSACTION (1ULL<<38)
+#define CMD_CATEGORY_SCRIPTING (1ULL<<39)
 
 /* AOF states */
 #define AOF_OFF 0             /* AOF is off */
@@ -250,10 +251,8 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
 #define CLIENT_PENDING_READ (1<<29) /* The client has pending reads and was put
                                        in the list of clients we can read
                                        from. */
-#define CLIENT_PENDING_COMMAND (1<<30) /* Used in threaded I/O to signal after
-                                          we return single threaded that the
-                                          client has already pending commands
-                                          to be executed. */
+#define CLIENT_PENDING_COMMAND (1<<30) /* Indicates the client has a fully
+                                        * parsed command ready for execution. */
 #define CLIENT_TRACKING (1ULL<<31) /* Client enabled keys tracking in order to
                                    perform client side caching. */
 #define CLIENT_TRACKING_BROKEN_REDIR (1ULL<<32) /* Target client is invalid. */
@@ -280,7 +279,8 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
 #define BLOCKED_MODULE 3  /* Blocked by a loadable module. */
 #define BLOCKED_STREAM 4  /* XREAD. */
 #define BLOCKED_ZSET 5    /* BZPOP et al. */
-#define BLOCKED_NUM 6     /* Number of blocked states. */
+#define BLOCKED_PAUSE 6   /* Blocked by CLIENT PAUSE */
+#define BLOCKED_NUM 7     /* Number of blocked states. */
 
 /* Client request types */
 #define PROTO_REQ_INLINE 1
@@ -436,6 +436,14 @@ typedef enum {
 #define PROPAGATE_AOF 1
 #define PROPAGATE_REPL 2
 
+/* Client pause types, larger types are more restrictive
+ * pause types than smaller pause types. */
+typedef enum {
+    CLIENT_PAUSE_OFF = 0, /* Pause no commands */
+    CLIENT_PAUSE_WRITE,   /* Pause write commands */
+    CLIENT_PAUSE_ALL      /* Pause all commands */
+} pause_type;
+
 /* RDB active child save type. */
 #define RDB_CHILD_TYPE_NONE 0
 #define RDB_CHILD_TYPE_DISK 1     /* RDB is written to disk. */
@@ -893,6 +901,7 @@ typedef struct client {
     sds peerid;             /* Cached peer ID. */
     sds sockname;           /* Cached connection target address. */
     listNode *client_list_node; /* list node in client list */
+    listNode *paused_list_node; /* list node within the pause list */
     RedisModuleUserChangedFunc auth_callback; /* Module callback to execute
                                                * when the authenticated user
                                                * changes. */
@@ -1137,6 +1146,7 @@ struct redisServer {
     int in_exec;                /* Are we inside EXEC? */
     int propagate_in_transaction;  /* Make sure we don't propagate nested MULTI/EXEC */
     char *ignore_warnings;      /* Config: warnings that should be ignored. */
+    int client_pause_in_transaction; /* Was a client pause executed during this Exec? */
     /* Modules */
     dict *moduleapi;            /* Exported core APIs dictionary for modules. */
     dict *sharedapi;            /* Like moduleapi but containing the APIs that
@@ -1171,8 +1181,9 @@ struct redisServer {
     rax *clients_timeout_table; /* Radix tree for blocked clients timeouts. */
     long fixed_time_expire;     /* If > 0, expire keys against server.mstime. */
     rax *clients_index;         /* Active clients dictionary by client ID. */
-    int clients_paused;         /* True if clients are currently paused */
-    mstime_t clients_pause_end_time; /* Time when we undo clients_paused */
+    pause_type client_pause_type;      /* True if clients are currently paused */
+    list *paused_clients;       /* List of pause clients */
+    mstime_t client_pause_end_time;    /* Time when we undo clients_paused */
     char neterr[ANET_ERR_LEN];   /* Error buffer for anet.c */
     dict *migrate_cached_sockets;/* MIGRATE cached sockets */
     redisAtomic uint64_t next_client_id; /* Next client unique ID. Incremental. */
@@ -1794,8 +1805,10 @@ char *getClientTypeName(int class);
 void flushSlavesOutputBuffers(void);
 void disconnectSlaves(void);
 int listenToPort(int port, int *fds, int *count);
-void pauseClients(mstime_t duration);
-int clientsArePaused(void);
+void pauseClients(mstime_t duration, pause_type type);
+void unpauseClients(void);
+int areClientsPaused(void);
+int checkClientPauseTimeoutAndReturnIfPaused(void);
 void processEventsWhileBlocked(void);
 void loadingCron(void);
 void whileBlockedCron();
@@ -2111,6 +2124,7 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev
 size_t freeMemoryGetNotCountedMemory();
 int overMaxmemoryAfterAlloc(size_t moremem);
 int processCommand(client *c);
+int processPendingCommandsAndResetClient(client *c);
 void setupSignalHandlers(void);
 void removeSignalHandlers(void);
 struct redisCommand *lookupCommand(sds name);
diff --git a/tests/unit/pause.tcl b/tests/unit/pause.tcl
new file mode 100644
index 000000000..9f5cfd607
--- /dev/null
+++ b/tests/unit/pause.tcl
@@ -0,0 +1,200 @@
+start_server {tags {"pause"}} {
+    test "Test read commands are not blocked by client pause" {
+        r client PAUSE 100000000 WRITE
+        set rd [redis_deferring_client]
+        $rd GET FOO
+        $rd PING
+        $rd INFO
+        assert_equal [s 0 blocked_clients] 0
+        r client unpause
+        $rd close
+    }
+
+    test "Test write commands are paused by RO" {
+        r client PAUSE 100000000 WRITE
+
+        set rd [redis_deferring_client]
+        $rd SET FOO BAR
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
+
+        r client unpause
+        assert_match "OK" [$rd read]
+        $rd close
+    }
+
+    test "Test special commands are paused by RO" {
+        r PFADD pause-hll test
+        r client PAUSE 100000000 WRITE
+
+        # Test that pfcount, which can replicate, is also blocked
+        set rd [redis_deferring_client]
+        $rd PFCOUNT pause-hll
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
+
+        # Test that publish, which adds the message to the replication
+        # stream is blocked.
+        set rd2 [redis_deferring_client]
+        $rd2 publish foo bar
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {2}
+        } else {
+            fail "Clients are not blocked"
+        }
+
+        # Test that SCRIPT LOAD, which is replicated. 
+        set rd3 [redis_deferring_client]
+        $rd3 script load "return 1"
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {3}
+        } else {
+            fail "Clients are not blocked"
+        }
+
+        r client unpause 
+        assert_match "1" [$rd read]
+        assert_match "0" [$rd2 read]
+        assert_match "*" [$rd3 read]
+        $rd close
+        $rd2 close
+        $rd3 close
+    }
+
+    test "Test read/admin mutli-execs are not blocked by pause RO" {
+        r SET FOO BAR
+        r client PAUSE 100000000 WRITE
+        set rd [redis_deferring_client]
+        $rd MULTI
+        assert_equal [$rd read] "OK"
+        $rd PING
+        assert_equal [$rd read] "QUEUED"
+        $rd GET FOO
+        assert_equal [$rd read] "QUEUED"
+        $rd EXEC
+        assert_equal [s 0 blocked_clients] 0
+        r client unpause 
+        assert_match "PONG BAR" [$rd read]
+        $rd close
+    }
+
+    test "Test write mutli-execs are blocked by pause RO" {
+        set rd [redis_deferring_client]
+        $rd MULTI
+        assert_equal [$rd read] "OK"
+        $rd SET FOO BAR
+        r client PAUSE 100000000 WRITE
+        assert_equal [$rd read] "QUEUED"
+        $rd EXEC
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
+        r client unpause 
+        assert_match "OK" [$rd read]
+        $rd close
+    }
+
+    test "Test scripts are blocked by pause RO" {
+        r client PAUSE 100000000 WRITE
+        set rd [redis_deferring_client]
+        $rd EVAL "return 1" 0
+        
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
+        r client unpause 
+        assert_match "1" [$rd read]
+        $rd close
+    }
+
+    test "Test multiple clients can be queued up and unblocked" {
+        r client PAUSE 100000000 WRITE
+        set clients [list [redis_deferring_client] [redis_deferring_client] [redis_deferring_client]]
+        foreach client $clients {
+            $client SET FOO BAR
+        }
+        
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {3}
+        } else {
+            fail "Clients are not blocked"
+        }
+        r client unpause
+        foreach client $clients {
+            assert_match "OK" [$client read]
+            $client close
+        }
+    }
+
+    test "Test clients with syntax errors will get responses immediately" {
+        r client PAUSE 100000000 WRITE
+        catch {r set FOO} err
+        assert_match "ERR wrong number of arguments for *" $err
+        r client unpause
+    }
+
+    test "Test both active and passive expiries are skipped during client pause" {
+        set expired_keys [s 0 expired_keys]
+        r multi
+        r set foo bar PX 10
+        r set bar foo PX 10
+        r client PAUSE 100000000 WRITE
+        r exec
+
+        wait_for_condition 10 100 {
+            [r get foo] == {} && [r get bar] == {}
+        } else {
+            fail "Keys were never logically expired"
+        }
+
+        # No keys should actually have been expired
+        assert_match $expired_keys [s 0 expired_keys]
+
+        r client unpause
+
+        # Force the keys to expire
+        r get foo
+        r get bar
+
+        # Now that clients have been unpaused, expires should go through
+        assert_match [expr $expired_keys + 2] [s 0 expired_keys]   
+    }
+
+    test "Test that client pause starts at the end of a transaction" {
+        r MULTI
+        r SET FOO1 BAR
+        r client PAUSE 100000000 WRITE
+        r SET FOO2 BAR
+        r exec
+
+        set rd [redis_deferring_client]
+        $rd SET FOO3 BAR
+        
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Clients are not blocked"
+        }
+
+        assert_match "BAR" [r GET FOO1]
+        assert_match "BAR" [r GET FOO2]
+        assert_match "" [r GET FOO3]
+
+        r client unpause 
+        assert_match "OK" [$rd read]
+        $rd close
+    }
+
+    # Make sure we unpause at the end
+    r client unpause
+}
diff --git a/tests/unit/wait.tcl b/tests/unit/wait.tcl
index b1500cff8..b20900c23 100644
--- a/tests/unit/wait.tcl
+++ b/tests/unit/wait.tcl
@@ -42,4 +42,14 @@ start_server {} {
         $master incr foo
         assert {[$master wait 1 3000] == 0}
     }
+
+    test {WAIT implicitly blocks on client pause since ACKs aren't sent} {
+        $master multi
+        $master incr foo
+        $master client pause 10000 write
+        $master exec
+        assert {[$master wait 1 1000] == 0}
+        $master client unpause
+        assert {[$master wait 1 1000] == 1}
+    }
 }}

From 999494cef8247e660f079dd8ae884109444c8345 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Fri, 8 Jan 2021 00:00:35 -0800
Subject: [PATCH 0953/1098] Throw error for conflicting bcast tracking prefixes
 (#8176)

Throw an error if there are conflicting bcast tracking prefixes.
---
 src/networking.c        |  7 ++++++
 src/server.h            |  1 +
 src/tracking.c          | 51 +++++++++++++++++++++++++++++++++++++++++
 tests/unit/tracking.tcl | 20 +++++++++++++++-
 4 files changed, 78 insertions(+), 1 deletion(-)

diff --git a/src/networking.c b/src/networking.c
index 9a86e6085..e624dd8f9 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2800,6 +2800,13 @@ NULL
                 return;
             }
 
+            if (options & CLIENT_TRACKING_BCAST) {
+                if (!checkPrefixCollisionsOrReply(c,prefix,numprefix)) {
+                    zfree(prefix);
+                    return;
+                }
+            }
+
             enableTracking(c,redir,options,prefix,numprefix);
         } else if (!strcasecmp(c->argv[2]->ptr,"off")) {
             disableTracking(c);
diff --git a/src/server.h b/src/server.h
index 586e085e9..9fe15685e 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1849,6 +1849,7 @@ uint64_t trackingGetTotalItems(void);
 uint64_t trackingGetTotalKeys(void);
 uint64_t trackingGetTotalPrefixes(void);
 void trackingBroadcastInvalidationMessages(void);
+int checkPrefixCollisionsOrReply(client *c, robj **prefix, size_t numprefix);
 
 /* List data type */
 void listTypeTryConversion(robj *subject, robj *value);
diff --git a/src/tracking.c b/src/tracking.c
index 852fa229b..1cf226e52 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -99,6 +99,57 @@ void disableTracking(client *c) {
     }
 }
 
+static int stringCheckPrefix(unsigned char *s1, size_t s1_len, unsigned char *s2, size_t s2_len) {
+    size_t min_length = s1_len < s2_len ? s1_len : s2_len;
+    return memcmp(s1,s2,min_length) == 0;   
+}
+
+/* Check if any of the provided prefixes collide with one another or
+ * with an existing prefix for the client. A collision is defined as two 
+ * prefixes that will emit an invalidation for the same key. If no prefix 
+ * collision is found, 1 is return, otherwise 0 is returned and the client 
+ * has an error emitted describing the error. */
+int checkPrefixCollisionsOrReply(client *c, robj **prefixes, size_t numprefix) {
+    for (size_t i = 0; i < numprefix; i++) {
+        /* Check input list has no overlap with existing prefixes. */
+        if (c->client_tracking_prefixes) {
+            raxIterator ri;
+            raxStart(&ri,c->client_tracking_prefixes);
+            raxSeek(&ri,"^",NULL,0);
+            while(raxNext(&ri)) {
+                if (stringCheckPrefix(ri.key,ri.key_len,
+                    prefixes[i]->ptr,sdslen(prefixes[i]->ptr))) 
+                {
+                    sds collision = sdsnewlen(ri.key,ri.key_len);
+                    addReplyErrorFormat(c,
+                        "Prefix '%s' overlaps with an existing prefix '%s'. "
+                        "Prefixes for a single client must not overlap.",
+                        (unsigned char *)prefixes[i]->ptr,
+                        (unsigned char *)collision);
+                    sdsfree(collision);
+                    raxStop(&ri);
+                    return 0;
+                }
+            }
+            raxStop(&ri);
+        }
+        /* Check input has no overlap with itself. */
+        for (size_t j = i + 1; j < numprefix; j++) {
+            if (stringCheckPrefix(prefixes[i]->ptr,sdslen(prefixes[i]->ptr),
+                prefixes[j]->ptr,sdslen(prefixes[j]->ptr)))
+            {
+                addReplyErrorFormat(c,
+                    "Prefix '%s' overlaps with another provided prefix '%s'. "
+                    "Prefixes for a single client must not overlap.",
+                    (unsigned char *)prefixes[i]->ptr,
+                    (unsigned char *)prefixes[j]->ptr);
+                return i;
+            }
+        }
+    }
+    return -1;
+}
+
 /* Set the client 'c' to track the prefix 'prefix'. If the client 'c' is
  * already registered for the specified prefix, no operation is performed. */
 void enableBcastTrackingForPrefix(client *c, char *prefix, size_t plen) {
diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index 8ed2801e7..88cf9dc42 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -323,7 +323,25 @@ start_server {tags {"tracking"}} {
         set ping_reply [$rd read]
         assert {$inv_msg eq {invalidate key1}}
         assert {$ping_reply eq {PONG}}
-    } 
+    }
+
+    test {BCAST with prefix collisions throw errors} {
+        set r [redis_client] 
+        catch {$r CLIENT TRACKING ON BCAST PREFIX FOOBAR PREFIX FOO} output
+        assert_match {ERR Prefix 'FOOBAR'*'FOO'*} $output
+
+        catch {$r CLIENT TRACKING ON BCAST PREFIX FOO PREFIX FOOBAR} output
+        assert_match {ERR Prefix 'FOO'*'FOOBAR'*} $output
+
+        $r CLIENT TRACKING ON BCAST PREFIX FOO PREFIX BAR
+        catch {$r CLIENT TRACKING ON BCAST PREFIX FO} output
+        assert_match {ERR Prefix 'FO'*'FOO'*} $output
+
+        catch {$r CLIENT TRACKING ON BCAST PREFIX BARB} output
+        assert_match {ERR Prefix 'BARB'*'BAR'*} $output
+
+        $r CLIENT TRACKING OFF
+    }
 
     test {Tracking gets notification on tracking table key eviction} {
         $rd_redirection HELLO 2

From 5843a45d01f76c55288abe00c29931a2520fe521 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 8 Jan 2021 10:03:21 +0200
Subject: [PATCH 0954/1098] Skip defrag tests on systems with bigger page sizes
 (#8294)

The defragger works well on these systems, but the tests and their
thresholds are not adjusted for these big pages, so the defragger isn't
able to get down the fragmentation to the levels the test expects and it
fails on "defrag didn't stop".

Randomly choosing 8k as the threshold for the skipping

Fixes #8265 (which had 65k pages)
---
 tests/unit/memefficiency.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl
index 357089c8f..88fcd6f51 100644
--- a/tests/unit/memefficiency.tcl
+++ b/tests/unit/memefficiency.tcl
@@ -38,7 +38,7 @@ start_server {tags {"memefficiency"}} {
 
 run_solo {defrag} {
 start_server {tags {"defrag"} overrides {appendonly yes auto-aof-rewrite-percentage 0 save ""}} {
-    if {[string match {*jemalloc*} [s mem_allocator]]} {
+    if {[string match {*jemalloc*} [s mem_allocator]] && [r debug mallctl arenas.page] <= 8192} {
         test "Active defrag" {
             r config set hz 100
             r config set activedefrag no

From 814aad65f16ed27cb68c4177e4f8b61d7edb31e1 Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Fri, 8 Jan 2021 18:13:25 +0200
Subject: [PATCH 0955/1098] XADD and XTRIM, Trim by MINID, and new LIMIT
 argument (#8169)

This PR adds another trimming strategy to XADD and XTRIM named MINID
(complements the existing MAXLEN).
It also adds a new LIMIT argument that allows incremental trimming by repeated
calls (rather than all at once).

This provides the ability to trim all records older than a certain ID (which makes it
possible for the user to trim by age too).
Example:
XTRIM mystream MINID ~ 1608540753 will trim entries with id < 1608540753,
but might not trim all (because of the ~ modifier)

The purpose is to ease the use of streams. many users use streams as logs and
the common case is wanting a log
of the last X seconds rather than a log that contains maximum X entries (new
MINID vs existing MAXLEN)

The new LIMIT modifier is only supported when the trim strategy uses ~.
i.e. when the user asked for exact trimming, it all happens in one go (no
possibility for incremental trimming).
However, when ~ is provided, we trim full rax nodes, up to the limit number
of records.
The default limit is 100*stream_node_max_entries (used when LIMIT is not
provided).
I.e. this is a behavior change (even if the existing MAXLEN strategy is used).
An explicit limit of 0 means unlimited (but note that it's not the default).

Other changes:

Refactor arg parsing code for XADD and XTRIM to use common code.
---
 src/t_stream.c             | 583 +++++++++++++++++++++++++++----------
 tests/unit/type/stream.tcl | 140 ++++++++-
 2 files changed, 571 insertions(+), 152 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index d085a8948..f991765eb 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -46,6 +46,8 @@
 void streamFreeCG(streamCG *cg);
 void streamFreeNACK(streamNACK *na);
 size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start, streamID *end, size_t count, streamConsumer *consumer);
+int streamParseStrictIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq);
+int streamParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq);
 
 /* -----------------------------------------------------------------------
  * Low level stream encoding: a radix tree of listpacks.
@@ -282,6 +284,65 @@ static inline int64_t lpGetIntegerIfValid(unsigned char *ele, int *valid) {
 
 #define lpGetInteger(ele) lpGetIntegerIfValid(ele, NULL)
 
+/* Get an edge streamID of a given listpack.
+ * 'master_id' is an input param, used to build the 'edge_id' output param */
+int lpGetEdgeStreamID(unsigned char *lp, int first, streamID *master_id, streamID *edge_id)
+{
+   if (lp == NULL)
+       return 0;
+
+   unsigned char *lp_ele;
+
+   /* We need to seek either the first or the last entry depending
+    * on the direction of the iteration. */
+   if (first) {
+       /* Get the master fields count. */
+       lp_ele = lpFirst(lp);        /* Seek items count */
+       lp_ele = lpNext(lp, lp_ele); /* Seek deleted count. */
+       lp_ele = lpNext(lp, lp_ele); /* Seek num fields. */
+       int64_t master_fields_count = lpGetInteger(lp_ele);
+       lp_ele = lpNext(lp, lp_ele); /* Seek first field. */
+
+       /* If we are iterating in normal order, skip the master fields
+        * to seek the first actual entry. */
+       for (int64_t i = 0; i < master_fields_count; i++)
+           lp_ele = lpNext(lp, lp_ele);
+
+       /* If we are going forward, skip the previous entry's
+        * lp-count field (or in case of the master entry, the zero
+        * term field) */
+       lp_ele = lpNext(lp, lp_ele);
+       if (lp_ele == NULL)
+           return 0;
+   } else {
+       /* If we are iterating in reverse direction, just seek the
+        * last part of the last entry in the listpack (that is, the
+        * fields count). */
+       lp_ele = lpLast(lp);
+
+       /* If we are going backward, read the number of elements this
+        * entry is composed of, and jump backward N times to seek
+        * its start. */
+       int64_t lp_count = lpGetInteger(lp_ele);
+       if (lp_count == 0) /* We reached the master entry. */
+           return 0;
+
+       while (lp_count--)
+           lp_ele = lpPrev(lp, lp_ele);
+   }
+
+   lp_ele = lpNext(lp, lp_ele); /* Seek ID (lp_ele currently points to 'flags'). */
+
+   /* Get the ID: it is encoded as difference between the master
+    * ID and this entry ID. */
+   streamID id = *master_id;
+   id.ms += lpGetInteger(lp_ele);
+   lp_ele = lpNext(lp, lp_ele);
+   id.seq += lpGetInteger(lp_ele);
+   *edge_id = id;
+   return 1;
+}
+
 /* Debugging function to log the full content of a listpack. Useful
  * for development and debugging. */
 void streamLogListpackContent(unsigned char *lp) {
@@ -325,6 +386,39 @@ int streamCompareID(streamID *a, streamID *b) {
     return 0;
 }
 
+void streamGetEdgeID(stream *s, int first, streamID *edge_id)
+{
+    raxIterator ri;
+    raxStart(&ri, s->rax);
+    int empty;
+    if (first) {
+        raxSeek(&ri, "^", NULL, 0);
+        empty = !raxNext(&ri);
+    } else {
+        raxSeek(&ri, "$", NULL, 0);
+        empty = !raxPrev(&ri);
+    }
+
+    if (empty) {
+        /* Stream is empty, mark edge ID as lowest/highest possible. */
+        edge_id->ms = first ? UINT64_MAX : 0;
+        edge_id->seq = first ? UINT64_MAX : 0;
+        raxStop(&ri);
+        return;
+    }
+
+    unsigned char *lp = ri.data;
+
+    /* Read the master ID from the radix tree key. */
+    streamID master_id;
+    streamDecodeID(ri.key, &master_id);
+
+    /* Construct edge ID. */
+    lpGetEdgeStreamID(lp, first, &master_id, edge_id);
+
+    raxStop(&ri);
+}
+
 /* Adds a new item into the stream 's' having the specified number of
  * field-value pairs as specified in 'numfields' and stored into 'argv'.
  * Returns the new entry ID populating the 'added_id' structure.
@@ -525,35 +619,96 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_
     return C_OK;
 }
 
-/* Trim the stream 's' to have no more than maxlen elements, and return the
+typedef struct {
+    /* XADD options */
+    streamID id; /* User-provided ID, for XADD only. */
+    int id_given; /* Was an ID different than "*" specified? for XADD only. */
+    int no_mkstream; /* if set to 1 do not create new stream */
+
+    /* XADD + XTRIM common options */
+    int trim_strategy; /* TRIM_STRATEGY_* */
+    int trim_strategy_arg_idx; /* Index of the count in MAXLEN/MINID, for rewriting. */
+    int approx_trim; /* If 1 only delete whole radix tree nodes, so
+                      * the trim argument is not applied verbatim. */
+    long long limit; /* Maximum amount of entries to trim. If 0, no limitation
+                      * on the amount of trimming work is enforced. */
+    /* TRIM_STRATEGY_MAXLEN options */
+    long long maxlen; /* After trimming, leave stream at this length . */
+    /* TRIM_STRATEGY_MINID options */
+    streamID minid; /* Trim by ID (No stream entries with ID < 'minid' will remain) */
+} streamAddTrimArgs;
+
+#define TRIM_STRATEGY_NONE 0
+#define TRIM_STRATEGY_MAXLEN 1
+#define TRIM_STRATEGY_MINID 2
+
+/* Trim the stream 's' according to args->trim_strategy, and return the
  * number of elements removed from the stream. The 'approx' option, if non-zero,
  * specifies that the trimming must be performed in a approximated way in
  * order to maximize performances. This means that the stream may contain
- * more elements than 'maxlen', and elements are only removed if we can remove
+ * entries with IDs < 'id' in case of MINID (or more elements than 'maxlen'
+ * in case of MAXLEN), and elements are only removed if we can remove
  * a *whole* node of the radix tree. The elements are removed from the head
  * of the stream (older elements).
  *
  * The function may return zero if:
  *
- * 1) The stream is already shorter or equal to the specified max length.
- * 2) The 'approx' option is true and the head node had not enough elements
- *    to be deleted, leaving the stream with a number of elements >= maxlen.
+ * 1) The minimal entry ID of the stream is already < 'id' (MINID); or
+ * 2) The stream is already shorter or equal to the specified max length (MAXLEN); or
+ * 3) The 'approx' option is true and the head node did not have enough elements
+ *    to be deleted.
+ *
+ * args->limit is the maximum number of entries to delete. The purpose is to
+ * prevent this function from taking to long.
+ * If 'limit' is 0 then we do not limit the number of deleted entries.
+ * Much like the 'approx', if 'limit' is smaller than the number of entries
+ * that should be trimmed, there is a chance we will still have entries with
+ * IDs < 'id' (or number of elements >= maxlen in case of MAXLEN).
  */
-int64_t streamTrimByLength(stream *s, size_t maxlen, int approx) {
-    if (s->length <= maxlen) return 0;
+int64_t streamTrim(stream *s, streamAddTrimArgs *args) {
+    size_t maxlen = args->maxlen;
+    streamID *id = &args->minid;
+    int approx = args->approx_trim;
+    int64_t limit = args->limit;
+    int trim_strategy = args->trim_strategy;
+
+    if (trim_strategy == TRIM_STRATEGY_NONE)
+        return 0;
 
     raxIterator ri;
     raxStart(&ri,s->rax);
     raxSeek(&ri,"^",NULL,0);
 
     int64_t deleted = 0;
-    while(s->length > maxlen && raxNext(&ri)) {
+    while (raxNext(&ri)) {
+        /* Check if we exceeded the amount of work we could do */
+        if (limit && deleted >= limit)
+            break;
+
+        if (trim_strategy == TRIM_STRATEGY_MAXLEN && s->length <= maxlen)
+            break;
+
         unsigned char *lp = ri.data, *p = lpFirst(lp);
         int64_t entries = lpGetInteger(p);
 
-        /* Check if we can remove the whole node, and still have at
-         * least maxlen elements. */
-        if (s->length - entries >= maxlen) {
+        /* Check if we can remove the whole node. */
+        int remove_node;
+        streamID master_id = {0}; /* For MINID */
+        if (trim_strategy == TRIM_STRATEGY_MAXLEN) {
+            remove_node = s->length - entries >= maxlen;
+        } else {
+            /* Read the master ID from the radix tree key. */
+            streamDecodeID(ri.key, &master_id);
+
+            /* Read last ID. */
+            streamID last_id;
+            lpGetEdgeStreamID(lp, 0, &master_id, &last_id);
+
+            /* We can remove the entire node id its last ID < 'id' */
+            remove_node = streamCompareID(&last_id, id) < 0;
+        }
+
+        if (remove_node) {
             lpFree(lp);
             raxRemove(s->rax,ri.key,ri.key_len,NULL);
             raxSeek(&ri,">=",ri.key,ri.key_len);
@@ -566,19 +721,15 @@ int64_t streamTrimByLength(stream *s, size_t maxlen, int approx) {
          * stop here. */
         if (approx) break;
 
-        /* Otherwise, we have to mark single entries inside the listpack
-         * as deleted. We start by updating the entries/deleted counters. */
-        int64_t to_delete = s->length - maxlen;
-        serverAssert(to_delete < entries);
-        lp = lpReplaceInteger(lp,&p,entries-to_delete);
-        p = lpNext(lp,p); /* Seek deleted field. */
-        int64_t marked_deleted = lpGetInteger(p);
-        lp = lpReplaceInteger(lp,&p,marked_deleted+to_delete);
-        p = lpNext(lp,p); /* Seek num-of-fields in the master entry. */
+        /* Now we have to trim entries from within 'lp' */
+        int64_t deleted_from_lp = 0;
+
+        p = lpNext(lp, p); /* Skip deleted field. */
+        p = lpNext(lp, p); /* Skip num-of-fields in the master entry. */
 
         /* Skip all the master fields. */
         int64_t master_fields_count = lpGetInteger(p);
-        p = lpNext(lp,p); /* Seek the first field. */
+        p = lpNext(lp,p); /* Skip the first field. */
         for (int64_t j = 0; j < master_fields_count; j++)
             p = lpNext(lp,p); /* Skip all master fields. */
         p = lpNext(lp,p); /* Skip the zero master entry terminator. */
@@ -586,37 +737,72 @@ int64_t streamTrimByLength(stream *s, size_t maxlen, int approx) {
         /* 'p' is now pointing to the first entry inside the listpack.
          * We have to run entry after entry, marking entries as deleted
          * if they are already not deleted. */
-        while(p) {
+        while (p) {
+            /* We keep a copy of p (which point to flags part) in order to
+             * update it after (and if) we actually remove the entry */
+            unsigned char *pcopy = p;
+
             int flags = lpGetInteger(p);
+            p = lpNext(lp, p); /* Skip flags. */
             int to_skip;
 
-            /* Mark the entry as deleted. */
-            if (!(flags & STREAM_ITEM_FLAG_DELETED)) {
-                flags |= STREAM_ITEM_FLAG_DELETED;
-                lp = lpReplaceInteger(lp,&p,flags);
-                deleted++;
-                s->length--;
-                if (s->length <= maxlen) break; /* Enough entries deleted. */
+            int ms_delta = lpGetInteger(p);
+            p = lpNext(lp, p); /* Skip ID ms delta */
+            int seq_delta = lpGetInteger(p);
+            p = lpNext(lp, p); /* Skip ID seq delta */
+
+            streamID currid = {0}; /* For MINID */
+            if (trim_strategy == TRIM_STRATEGY_MINID) {
+                currid.ms = master_id.ms + ms_delta;
+                currid.seq = master_id.seq + seq_delta;
             }
 
-            p = lpNext(lp,p); /* Skip ID ms delta. */
-            p = lpNext(lp,p); /* Skip ID seq delta. */
-            p = lpNext(lp,p); /* Seek num-fields or values (if compressed). */
+            int stop;
+            if (trim_strategy == TRIM_STRATEGY_MAXLEN) {
+                stop = s->length <= maxlen;
+            } else {
+                /* Following IDs will definitely be greater because the rax
+                 * tree is sorted, no point of continuing. */
+                stop = streamCompareID(&currid, id) >= 0;
+            }
+            if (stop)
+                break;
+
             if (flags & STREAM_ITEM_FLAG_SAMEFIELDS) {
                 to_skip = master_fields_count;
             } else {
-                to_skip = lpGetInteger(p);
-                to_skip = 1+(to_skip*2);
+                to_skip = lpGetInteger(p); /* Get num-fields. */
+                p = lpNext(lp,p); /* Skip num-fields. */
+                to_skip *= 2; /* Fields and values. */
             }
 
             while(to_skip--) p = lpNext(lp,p); /* Skip the whole entry. */
             p = lpNext(lp,p); /* Skip the final lp-count field. */
+
+            /* Mark the entry as deleted. */
+            if (!(flags & STREAM_ITEM_FLAG_DELETED)) {
+                intptr_t delta = p - lp;
+                flags |= STREAM_ITEM_FLAG_DELETED;
+                lp = lpReplaceInteger(lp, &pcopy, flags);
+                deleted_from_lp++;
+                s->length--;
+                p = lp + delta;
+            }
         }
+        deleted += deleted_from_lp;
+
+        /* Now we the entries/deleted counters. */
+        p = lpFirst(lp);
+        lp = lpReplaceInteger(lp,&p,entries-deleted_from_lp);
+        p = lpNext(lp,p); /* Skip deleted field. */
+        int64_t marked_deleted = lpGetInteger(p);
+        lp = lpReplaceInteger(lp,&p,marked_deleted+deleted_from_lp);
+        p = lpNext(lp,p); /* Skip num-of-fields in the master entry. */
 
         /* Here we should perform garbage collection in case at this point
          * there are too many entries deleted inside the listpack. */
-        entries -= to_delete;
-        marked_deleted += to_delete;
+        entries -= deleted_from_lp;
+        marked_deleted += deleted_from_lp;
         if (entries + marked_deleted > 10 && marked_deleted > entries/2) {
             /* TODO: perform a garbage collection. */
         }
@@ -632,6 +818,142 @@ int64_t streamTrimByLength(stream *s, size_t maxlen, int approx) {
     return deleted;
 }
 
+/* Parse the arguements of XADD/XTRIM.
+ *
+ * See streamAddTrimArgs for more details about the arguments handled.
+ *
+ * This function returns the position of the ID argument (relevant only to XADD).
+ * On error -1 is returned and a reply is sent. */
+static int streamParseAddOrTrimArgsOrReply(client *c, streamAddTrimArgs *args, int xadd) {
+    /* Initialize arguments to defaults */
+    memset(args, 0, sizeof(*args));
+
+    /* Parse options. */
+    int i = 2; /* This is the first argument position where we could
+                  find an option, or the ID. */
+    int limit_given = 0;
+    for (; i < c->argc; i++) {
+        int moreargs = (c->argc-1) - i; /* Number of additional arguments. */
+        char *opt = c->argv[i]->ptr;
+        if (xadd && opt[0] == '*' && opt[1] == '\0') {
+            /* This is just a fast path for the common case of auto-ID
+             * creation. */
+            break;
+        } else if (!strcasecmp(opt,"maxlen") && moreargs) {
+            if (args->trim_strategy != TRIM_STRATEGY_NONE) {
+                addReplyError(c,"syntax error, MAXLEN and MINID options at the same time are not compatible");
+                return -1;
+            }
+            args->approx_trim = 0;
+            char *next = c->argv[i+1]->ptr;
+            /* Check for the form MAXLEN ~ <count>. */
+            if (moreargs >= 2 && next[0] == '~' && next[1] == '\0') {
+                args->approx_trim = 1;
+                i++;
+            } else if (moreargs >= 2 && next[0] == '=' && next[1] == '\0') {
+                i++;
+            }
+            if (getLongLongFromObjectOrReply(c,c->argv[i+1],&args->maxlen,NULL)
+                != C_OK) return -1;
+
+            if (args->maxlen < 0) {
+                addReplyError(c,"The MAXLEN argument must be >= 0.");
+                return -1;
+            }
+            i++;
+            args->trim_strategy = TRIM_STRATEGY_MAXLEN;
+            args->trim_strategy_arg_idx = i;
+        } else if (!strcasecmp(opt,"minid") && moreargs) {
+            if (args->trim_strategy != TRIM_STRATEGY_NONE) {
+                addReplyError(c,"syntax error, MAXLEN and MINID options at the same time are not compatible");
+                return -1;
+            }
+            args->approx_trim = 0;
+            char *next = c->argv[i+1]->ptr;
+            /* Check for the form MINID ~ <id>|<age>. */
+            if (moreargs >= 2 && next[0] == '~' && next[1] == '\0') {
+                args->approx_trim = 1;
+                i++;
+            } else if (moreargs >= 2 && next[0] == '=' && next[1] == '\0') {
+                i++;
+            }
+
+            if (streamParseStrictIDOrReply(c,c->argv[i+1],&args->minid,0) != C_OK)
+                return -1;
+            
+            i++;
+            args->trim_strategy = TRIM_STRATEGY_MINID;
+            args->trim_strategy_arg_idx = i;
+        } else if (!strcasecmp(opt,"limit") && moreargs) {
+            /* Note about LIMIT: If it was not provided by the caller we set
+             * it to 100*server.stream_node_max_entries, and that's to prevent the
+             * trimming from taking too long, on the expense of not deleting entries
+             * that should be trimmed.
+             * If user wanted exact trimming (i.e. no '~') we never limit the number
+             * of trimmed entries */
+            if (getLongLongFromObjectOrReply(c,c->argv[i+1],&args->limit,NULL) != C_OK)
+                return -1;
+
+            if (args->limit < 0) {
+                addReplyError(c,"The LIMIT argument must be >= 0.");
+                return -1;
+            }
+            limit_given = 1;
+            i++;
+        } else if (xadd && !strcasecmp(opt,"nomkstream")) {
+            args->no_mkstream = 1;
+        } else if (xadd) {
+            /* If we are here is a syntax error or a valid ID. */
+            if (streamParseStrictIDOrReply(c,c->argv[i],&args->id,0) != C_OK)
+                return -1;
+            args->id_given = 1;
+            break;
+        } else {
+            addReplyErrorObject(c,shared.syntaxerr);
+            return -1;
+        }
+    }
+
+    if (args->limit && args->trim_strategy == TRIM_STRATEGY_NONE) {
+        addReplyError(c,"syntax error, LIMIT cannot be used without specifying a trimming strategy");
+        return -1;
+    }
+
+    if (!xadd && args->trim_strategy == TRIM_STRATEGY_NONE) {
+        addReplyError(c,"syntax error, XTRIM must be called with a trimming strategy");
+        return -1;
+    }
+
+    if (c == server.master || c->id == CLIENT_ID_AOF) {
+        /* If command cam from master or from AOF we must not enforce maxnodes
+         * (The maxlen/minid argument was re-written to make sure there's no
+         * inconsistency). */
+        args->limit = 0;
+    } else {
+        /* We need to set the limit (only if we got '~') */
+        if (limit_given) {
+            if (!args->approx_trim) {
+                /* LIMIT was provided without ~ */
+                addReplyError(c,"syntax error, LIMIT cannot be used without the special ~ option");
+                return -1;
+            }
+        } else {
+            /* User didn't provide LIMIT, we must set it. */
+
+            if (args->approx_trim) {
+                /* In order to prevent from trimming to do too much work and cause
+                 * latency spikes we limit the amount of work it can do */
+                args->limit = 100 * server.stream_node_max_entries; /* Maximum 100 rax nodes. */
+            } else {
+                /* No LIMIT for exact trimming */
+                args->limit = 0;
+            }
+        }
+    }
+
+    return i;
+}
+
 /* Initialize the stream iterator, so that we can call iterating functions
  * to get the next items. This requires a corresponding streamIteratorStop()
  * at the end. The 'rev' parameter controls the direction. If it's zero the
@@ -1375,68 +1697,36 @@ int streamParseIntervalIDOrReply(client *c, robj *o, streamID *id, int *exclude,
     return C_OK;
 }
 
-/* We propagate MAXLEN ~ <count> as MAXLEN = <resulting-len-of-stream>
- * otherwise trimming is no longer determinsitic on replicas / AOF. */
-void streamRewriteApproxMaxlen(client *c, stream *s, int maxlen_arg_idx) {
-    robj *maxlen_obj = createStringObjectFromLongLong(s->length);
+void streamRewriteApproxSpecifier(client *c, int idx) {
     robj *equal_obj = createStringObject("=",1);
-
-    rewriteClientCommandArgument(c,maxlen_arg_idx,maxlen_obj);
-    rewriteClientCommandArgument(c,maxlen_arg_idx-1,equal_obj);
-
+    rewriteClientCommandArgument(c,idx,equal_obj);
     decrRefCount(equal_obj);
-    decrRefCount(maxlen_obj);
 }
 
-/* XADD key [MAXLEN [~|=] <count>] [NOMKSTREAM] <ID or *> [field value] [field value] ... */
-void xaddCommand(client *c) {
-    streamID id;
-    int id_given = 0; /* Was an ID different than "*" specified? */
-    long long maxlen = -1;  /* If left to -1 no trimming is performed. */
-    int approx_maxlen = 0;  /* If 1 only delete whole radix tree nodes, so
-                               the maximum length is not applied verbatim. */
-    int maxlen_arg_idx = 0; /* Index of the count in MAXLEN, for rewriting. */
-    int no_mkstream = 0; /* if set to 1 do not create new stream */
-
-    /* Parse options. */
-    int i = 2; /* This is the first argument position where we could
-                  find an option, or the ID. */
-    for (; i < c->argc; i++) {
-        int moreargs = (c->argc-1) - i; /* Number of additional arguments. */
-        char *opt = c->argv[i]->ptr;
-        if (opt[0] == '*' && opt[1] == '\0') {
-            /* This is just a fast path for the common case of auto-ID
-             * creation. */
-            break;
-        } else if (!strcasecmp(opt,"maxlen") && moreargs) {
-            approx_maxlen = 0;
-            char *next = c->argv[i+1]->ptr;
-            /* Check for the form MAXLEN ~ <count>. */
-            if (moreargs >= 2 && next[0] == '~' && next[1] == '\0') {
-                approx_maxlen = 1;
-                i++;
-            } else if (moreargs >= 2 && next[0] == '=' && next[1] == '\0') {
-                i++;
-            }
-            if (getLongLongFromObjectOrReply(c,c->argv[i+1],&maxlen,NULL)
-                != C_OK) return;
-
-            if (maxlen < 0) {
-                addReplyError(c,"The MAXLEN argument must be >= 0.");
-                return;
-            }
-            i++;
-            maxlen_arg_idx = i;
-        } else if (!strcasecmp(opt,"nomkstream")) {
-            no_mkstream = 1;
-        } else {
-            /* If we are here is a syntax error or a valid ID. */
-            if (streamParseStrictIDOrReply(c,c->argv[i],&id,0) != C_OK) return;
-            id_given = 1;
-            break;
-        }
+/* We propagate MAXLEN/MINID ~ <count> as MAXLEN/MINID = <resulting-len-of-stream>
+ * otherwise trimming is no longer deterministic on replicas / AOF. */
+void streamRewriteTrimArgument(client *c, stream *s, int trim_strategy, int idx) {
+    robj *arg;
+    if (trim_strategy == TRIM_STRATEGY_MAXLEN) {
+        arg = createStringObjectFromLongLong(s->length);
+    } else {
+        streamID first_id;
+        streamGetEdgeID(s, 1, &first_id);
+        arg = createObjectFromStreamID(&first_id);
     }
-    int field_pos = i+1;
+
+    rewriteClientCommandArgument(c,idx,arg);
+    decrRefCount(arg);
+}
+
+/* XADD key [(MAXLEN [~|=] <count> | MINID [~|=] <id>) [LIMIT <entries>]] [NOMKSTREAM] <ID or *> [field value] [field value] ... */
+void xaddCommand(client *c) {
+    /* Parse options. */
+    streamAddTrimArgs parsed_args;
+    int idpos = streamParseAddOrTrimArgsOrReply(c, &parsed_args, 1);
+    if (idpos < 0)
+        return; /* streamParseAddOrTrimArgsOrReply already replied. */
+    int field_pos = idpos+1; /* The ID is always one argument before the first field */
 
     /* Check arity. */
     if ((c->argc - field_pos) < 2 || ((c->argc-field_pos) % 2) == 1) {
@@ -1447,7 +1737,9 @@ void xaddCommand(client *c) {
     /* Return ASAP if minimal ID (0-0) was given so we avoid possibly creating
      * a new stream and have streamAppendItem fail, leaving an empty key in the
      * database. */
-    if (id_given && id.ms == 0 && id.seq == 0) {
+    if (parsed_args.id_given &&
+        parsed_args.id.ms == 0 && parsed_args.id.seq == 0)
+    {
         addReplyError(c,"The ID specified in XADD must be greater than 0-0");
         return;
     }
@@ -1455,7 +1747,7 @@ void xaddCommand(client *c) {
     /* Lookup the stream at key. */
     robj *o;
     stream *s;
-    if ((o = streamTypeLookupWriteOrCreate(c,c->argv[1],no_mkstream)) == NULL) return;
+    if ((o = streamTypeLookupWriteOrCreate(c,c->argv[1],parsed_args.no_mkstream)) == NULL) return;
     s = o->ptr;
 
     /* Return ASAP if the stream has reached the last possible ID */
@@ -1466,8 +1758,9 @@ void xaddCommand(client *c) {
     }
 
     /* Append using the low level function and return the ID. */
+    streamID id;
     if (streamAppendItem(s,c->argv+field_pos,(c->argc-field_pos)/2,
-        &id, id_given ? &id : NULL)
+        &id, parsed_args.id_given ? &parsed_args.id : NULL)
         == C_ERR)
     {
         addReplyError(c,"The ID specified in XADD is equal or smaller than the "
@@ -1480,18 +1773,26 @@ void xaddCommand(client *c) {
     notifyKeyspaceEvent(NOTIFY_STREAM,"xadd",c->argv[1],c->db->id);
     server.dirty++;
 
-    if (maxlen >= 0) {
-        /* Notify xtrim event if needed. */
-        if (streamTrimByLength(s,maxlen,approx_maxlen)) {
+    /* Trim if needed. */
+    if (parsed_args.trim_strategy != TRIM_STRATEGY_NONE) {
+        if (streamTrim(s, &parsed_args)) {
             notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id);
         }
-        if (approx_maxlen) streamRewriteApproxMaxlen(c,s,maxlen_arg_idx);
+        if (parsed_args.approx_trim) {
+            /* In case our trimming was limited (by LIMIT or by ~) we must
+             * re-write the relevant trim argument to make sure there will be
+             * no inconsistencies in AOF loading or in the replica.
+             * It's enough to check only args->approx because there is no
+             * way LIMIT is given without the ~ option. */
+            streamRewriteApproxSpecifier(c,parsed_args.trim_strategy_arg_idx-1);
+            streamRewriteTrimArgument(c,s,parsed_args.trim_strategy,parsed_args.trim_strategy_arg_idx);
+        }
     }
 
     /* Let's rewrite the ID argument with the one actually generated for
      * AOF/replication propagation. */
     robj *idarg = createObjectFromStreamID(&id);
-    rewriteClientCommandArgument(c,i,idarg);
+    rewriteClientCommandArgument(c,idpos,idarg);
     decrRefCount(idarg);
 
     /* We need to signal to blocked clients that there is new data on this
@@ -2871,14 +3172,25 @@ cleanup:
  *
  * List of options:
  *
+ * Trim strategies:
+ *
  * MAXLEN [~|=] <count>     -- Trim so that the stream will be capped at
  *                             the specified length. Use ~ before the
  *                             count in order to demand approximated trimming
  *                             (like XADD MAXLEN option).
+ * MINID [~|=] <id>         -- Trim so that the stream will not contain entries
+ *                             with IDs smaller than 'id'. Use ~ before the
+ *                             count in order to demand approximated trimming
+ *                             (like XADD MINID option).
+ *
+ * Other options:
+ *
+ * LIMIT <entries>          -- The maximum number of entries to trim.
+ *                             0 means unlimited. Unless specified, it is set
+ *                             to a default of 100*server.stream_node_max_entries,
+ *                             and that's in order to keep the trimming time sane.
+ *                             Has meaning only if `~` was provided.
  */
-
-#define TRIM_STRATEGY_NONE 0
-#define TRIM_STRATEGY_MAXLEN 1
 void xtrimCommand(client *c) {
     robj *o;
 
@@ -2889,58 +3201,27 @@ void xtrimCommand(client *c) {
     stream *s = o->ptr;
 
     /* Argument parsing. */
-    int trim_strategy = TRIM_STRATEGY_NONE;
-    long long maxlen = -1;  /* If left to -1 no trimming is performed. */
-    int approx_maxlen = 0;  /* If 1 only delete whole radix tree nodes, so
-                               the maxium length is not applied verbatim. */
-    int maxlen_arg_idx = 0; /* Index of the count in MAXLEN, for rewriting. */
-
-    /* Parse options. */
-    int i = 2; /* Start of options. */
-    for (; i < c->argc; i++) {
-        int moreargs = (c->argc-1) - i; /* Number of additional arguments. */
-        char *opt = c->argv[i]->ptr;
-        if (!strcasecmp(opt,"maxlen") && moreargs) {
-            approx_maxlen = 0;
-            trim_strategy = TRIM_STRATEGY_MAXLEN;
-            char *next = c->argv[i+1]->ptr;
-            /* Check for the form MAXLEN ~ <count>. */
-            if (moreargs >= 2 && next[0] == '~' && next[1] == '\0') {
-                approx_maxlen = 1;
-                i++;
-            } else if (moreargs >= 2 && next[0] == '=' && next[1] == '\0') {
-                i++;
-            }
-            if (getLongLongFromObjectOrReply(c,c->argv[i+1],&maxlen,NULL)
-                != C_OK) return;
-
-            if (maxlen < 0) {
-                addReplyError(c,"The MAXLEN argument must be >= 0.");
-                return;
-            }
-            i++;
-            maxlen_arg_idx = i;
-        } else {
-            addReplyErrorObject(c,shared.syntaxerr);
-            return;
-        }
-    }
+    streamAddTrimArgs parsed_args;
+    if (streamParseAddOrTrimArgsOrReply(c, &parsed_args, 1) < 0)
+        return; /* streamParseAddOrTrimArgsOrReply already replied. */
 
     /* Perform the trimming. */
-    int64_t deleted = 0;
-    if (trim_strategy == TRIM_STRATEGY_MAXLEN) {
-        deleted = streamTrimByLength(s,maxlen,approx_maxlen);
-    } else {
-        addReplyError(c,"XTRIM called without an option to trim the stream");
-        return;
-    }
-
-    /* Propagate the write if needed. */
+    int64_t deleted = streamTrim(s, &parsed_args);
     if (deleted) {
-        signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id);
+        if (parsed_args.approx_trim) {
+            /* In case our trimming was limited (by LIMIT or by ~) we must
+             * re-write the relevant trim argument to make sure there will be
+             * no inconsistencies in AOF loading or in the replica.
+             * It's enough to check only args->approx because there is no
+             * way LIMIT is given without the ~ option. */
+            streamRewriteApproxSpecifier(c,parsed_args.trim_strategy_arg_idx-1);
+            streamRewriteTrimArgument(c,s,parsed_args.trim_strategy,parsed_args.trim_strategy_arg_idx);
+        }
+
+        /* Propagate the write. */
+        signalModifiedKey(c, c->db,c->argv[1]);
         server.dirty += deleted;
-        if (approx_maxlen) streamRewriteApproxMaxlen(c,s,maxlen_arg_idx);
     }
     addReplyLongLong(c,deleted);
 }
diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl
index 63cc697c2..a89a65299 100644
--- a/tests/unit/type/stream.tcl
+++ b/tests/unit/type/stream.tcl
@@ -94,6 +94,7 @@ start_server {
                 r XADD mystream MAXLEN 5 * yitem $j
             }
         }
+        assert {[r xlen mystream] == 5}
         set res [r xrange mystream - +]
         set expected 995
         foreach r $res {
@@ -138,6 +139,39 @@ start_server {
         assert_equal [lindex $items 1 1] {item 2 value b}
     }
 
+    test {XADD with MINID option} {
+        r DEL mystream
+        for {set j 1} {$j < 1001} {incr j} {
+            set minid 1000
+            if {$j >= 5} {
+                set minid [expr {$j-5}]
+            }
+            if {rand() < 0.9} {
+                r XADD mystream MINID $minid $j xitem $j
+            } else {
+                r XADD mystream MINID $minid $j yitem $j
+            }
+        }
+        assert {[r xlen mystream] == 6}
+        set res [r xrange mystream - +]
+        set expected 995
+        foreach r $res {
+            assert {[lindex $r 1 1] == $expected}
+            incr expected
+        }
+    }
+
+    test {XTRIM with MINID option} {
+        r DEL mystream
+        r XADD mystream 1-0 f v
+        r XADD mystream 2-0 f v
+        r XADD mystream 3-0 f v
+        r XADD mystream 4-0 f v
+        r XADD mystream 5-0 f v
+        r XTRIM mystream MINID = 3-0
+        assert_equal [r XRANGE mystream - +] {{3-0 {f v}} {4-0 {f v}} {5-0 {f v}}}
+    }
+
     test {XADD mass insertion and XLEN} {
         r DEL mystream
         r multi
@@ -448,7 +482,49 @@ start_server {
         assert {[r XLEN mystream] == 400}
     }
 
-    
+    test {XADD with LIMIT consecutive calls} {
+        r del mystream
+        r config set stream-node-max-entries 10
+        for {set j 0} {$j < 100} {incr j} {
+            r XADD mystream * xitem v
+        }
+        r XADD mystream MAXLEN ~ 55 LIMIT 30 * xitem v
+        assert {[r xlen mystream] == 71}
+        r XADD mystream MAXLEN ~ 55 LIMIT 30 * xitem v
+        assert {[r xlen mystream] == 62}
+        r config set stream-node-max-entries 100
+    }
+
+    test {XTRIM with ~ is limited} {
+        r del mystream
+        r config set stream-node-max-entries 1
+        for {set j 0} {$j < 102} {incr j} {
+            r XADD mystream * xitem v
+        }
+        r XTRIM mystream MAXLEN ~ 1
+        assert {[r xlen mystream] == 2}
+        r config set stream-node-max-entries 100
+    }
+
+    test {XTRIM without ~ is not limited} {
+        r del mystream
+        r config set stream-node-max-entries 1
+        for {set j 0} {$j < 102} {incr j} {
+            r XADD mystream * xitem v
+        }
+        r XTRIM mystream MAXLEN 1
+        assert {[r xlen mystream] == 1}
+        r config set stream-node-max-entries 100
+    }
+
+    test {XTRIM without ~ and with LIMIT} {
+        r del mystream
+        r config set stream-node-max-entries 1
+        for {set j 0} {$j < 102} {incr j} {
+            r XADD mystream * xitem v
+        }
+        assert_error ERR* {r XTRIM mystream MAXLEN 1 LIMIT 30}
+    }
 }
 
 start_server {tags {"stream"} overrides {appendonly yes}} {
@@ -466,6 +542,22 @@ start_server {tags {"stream"} overrides {appendonly yes}} {
     }
 }
 
+start_server {tags {"stream"} overrides {appendonly yes}} {
+    test {XADD with MINID > lastid can propagate correctly} {
+        for {set j 0} {$j < 100} {incr j} {
+            set id [expr {$j+1}]
+            r XADD mystream $id xitem v
+        }
+        r XADD mystream MINID 1 * xitem v
+        incr j
+        assert {[r xlen mystream] == $j}
+        r debug loadaof
+        r XADD mystream * xitem v
+        incr j
+        assert {[r xlen mystream] == $j}
+    }
+}
+
 start_server {tags {"stream"} overrides {appendonly yes}} {
     test {XADD with ~ MAXLEN can propagate correctly} {
         for {set j 0} {$j < 100} {incr j} {
@@ -482,6 +574,52 @@ start_server {tags {"stream"} overrides {appendonly yes}} {
     }
 }
 
+start_server {tags {"stream"} overrides {appendonly yes stream-node-max-entries 10}} {
+    test {XADD with ~ MAXLEN and LIMIT can propagate correctly} {
+        for {set j 0} {$j < 100} {incr j} {
+            r XADD mystream * xitem v
+        }
+        r XADD mystream MAXLEN ~ 55 LIMIT 30 * xitem v
+        assert {[r xlen mystream] == 71}
+        r config set stream-node-max-entries 1
+        r debug loadaof
+        r XADD mystream * xitem v
+        assert {[r xlen mystream] == 72}
+    }
+}
+
+start_server {tags {"stream"} overrides {appendonly yes}} {
+    test {XADD with ~ MINID can propagate correctly} {
+        for {set j 0} {$j < 100} {incr j} {
+            set id [expr {$j+1}]
+            r XADD mystream $id xitem v
+        }
+        r XADD mystream MINID ~ $j * xitem v
+        incr j
+        assert {[r xlen mystream] == $j}
+        r config set stream-node-max-entries 1
+        r debug loadaof
+        r XADD mystream * xitem v
+        incr j
+        assert {[r xlen mystream] == $j}
+    }
+}
+
+start_server {tags {"stream"} overrides {appendonly yes stream-node-max-entries 10}} {
+    test {XADD with ~ MINID and LIMIT can propagate correctly} {
+        for {set j 0} {$j < 100} {incr j} {
+            set id [expr {$j+1}]
+            r XADD mystream $id xitem v
+        }
+        r XADD mystream MINID ~ 55 LIMIT 30 * xitem v
+        assert {[r xlen mystream] == 71}
+        r config set stream-node-max-entries 1
+        r debug loadaof
+        r XADD mystream * xitem v
+        assert {[r xlen mystream] == 72}
+    }
+}
+
 start_server {tags {"stream"} overrides {appendonly yes stream-node-max-entries 10}} {
     test {XTRIM with ~ MAXLEN can propagate correctly} {
         for {set j 0} {$j < 100} {incr j} {

From ea5350c5ecb08d8915bd59b2dc98982454adc9f7 Mon Sep 17 00:00:00 2001
From: Yang Bodong <bodong.ybd@alibaba-inc.com>
Date: Sat, 9 Jan 2021 00:29:44 +0800
Subject: [PATCH 0956/1098] GEOSEARCH - ANY option, for limited search that
 returns ASAP (#8259)

Support ANY option to return some results that match the criteria ASAP,
without a complete search and implicit sorting.
---
 src/geo.c          | 41 +++++++++++++++++++++++++++--------------
 tests/unit/geo.tcl | 13 +++++++++++++
 2 files changed, 40 insertions(+), 14 deletions(-)

diff --git a/src/geo.c b/src/geo.c
index e9c8bf5cb..9c668288b 100644
--- a/src/geo.c
+++ b/src/geo.c
@@ -249,7 +249,7 @@ int geoAppendIfWithinShape(geoArray *ga, GeoShape *shape, double score, sds memb
  * using multiple queries to the sorted set, that we later need to sort
  * via qsort. Similarly we need to be able to reject points outside the search
  * radius area ASAP in order to allocate and process more points than needed. */
-int geoGetPointsInRange(robj *zobj, double min, double max, GeoShape *shape, geoArray *ga) {
+int geoGetPointsInRange(robj *zobj, double min, double max, GeoShape *shape, geoArray *ga, unsigned long limit) {
     /* minex 0 = include min in range; maxex 1 = exclude max in range */
     /* That's: min <= val < max */
     zrangespec range = { .min = min, .max = max, .minex = 0, .maxex = 1 };
@@ -283,6 +283,7 @@ int geoGetPointsInRange(robj *zobj, double min, double max, GeoShape *shape, geo
                                       sdsnewlen(vstr,vlen);
             if (geoAppendIfWithinShape(ga,shape,score,member)
                 == C_ERR) sdsfree(member);
+            if (ga->used && limit && ga->used >= limit) break;
             zzlNext(zl, &eptr, &sptr);
         }
     } else if (zobj->encoding == OBJ_ENCODING_SKIPLIST) {
@@ -304,6 +305,7 @@ int geoGetPointsInRange(robj *zobj, double min, double max, GeoShape *shape, geo
             ele = sdsdup(ele);
             if (geoAppendIfWithinShape(ga,shape,ln->score,ele)
                 == C_ERR) sdsfree(ele);
+            if (ga->used && limit && ga->used >= limit) break;
             ln = ln->level[0].forward;
         }
     }
@@ -342,15 +344,15 @@ void scoresOfGeoHashBox(GeoHashBits hash, GeoHashFix52Bits *min, GeoHashFix52Bit
 /* Obtain all members between the min/max of this geohash bounding box.
  * Populate a geoArray of GeoPoints by calling geoGetPointsInRange().
  * Return the number of points added to the array. */
-int membersOfGeoHashBox(robj *zobj, GeoHashBits hash, geoArray *ga, GeoShape *shape) {
+int membersOfGeoHashBox(robj *zobj, GeoHashBits hash, geoArray *ga, GeoShape *shape, unsigned long limit) {
     GeoHashFix52Bits min, max;
 
     scoresOfGeoHashBox(hash,&min,&max);
-    return geoGetPointsInRange(zobj, min, max, shape, ga);
+    return geoGetPointsInRange(zobj, min, max, shape, ga, limit);
 }
 
 /* Search all eight neighbors + self geohash box */
-int membersOfAllNeighbors(robj *zobj, GeoHashRadius n, GeoShape *shape, geoArray *ga) {
+int membersOfAllNeighbors(robj *zobj, GeoHashRadius n, GeoShape *shape, geoArray *ga, unsigned long limit) {
     GeoHashBits neighbors[9];
     unsigned int i, count = 0, last_processed = 0;
     int debugmsg = 0;
@@ -401,7 +403,8 @@ int membersOfAllNeighbors(robj *zobj, GeoHashRadius n, GeoShape *shape, geoArray
                 D("Skipping processing of %d, same as previous\n",i);
             continue;
         }
-        count += membersOfGeoHashBox(zobj, neighbors[i], ga, shape);
+        if (ga->used && limit && ga->used >= limit) break;
+        count += membersOfGeoHashBox(zobj, neighbors[i], ga, shape, limit);
         last_processed = i;
     }
     return count;
@@ -500,12 +503,12 @@ void geoaddCommand(client *c) {
 #define GEOSEARCHSTORE (1<<4)   /* GEOSEARCHSTORE just accept STOREDIST option */
 
 /* GEORADIUS key x y radius unit [WITHDIST] [WITHHASH] [WITHCOORD] [ASC|DESC]
- *                               [COUNT count] [STORE key] [STOREDIST key]
+ *                               [COUNT count [ANY]] [STORE key] [STOREDIST key]
  * GEORADIUSBYMEMBER key member radius unit ... options ...
  * GEOSEARCH key [FROMMEMBER member] [FROMLONLAT long lat] [BYRADIUS radius unit]
- *               [BYBOX width height unit] [WITHCORD] [WITHDIST] [WITHASH] [COUNT count] [ASC|DESC]
+ *               [BYBOX width height unit] [WITHCORD] [WITHDIST] [WITHASH] [COUNT count [ANY]] [ASC|DESC]
  * GEOSEARCHSTORE dest_key src_key [FROMMEMBER member] [FROMLONLAT long lat] [BYRADIUS radius unit]
- *               [BYBOX width height unit] [WITHCORD] [WITHDIST] [WITHASH] [COUNT count] [ASC|DESC] [STOREDIST]
+ *               [BYBOX width height unit] [WITHCORD] [WITHDIST] [WITHASH] [COUNT count [ANY]] [ASC|DESC] [STOREDIST]
  *  */
 void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
     robj *storekey = NULL;
@@ -550,7 +553,8 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
     int withdist = 0, withhash = 0, withcoords = 0;
     int frommember = 0, fromloc = 0, byradius = 0, bybox = 0;
     int sort = SORT_NONE;
-    long long count = 0;
+    int any = 0; /* any=1 means a limited search, stop as soon as enough results were found. */
+    long long count = 0;  /* Max number of results to return. 0 means unlimited. */
     if (c->argc > base_args) {
         int remaining = c->argc - base_args;
         for (int i = 0; i < remaining; i++) {
@@ -561,13 +565,15 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
                 withhash = 1;
             } else if (!strcasecmp(arg, "withcoord")) {
                 withcoords = 1;
+            } else if (!strcasecmp(arg, "any")) {
+                any = 1;
             } else if (!strcasecmp(arg, "asc")) {
                 sort = SORT_ASC;
             } else if (!strcasecmp(arg, "desc")) {
                 sort = SORT_DESC;
             } else if (!strcasecmp(arg, "count") && (i+1) < remaining) {
                 if (getLongLongFromObjectOrReply(c, c->argv[base_args+i+1],
-                    &count, NULL) != C_OK) return;
+                                                 &count, NULL) != C_OK) return;
                 if (count <= 0) {
                     addReplyError(c,"COUNT must be > 0");
                     return;
@@ -662,16 +668,23 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
         return;
     }
 
-    /* COUNT without ordering does not make much sense, force ASC
-     * ordering if COUNT was specified but no sorting was requested. */
-    if (count != 0 && sort == SORT_NONE) sort = SORT_ASC;
+    if (any && !count) {
+        addReplyErrorFormat(c, "the ANY argument requires COUNT argument");
+        return;
+    }
+
+    /* COUNT without ordering does not make much sense (we need to
+     * sort in order to return the closest N entries),
+     * force ASC ordering if COUNT was specified but no sorting was
+     * requested. Note that this is not needed for ANY option. */
+    if (count != 0 && sort == SORT_NONE && !any) sort = SORT_ASC;
 
     /* Get all neighbor geohash boxes for our radius search */
     GeoHashRadius georadius = geohashCalculateAreasByShapeWGS84(&shape);
 
     /* Search the zset for all matching points */
     geoArray *ga = geoArrayCreate();
-    membersOfAllNeighbors(zobj, georadius, &shape, ga);
+    membersOfAllNeighbors(zobj, georadius, &shape, ga, any ? count : 0);
 
     /* If no matching results, the user gets an empty reply. */
     if (ga->used == 0 && storekey == NULL) {
diff --git a/tests/unit/geo.tcl b/tests/unit/geo.tcl
index 6c1f2ef11..0ac95f7c5 100644
--- a/tests/unit/geo.tcl
+++ b/tests/unit/geo.tcl
@@ -178,6 +178,19 @@ start_server {tags {"geo"}} {
         r georadius nyc -73.9798091 40.7598464 10 km COUNT 3
     } {{central park n/q/r} 4545 {union square}}
 
+    test {GEORADIUS with ANY not sorted by default} {
+        r georadius nyc -73.9798091 40.7598464 10 km COUNT 3 ANY
+    } {{wtc one} {union square} {central park n/q/r}}
+
+    test {GEORADIUS with ANY sorted by ASC} {
+        r georadius nyc -73.9798091 40.7598464 10 km COUNT 3 ANY ASC
+    } {{central park n/q/r} {union square} {wtc one}}
+
+    test {GEORADIUS with ANY but no COUNT} {
+        catch {r georadius nyc -73.9798091 40.7598464 10 km ANY ASC} e
+        set e
+    } {ERR*ANY*requires*COUNT*}
+
     test {GEORADIUS with COUNT but missing integer argument} {
         catch {r georadius nyc -73.9798091 40.7598464 10 km COUNT} e
         set e

From 8dd16caec833aebde3a12c7d7f9b6456554c8616 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Fri, 8 Jan 2021 23:35:30 +0200
Subject: [PATCH 0957/1098] Fix last COW INFO report, Skip test on non-linux
 platforms (#8301)

- the last COW report wasn't always read from the pipe
  (receiveLastChildInfo wasn't used)
- but in fact, there's no reason we won't always try to drain that pipe
  so i'm unifying receiveLastChildInfo with receiveChildInfo
- adjust threshold of the COW test when run in accurate mode
- add some prints in case this test fails again
- fix indentation, page size, and PID! in MacOS proc info

p.s. it seems that pri_pages_dirtied is always 0
---
 src/childinfo.c           | 18 +++---------------
 src/server.h              |  1 -
 src/zmalloc.c             | 19 +++++++++++--------
 tests/integration/rdb.tcl | 24 ++++++++++++++++++++----
 4 files changed, 34 insertions(+), 28 deletions(-)

diff --git a/src/childinfo.c b/src/childinfo.c
index 85adfe2f6..cae73fe46 100644
--- a/src/childinfo.c
+++ b/src/childinfo.c
@@ -127,21 +127,9 @@ void receiveChildInfo(void) {
     int process_type;
     int on_exit;
     size_t cow_size;
-    if (readChildInfo(&process_type, &on_exit, &cow_size)) {
-        updateChildInfo(process_type, on_exit, cow_size);
-    }
-}
-
-/* Receive last COW data from child. */
-void receiveLastChildInfo(void) {
-    if (server.child_info_pipe[0] == -1) return;
-
-    /* Drain the pipe and update child info */
-    int process_type;
-    int on_exit;
-    size_t cow_size;
-
-    while (readChildInfo(&process_type, &on_exit, &cow_size) > 0) {
+
+    /* Drain the pipe and update child info so that we get the final message. */
+    while (readChildInfo(&process_type, &on_exit, &cow_size)) {
         updateChildInfo(process_type, on_exit, cow_size);
     }
 }
diff --git a/src/server.h b/src/server.h
index 9fe15685e..eb967a042 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2013,7 +2013,6 @@ void openChildInfoPipe(void);
 void closeChildInfoPipe(void);
 void sendChildInfo(int process_type, int on_exit, size_t cow_size);
 void receiveChildInfo(void);
-void receiveLastChildInfo(void);
 
 /* Fork helpers */
 int redisFork(int type);
diff --git a/src/zmalloc.c b/src/zmalloc.c
index 86b15b0ee..eacce67bd 100644
--- a/src/zmalloc.c
+++ b/src/zmalloc.c
@@ -580,15 +580,18 @@ size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) {
 size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) {
 #if defined(__APPLE__)
     struct proc_regioninfo pri;
-    if (proc_pidinfo(pid, PROC_PIDREGIONINFO, 0, &pri, PROC_PIDREGIONINFO_SIZE) ==
-	PROC_PIDREGIONINFO_SIZE) {
-	if (!strcmp(field, "Private_Dirty:")) {
-            return (size_t)pri.pri_pages_dirtied * 4096;
-	} else if (!strcmp(field, "Rss:")) {
-            return (size_t)pri.pri_pages_resident * 4096;
-	} else if (!strcmp(field, "AnonHugePages:")) {
+    if (pid == -1) pid = getpid();
+    if (proc_pidinfo(pid, PROC_PIDREGIONINFO, 0, &pri,
+                     PROC_PIDREGIONINFO_SIZE) == PROC_PIDREGIONINFO_SIZE)
+    {
+        int pagesize = getpagesize();
+        if (!strcmp(field, "Private_Dirty:")) {
+            return (size_t)pri.pri_pages_dirtied * pagesize;
+        } else if (!strcmp(field, "Rss:")) {
+            return (size_t)pri.pri_pages_resident * pagesize;
+        } else if (!strcmp(field, "AnonHugePages:")) {
             return 0;
-	}
+        }
     }
     return 0;
 #endif
diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl
index cd4dfbadb..99495b2b7 100644
--- a/tests/integration/rdb.tcl
+++ b/tests/integration/rdb.tcl
@@ -199,6 +199,10 @@ test {client freed during loading} {
     }
 }
 
+# Our COW metrics (Private_Dirty) work only on Linux
+set system_name [string tolower [exec uname -s]]
+if {$system_name eq {linux}} {
+
 start_server {overrides {save ""}} {
     test {Test child sending COW info} {
         # make sure that rdb_last_cow_size and current_cow_size are zero (the test using new server),
@@ -209,6 +213,7 @@ start_server {overrides {save ""}} {
         # using a 200us delay, the bgsave is empirically taking about 10 seconds.
         # we need it to take more than some 5 seconds, since redis only report COW once a second.
         r config set rdb-key-save-delay 200
+        r config set loglevel debug
 
         # populate the db with 10k keys of 4k each
         set rd [redis_deferring_client 0]
@@ -247,8 +252,9 @@ start_server {overrides {save ""}} {
             } else {
                 if {$::verbose} {
                     puts "COW info on fail: [s current_cow_size]"
+                    puts [exec tail -n 100 < [srv 0 stdout]]
                 }
-                fail "COW info didn't report"
+                fail "COW info wasn't reported"
             }
 
             # for no accurate, stop after 2 iterations
@@ -265,11 +271,21 @@ start_server {overrides {save ""}} {
         }
 
         # make sure we saw report of current_cow_size
+        if {$iteration < 2 && $::verbose} {
+            puts [exec tail -n 100 < [srv 0 stdout]]
+        }
         assert_morethan_equal $iteration 2
 
-        # if bgsave completed, check that rdb_last_cow_size value is at least as last rdb_active_cow_size.
+        # if bgsave completed, check that rdb_last_cow_size (fork exit report)
+        # is at least 90% of last rdb_active_cow_size.
         if { [s rdb_bgsave_in_progress] == 0 } {
-            assert_morethan_equal [s rdb_last_cow_size] $cow_size
+            set final_cow [s rdb_last_cow_size]
+            set cow_size [expr $cow_size * 0.9]
+            if {$final_cow < $cow_size && $::verbose} {
+                puts [exec tail -n 100 < [srv 0 stdout]]
+            }
+            assert_morethan_equal $final_cow $cow_size
         }
     }
-}
\ No newline at end of file
+}
+} ;# system_name

From 36f1dea5e19982fac81d44de5bb0a4febe456b06 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Sun, 10 Jan 2021 01:52:42 +0800
Subject: [PATCH 0958/1098] Fix compile warning when define REDIS_TEST (#8261)

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/crc64.c     |  4 ++--
 src/intset.c    |  6 ------
 src/quicklist.c | 23 +++++++----------------
 src/rax.c       |  2 +-
 src/ziplist.c   |  4 ++--
 5 files changed, 12 insertions(+), 27 deletions(-)

diff --git a/src/crc64.c b/src/crc64.c
index 4cbc019f6..6c9432c4a 100644
--- a/src/crc64.c
+++ b/src/crc64.c
@@ -134,7 +134,7 @@ int crc64Test(int argc, char *argv[]) {
     printf("[calcula]: e9c6d914c4b8d9ca == %016" PRIx64 "\n",
            (uint64_t)_crc64(0, "123456789", 9));
     printf("[64speed]: e9c6d914c4b8d9ca == %016" PRIx64 "\n",
-           (uint64_t)crc64(0, "123456789", 9));
+           (uint64_t)crc64(0, (unsigned char*)"123456789", 9));
     char li[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed "
                 "do eiusmod tempor incididunt ut labore et dolore magna "
                 "aliqua. Ut enim ad minim veniam, quis nostrud exercitation "
@@ -146,7 +146,7 @@ int crc64Test(int argc, char *argv[]) {
     printf("[calcula]: c7794709e69683b3 == %016" PRIx64 "\n",
            (uint64_t)_crc64(0, li, sizeof(li)));
     printf("[64speed]: c7794709e69683b3 == %016" PRIx64 "\n",
-           (uint64_t)crc64(0, li, sizeof(li)));
+           (uint64_t)crc64(0, (unsigned char*)li, sizeof(li)));
     return 0;
 }
 
diff --git a/src/intset.c b/src/intset.c
index 54afae875..74de87acb 100644
--- a/src/intset.c
+++ b/src/intset.c
@@ -358,12 +358,6 @@ static long long usec(void) {
     return (((long long)tv.tv_sec)*1000000)+tv.tv_usec;
 }
 
-#define assert(_e) ((_e)?(void)0:(_assert(#_e,__FILE__,__LINE__),exit(1)))
-static void _assert(char *estr, char *file, int line) {
-    printf("\n\n=== ASSERTION FAILED ===\n");
-    printf("==> %s:%d '%s' is not true\n",file,line,estr);
-}
-
 static intset *createSet(int bits, int size) {
     uint64_t mask = (1<<bits)-1;
     uint64_t value;
diff --git a/src/quicklist.c b/src/quicklist.c
index 4dc3576ae..32871ac12 100644
--- a/src/quicklist.c
+++ b/src/quicklist.c
@@ -66,7 +66,7 @@ static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536};
 #else
 #define D(...)                                                                 \
     do {                                                                       \
-        printf("%s:%s:%d:\t", __FILE__, __FUNCTION__, __LINE__);               \
+        printf("%s:%s:%d:\t", __FILE__, __func__, __LINE__);                   \
         printf(__VA_ARGS__);                                                   \
         printf("\n");                                                          \
     } while (0)
@@ -1508,15 +1508,6 @@ void quicklistBookmarksClear(quicklist *ql) {
 #include <stdint.h>
 #include <sys/time.h>
 
-#define assert(_e)                                                             \
-    do {                                                                       \
-        if (!(_e)) {                                                           \
-            printf("\n\n=== ASSERTION FAILED ===\n");                          \
-            printf("==> %s:%d '%s' is not true\n", __FILE__, __LINE__, #_e);   \
-            err++;                                                             \
-        }                                                                      \
-    } while (0)
-
 #define yell(str, ...) printf("ERROR! " str "\n\n", __VA_ARGS__)
 
 #define OK printf("\tOK\n")
@@ -1529,7 +1520,7 @@ void quicklistBookmarksClear(quicklist *ql) {
 
 #define ERR(x, ...)                                                            \
     do {                                                                       \
-        printf("%s:%s:%d:\t", __FILE__, __FUNCTION__, __LINE__);               \
+        printf("%s:%s:%d:\t", __FILE__, __func__, __LINE__);                   \
         printf("ERROR! " x "\n", __VA_ARGS__);                                 \
         err++;                                                                 \
     } while (0)
@@ -1614,7 +1605,7 @@ static int _ql_verify(quicklist *ql, uint32_t len, uint32_t count,
 
     ql_info(ql);
     if (len != ql->len) {
-        yell("quicklist length wrong: expected %d, got %u", len, ql->len);
+        yell("quicklist length wrong: expected %d, got %lu", len, ql->len);
         errors++;
     }
 
@@ -1670,7 +1661,7 @@ static int _ql_verify(quicklist *ql, uint32_t len, uint32_t count,
                 if (node->encoding != QUICKLIST_NODE_ENCODING_RAW) {
                     yell("Incorrect compression: node %d is "
                          "compressed at depth %d ((%u, %u); total "
-                         "nodes: %u; size: %u; recompress: %d)",
+                         "nodes: %lu; size: %u; recompress: %d)",
                          at, ql->compress, low_raw, high_raw, ql->len, node->sz,
                          node->recompress);
                     errors++;
@@ -1680,7 +1671,7 @@ static int _ql_verify(quicklist *ql, uint32_t len, uint32_t count,
                     !node->attempted_compress) {
                     yell("Incorrect non-compression: node %d is NOT "
                          "compressed at depth %d ((%u, %u); total "
-                         "nodes: %u; size: %u; recompress: %d; attempted: %d)",
+                         "nodes: %lu; size: %u; recompress: %d; attempted: %d)",
                          at, ql->compress, low_raw, high_raw, ql->len, node->sz,
                          node->recompress, node->attempted_compress);
                     errors++;
@@ -2706,7 +2697,7 @@ int quicklistTest(int argc, char *argv[]) {
                             if (node->encoding != QUICKLIST_NODE_ENCODING_RAW) {
                                 ERR("Incorrect compression: node %d is "
                                     "compressed at depth %d ((%u, %u); total "
-                                    "nodes: %u; size: %u)",
+                                    "nodes: %lu; size: %u)",
                                     at, depth, low_raw, high_raw, ql->len,
                                     node->sz);
                             }
@@ -2714,7 +2705,7 @@ int quicklistTest(int argc, char *argv[]) {
                             if (node->encoding != QUICKLIST_NODE_ENCODING_LZF) {
                                 ERR("Incorrect non-compression: node %d is NOT "
                                     "compressed at depth %d ((%u, %u); total "
-                                    "nodes: %u; size: %u; attempted: %d)",
+                                    "nodes: %lu; size: %u; attempted: %d)",
                                     at, depth, low_raw, high_raw, ql->len,
                                     node->sz, node->attempted_compress);
                             }
diff --git a/src/rax.c b/src/rax.c
index 1ffb42287..0826b974a 100644
--- a/src/rax.c
+++ b/src/rax.c
@@ -61,7 +61,7 @@ void raxDebugShowNode(const char *msg, raxNode *n);
 #ifdef RAX_DEBUG_MSG
 #define debugf(...)                                                            \
     if (raxDebugMsg) {                                                         \
-        printf("%s:%s:%d:\t", __FILE__, __FUNCTION__, __LINE__);               \
+        printf("%s:%s:%d:\t", __FILE__, __func__, __LINE__);                   \
         printf(__VA_ARGS__);                                                   \
         fflush(stdout);                                                        \
     }
diff --git a/src/ziplist.c b/src/ziplist.c
index 96ba47e13..a4f38c5e8 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -1713,7 +1713,7 @@ int ziplistTest(int argc, char **argv) {
         if (p == NULL) {
             printf("No entry\n");
         } else {
-            printf("ERROR: Out of range index should return NULL, returned offset: %ld\n", p-zl);
+            printf("ERROR: Out of range index should return NULL, returned offset: %ld\n", (long)(p-zl));
             return 1;
         }
         printf("\n");
@@ -1763,7 +1763,7 @@ int ziplistTest(int argc, char **argv) {
         if (p == NULL) {
             printf("No entry\n");
         } else {
-            printf("ERROR: Out of range index should return NULL, returned offset: %ld\n", p-zl);
+            printf("ERROR: Out of range index should return NULL, returned offset: %ld\n", (long)(p-zl));
             return 1;
         }
         printf("\n");

From 430dadaf834e6de066006d5bf700d2daa70987ca Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sat, 9 Jan 2021 20:22:20 +0200
Subject: [PATCH 0959/1098] CLIENT PAUSE - don't drop together with other
 blocked clients (#8302)

When the server state changes and blocked clients are being dropped, the
paused clients should not be dropped, they're safe to keep since unlike
other blocked types, these commands are not half way though processing,
and the commands they sent may get rejected according to the new server
state.
---
 src/blocked.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/blocked.c b/src/blocked.c
index 250e4bd98..46935c79f 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -214,6 +214,13 @@ void disconnectAllBlockedClients(void) {
         client *c = listNodeValue(ln);
 
         if (c->flags & CLIENT_BLOCKED) {
+            /* PAUSED clients are an exception, when they'll be unblocked, the
+             * command processing will start from scratch, and the command will
+             * be either executed or rejected. (unlike LIST blocked clients for
+             * which the command is already in progress in a way. */
+            if (c->btype == BLOCKED_PAUSE)
+                continue;
+
             addReplyError(c,
                 "-UNBLOCKED force unblock from blocking operation, "
                 "instance state changed (master -> replica?)");

From fe2fdef7b0b65c86eb3cc2685cf0c4efe5007b1c Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Sun, 10 Jan 2021 02:24:58 +0800
Subject: [PATCH 0960/1098] Assert that clusterAddNode can't fail (#8296)

Assert that clusterAddNode can't fail
---
 src/cluster.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index c327334ee..78c36e8d1 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -47,7 +47,7 @@
 clusterNode *myself = NULL;
 
 clusterNode *createClusterNode(char *nodename, int flags);
-int clusterAddNode(clusterNode *node);
+void clusterAddNode(clusterNode *node);
 void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask);
 void clusterReadHandler(connection *conn);
 void clusterSendPing(clusterLink *link, int type);
@@ -961,12 +961,12 @@ void freeClusterNode(clusterNode *n) {
 }
 
 /* Add a node to the nodes hash table */
-int clusterAddNode(clusterNode *node) {
+void clusterAddNode(clusterNode *node) {
     int retval;
 
     retval = dictAdd(server.cluster->nodes,
             sdsnewlen(node->name,CLUSTER_NAMELEN), node);
-    return (retval == DICT_OK) ? C_OK : C_ERR;
+    serverAssert(retval == DICT_OK);
 }
 
 /* Remove a node from the cluster. The function performs the high level

From 542455ce2afbaffc065afbfdf6578399b8faecda Mon Sep 17 00:00:00 2001
From: Bob Li <342974971@qq.com>
Date: Tue, 12 Jan 2021 14:22:53 +0800
Subject: [PATCH 0961/1098] Fix overflow of rdbWriteRaw return value (#8306)

Saving string of more than 2GB to the RDB file, can result in corrupt RDB, or failure in rdbSave.
S
---
 src/rdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rdb.c b/src/rdb.c
index 25e900689..7deed2a2d 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -94,7 +94,7 @@ void rdbReportError(int corruption_error, int linenum, char *reason, ...) {
     exit(1);
 }
 
-static int rdbWriteRaw(rio *rdb, void *p, size_t len) {
+static ssize_t rdbWriteRaw(rio *rdb, void *p, size_t len) {
     if (rdb && rioWrite(rdb,p,len) == 0)
         return -1;
     return len;

From 264953871b97bcc3d663cd006578c151a13b48ca Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 12 Jan 2021 09:41:57 +0200
Subject: [PATCH 0962/1098] Fix cluster diskless load swapdb test (#8308)

The test was trying to wait for the replica to start loading the rdb
from the master before it kills the master, but it was actually waiting
for ROLE to be in "sync" mode, which corresponds to REPL_STATE_TRANSFER
that starts before the actual loading starts.
now instead it waits for the loading flag to be set.

Besides, the test was dependent on the previous configuration of the
servers, relying on the fact the replica is configured to persist
(either RDB of AOF), now it is set explicitly.
---
 tests/cluster/cluster.tcl                       |  5 +++++
 tests/cluster/tests/17-diskless-load-swapdb.tcl | 12 +++++++-----
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/tests/cluster/cluster.tcl b/tests/cluster/cluster.tcl
index e894823de..ffb268561 100644
--- a/tests/cluster/cluster.tcl
+++ b/tests/cluster/cluster.tcl
@@ -57,6 +57,11 @@ proc CI {n field} {
     get_info_field [R $n cluster info] $field
 }
 
+# Return the value of the specified INFO field.
+proc s {n field} {
+    get_info_field [R $n info] $field
+}
+
 # Assuming nodes are reest, this function performs slots allocation.
 # Only the first 'n' nodes are used.
 proc cluster_allocate_slots {n} {
diff --git a/tests/cluster/tests/17-diskless-load-swapdb.tcl b/tests/cluster/tests/17-diskless-load-swapdb.tcl
index a035be7be..612818cb7 100644
--- a/tests/cluster/tests/17-diskless-load-swapdb.tcl
+++ b/tests/cluster/tests/17-diskless-load-swapdb.tcl
@@ -22,6 +22,8 @@ test "Right to restore backups when fail to diskless load " {
 
     $replica READONLY
     $replica config set repl-diskless-load swapdb
+    $replica config set appendonly no
+    $replica config set save ""
     $replica config rewrite
     $master config set repl-backlog-size 1024
     $master config set repl-diskless-sync yes
@@ -38,7 +40,8 @@ test "Right to restore backups when fail to diskless load " {
     assert_equal {1} [$replica get $slot0_key]
     assert_equal $slot0_key [$replica CLUSTER GETKEYSINSLOT 0 1]
 
-    # Kill the replica
+    # Save an RDB and kill the replica
+    $replica save
     kill_instance redis $replica_id
 
     # Delete the key from master
@@ -60,13 +63,12 @@ test "Right to restore backups when fail to diskless load " {
     restart_instance redis $replica_id
     $replica READONLY
 
-    # Start full sync
+    # Start full sync, wait till after db is flushed (backed up)
     wait_for_condition 500 10 {
-        [string match "*sync*" [$replica role]]
+        [s $replica_id loading] eq 1
     } else {
         fail "Fail to full sync"
     }
-    after 100
 
     # Kill master, abort full sync
     kill_instance redis $master_id
@@ -74,4 +76,4 @@ test "Right to restore backups when fail to diskless load " {
     # Replica keys and keys to slots map still both are right
     assert_equal {1} [$replica get $slot0_key]
     assert_equal $slot0_key [$replica CLUSTER GETKEYSINSLOT 0 1]
-}
\ No newline at end of file
+}

From b24b49039348f7570a3e5e3673384080c093060d Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Mon, 11 Jan 2021 23:46:24 -0800
Subject: [PATCH 0963/1098] Fix issues in wait test (#8310)

This fixes three issues:
1.  Using debug SLEEP was impacting the subsequent test, and causing it to pass reliably even though it should have failed. There was exactly 5 seconds of artificial pause (after 1000, wait 3000, wait 1000) between the debug sleep 5 and when we needed to unblock the client in the subsequent test. Now the test properly makes sure the client is unblocked, and the subsequent test is fixed.
2. Minor, the client pause types were using & comparisons instead of ==, since it was previously a flag.
3. Test is faster now that some of the hand wavy time is removed.
---
 src/server.c        |  4 ++--
 tests/unit/wait.tcl | 11 +++++++----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/server.c b/src/server.c
index 2f9fa4f39..0551eb3e4 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4043,8 +4043,8 @@ int processCommand(client *c) {
     /* If the server is paused, block the client until
      * the pause has ended. Replicas are never paused. */
     if (!(c->flags & CLIENT_SLAVE) && 
-        ((server.client_pause_type & CLIENT_PAUSE_ALL) ||
-        (server.client_pause_type & CLIENT_PAUSE_WRITE && is_may_replicate_command)))
+        ((server.client_pause_type == CLIENT_PAUSE_ALL) ||
+        (server.client_pause_type == CLIENT_PAUSE_WRITE && is_may_replicate_command)))
     {
         c->bpop.timeout = 0;
         blockClient(c,BLOCKED_PAUSE);
diff --git a/tests/unit/wait.tcl b/tests/unit/wait.tcl
index b20900c23..0a4965c20 100644
--- a/tests/unit/wait.tcl
+++ b/tests/unit/wait.tcl
@@ -5,6 +5,7 @@ start_server {} {
     set slave [srv 0 client]
     set slave_host [srv 0 host]
     set slave_port [srv 0 port]
+    set slave_pid [srv 0 pid]
     set master [srv -1 client]
     set master_host [srv -1 host]
     set master_port [srv -1 port]
@@ -33,23 +34,25 @@ start_server {} {
     }
 
     test {WAIT should not acknowledge 1 additional copy if slave is blocked} {
-        set cmd [rediscli $slave_host $slave_port "debug sleep 5"]
-        exec {*}$cmd > /dev/null 2> /dev/null &
-        after 1000 ;# Give redis-cli the time to execute the command.
+        exec kill -SIGSTOP $slave_pid
         $master set foo 0
         $master incr foo
         $master incr foo
         $master incr foo
-        assert {[$master wait 1 3000] == 0}
+        assert {[$master wait 1 1000] == 0}
+        exec kill -SIGCONT $slave_pid
+        assert {[$master wait 1 1000] == 1}
     }
 
     test {WAIT implicitly blocks on client pause since ACKs aren't sent} {
+        exec kill -SIGSTOP $slave_pid
         $master multi
         $master incr foo
         $master client pause 10000 write
         $master exec
         assert {[$master wait 1 1000] == 0}
         $master client unpause
+        exec kill -SIGCONT $slave_pid
         assert {[$master wait 1 1000] == 1}
     }
 }}

From 4f8458d8d6c59a47c6ffbb1ac88c69854ccf1592 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 12 Jan 2021 10:03:45 +0200
Subject: [PATCH 0964/1098] fix race in cluster transactions test (#8312)

we didn't wait for the commands executed on the master to reach the replica.
---
 tests/cluster/tests/16-transactions-on-replica.tcl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/cluster/tests/16-transactions-on-replica.tcl b/tests/cluster/tests/16-transactions-on-replica.tcl
index 41083f421..baed15fbc 100644
--- a/tests/cluster/tests/16-transactions-on-replica.tcl
+++ b/tests/cluster/tests/16-transactions-on-replica.tcl
@@ -15,6 +15,7 @@ set replica [Rn 1]
 
 test "Cant read from replica without READONLY" {
     $primary SET a 1
+    wait_for_ofs_sync $primary $replica
     catch {$replica GET a} err
     assert {[string range $err 0 4] eq {MOVED}}
 }
@@ -28,6 +29,7 @@ test "Can preform HSET primary and HGET from replica" {
     $primary HSET h a 1
     $primary HSET h b 2
     $primary HSET h c 3
+    wait_for_ofs_sync $primary $replica
     assert {[$replica HGET h a] eq {1}}
     assert {[$replica HGET h b] eq {2}}
     assert {[$replica HGET h c] eq {3}}

From 049f2f080517c3f48d99726ac23f498c5b12851f Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 10 Jan 2021 21:50:12 +0200
Subject: [PATCH 0965/1098] Redis 6.2 RC2

---
 00-RELEASENOTES | 107 +++++++++++++++++++++++++++++++++++++++++++-----
 src/help.h      |  52 +++++++++++++++--------
 src/version.h   |   4 +-
 3 files changed, 135 insertions(+), 28 deletions(-)

diff --git a/00-RELEASENOTES b/00-RELEASENOTES
index 8fdfcf760..3434d4838 100644
--- a/00-RELEASENOTES
+++ b/00-RELEASENOTES
@@ -1,8 +1,84 @@
+================================================================================
+Redis 6.2 RC2   Released Tue Jan 12 16:17:20 IST 2021
+================================================================================
+
+Upgrade urgency LOW: This is the second Release Candidate of Redis 6.2.
+
+IMPORTANT: If you're running Redis on ARM64 or a big-endian system, upgrade may
+have significant implications. Please be sure to read the notes below.
+
+Here is a comprehensive list of changes in this release compared to 6.2 RC1,
+each one includes the PR number that added it, so you can get more details
+at https://github.com/redis/redis/pull/<number>
+
+New commands / args:
+* Add the REV, BYLEX and BYSCORE arguments to ZRANGE, and the ZRANGESTORE command (#7844)
+* Add the XAUTOCLAIM command (#7973)
+* Add the MINID trimming strategy and the LIMIT argument to XADD and XTRIM (#8169)
+* Add the ANY argument to GEOSEARCH and GEORADIUS (#8259)
+* Add the CH, NX, XX arguments to GEOADD (#8227)
+* Add the COUNT argument to LPOP and RPOP (#8179)
+* Add the WRITE argument to CLIENT PAUSE for pausing write commands exclusively (#8170)
+* Change the proto-ver argument of HELLO to optional (#7377)
+* Add the CLIENT TRACKINGINFO subcommand (#7309)
+
+Command behavior changes:
+* CLIENT TRACKING yields an error when given overlapping BCAST prefixes (#8176)
+* SWAPDB invalidates WATCHed keys (#8239)
+* SORT command behaves differently when used on a writable replica (#8283)
+
+Other behavior changes:
+* Avoid propagating MULTI/EXEC for read-only transactions (#8216)
+* Remove the read-only flag from TIME, ECHO, ROLE, LASTSAVE (#8216)
+* Fix the command flags of PFDEBUG (#8222)
+* Tracking clients will no longer receive unnecessary key invalidation messages after FLUSHDB (#8039)
+* Sentinel: Fix missing updates to the config file after SENTINEL SET command (#8229)
+
+Bug fixes with compatibility implications (bugs introduced in Redis 6.0):
+* Fix RDB CRC64 checksum on big-endian systems (#8270)
+  If you're using big-endian please consider the compatibility implications with
+  RESTORE, replication and persistence.
+* Fix wrong order of key/value in Lua's map response (#8266)
+  If your scripts use redis.setresp() or return a map (new in Redis 6.0), please
+  consider the implications.
+
+Bug fixes that are only applicable to previous releases of Redis 6.2:
+* Resolve rare assertions in active defragmentation while loading (#8284, #8281)
+
+Bug fixes:
+* Fix the selection of a random element from large hash tables (#8133)
+* Fix an issue where a forked process deletes the parent's pidfile (#8231)
+* Fix crashes when enabling io-threads-do-reads (#8230)
+* Fix a crash in redis-cli after executing cluster backup (#8267)
+* Fix redis-benchmark to use an IP address for the first cluster node (#8154)
+* Fix saving of strings larger than 2GB into RDB files (#8306)
+
+Additional improvements:
+* Improve replication handshake time (#8214)
+* Release client tracking table memory asynchronously in cases where the DB is also freed asynchronously (#8039)
+* Avoid wasteful transient memory allocation in certain cases (#8286, #5954)
+* Handle binary string values by the 'requirepass' and 'masterauth' configs (#8200)
+
+Platform and deployment-related changes:
+* Install redis-check-rdb and redis-check-aof as symlinks to redis-server (#5745)
+* Add a check for an ARM64 Linux kernel bug (#8224)
+  Due to the potential severity of this issue, Redis will refuse to run on
+  affected platforms by default.
+
+Info fields and introspection changes:
+* Add the errorstats section to the INFO command (#8217)
+* Add the failed_calls and rejected_calls fields INFO's commandstats section (#8217)
+* Report child copy-on-write metrics continuously (#8264)
+
+Module API changes:
+* Add the RedisModule_SendChildCOWInfo API (#8264)
+* Add the may-replicate command flag (#8170)
+
 ================================================================================
 Redis 6.2 RC1   Released Mon Dec 14 11:50:00 IST 2020
 ================================================================================
 
-Upgrade urgency LOW: This is the first RC of Redis 6.2.
+Upgrade urgency LOW: This is the first Release Candidate of Redis 6.2.
 
 Introduction to the Redis 6.2 release
 =====================================
@@ -29,7 +105,7 @@ New commands / args:
 * Add SMISMEMBER command that checks multiple members (#7615)
 * Add ZMSCORE command that returns an array of scores (#7593)
 * Add LMOVE and BLMOVE commands that pop and push arbitrarily (#6929)
-* Add RESET command that resets client connection state(#7982)
+* Add RESET command that resets client connection state (#7982)
 * Add COPY command that copies keys (#7953)
 * Add ZDIFF and ZDIFFSTORE commands (#7961)
 * Add ZINTER and ZUNION commands (#7794)
@@ -146,7 +222,7 @@ Platform / toolchain support related improvements:
 
 New configuration options:
 * Enable configuring OpenSSL using the standard openssl.cnf (#8143)
-* oom-score-adj-values can config can now take absolute values (besides relative ones) (#8046)
+* oom-score-adj-values config can now take absolute values (besides relative ones) (#8046)
 * TLS: Add different client cert support. (#8076)
 * Note that a few other changes listed above added their config options.
 
@@ -177,31 +253,42 @@ We'll follow up with more RC releases, until the code looks production ready
 and we don't get reports of serious issues for a while.
 
 A special thank you for the amount of work put into this release by:
+- Oran Agra
+- Yossi Gottlieb
+- Itamar Haber
+- Guy Benoish
+- Filipe Oliveira
+- Madelyn Olson
 - Wang Yuan
 - Felipe Machado
-- Filipe Oliveira
-- Tatsuya Arisawa
 - Yang Bodong
+- Tatsuya Arisawa
+- Jonah H. Harris
 - Jim Brunner
+- Yaacov Hazan
 - Wen Hui
-- Guy Benoish
 - Chen Yang
 - Nitai Caro
 - Meir Shpilraien
-- Madelyn Olson
 - maohuazhu
 - Valentino Geron
+- Qu Chen
+- sundb
+- George Prekas
+- Zhao Zhao
 - Tyson Andre
 - Michael Grunder
 - alexronke-channeladvisor
-- sundb
-- WuYunlong
+- Wu Yunlong
+- Wei Kukey
 - Yoav Steinberg
 - Uri Shachar
+- Greg Femec
 - Nykolas Laurentino de Lima
+- xhe
+- zhenwei pi
 - David CARLIER
 
-
 Migrating from 6.0 to 6.2
 =========================
 
diff --git a/src/help.h b/src/help.h
index d6754f570..edd15a3c9 100644
--- a/src/help.h
+++ b/src/help.h
@@ -194,7 +194,7 @@ struct commandHelp {
     8,
     "2.4.0" },
     { "CLIENT PAUSE",
-    "timeout",
+    "timeout [WRITE|ALL]",
     "Stop processing commands from clients for some time",
     8,
     "2.9.50" },
@@ -213,11 +213,21 @@ struct commandHelp {
     "Enable or disable server assisted client side caching support",
     8,
     "6.0.0" },
+    { "CLIENT TRACKINGINFO",
+    "-",
+    "Return information about server assisted client side caching for the current connection",
+    8,
+    "6.2.0" },
     { "CLIENT UNBLOCK",
     "client-id [TIMEOUT|ERROR]",
     "Unblock a client blocked in a blocking command from a different connection",
     8,
     "5.0.0" },
+    { "CLIENT UNPAUSE",
+    "-",
+    "Resume processing of clients that were paused",
+    8,
+    "6.2.0" },
     { "CLUSTER ADDSLOTS",
     "slot [slot ...]",
     "Assign new hash slots to receiving node",
@@ -459,7 +469,7 @@ struct commandHelp {
     9,
     "1.0.0" },
     { "GEOADD",
-    "key longitude latitude member [longitude latitude member ...]",
+    "key [NX|XX] [CH] longitude latitude member [longitude latitude member ...]",
     "Add one or more geospatial items in the geospatial index represented using a sorted set",
     13,
     "3.2.0" },
@@ -479,22 +489,22 @@ struct commandHelp {
     13,
     "3.2.0" },
     { "GEORADIUS",
-    "key longitude latitude radius m|km|ft|mi [WITHCOORD] [WITHDIST] [WITHHASH] [COUNT count] [ASC|DESC] [STORE key] [STOREDIST key]",
+    "key longitude latitude radius m|km|ft|mi [WITHCOORD] [WITHDIST] [WITHHASH] [COUNT count [ANY]] [ASC|DESC] [STORE key] [STOREDIST key]",
     "Query a sorted set representing a geospatial index to fetch members matching a given maximum distance from a point",
     13,
     "3.2.0" },
     { "GEORADIUSBYMEMBER",
-    "key member radius m|km|ft|mi [WITHCOORD] [WITHDIST] [WITHHASH] [COUNT count] [ASC|DESC] [STORE key] [STOREDIST key]",
+    "key member radius m|km|ft|mi [WITHCOORD] [WITHDIST] [WITHHASH] [COUNT count [ANY]] [ASC|DESC] [STORE key] [STOREDIST key]",
     "Query a sorted set representing a geospatial index to fetch members matching a given maximum distance from a member",
     13,
     "3.2.0" },
     { "GEOSEARCH",
-    "key [FROMMEMBER member] [FROMLONLAT longitude latitude] [BYRADIUS radius m|km|ft|mi] [BYBOX width height m|km|ft|mi] [ASC|DESC] [COUNT count] [WITHCOORD] [WITHDIST] [WITHHASH]",
+    "key [FROMMEMBER member] [FROMLONLAT longitude latitude] [BYRADIUS radius m|km|ft|mi] [BYBOX width height m|km|ft|mi] [ASC|DESC] [COUNT count [ANY]] [WITHCOORD] [WITHDIST] [WITHHASH]",
     "Query a sorted set representing a geospatial index to fetch members inside an area of a box or a circle.",
     13,
     "6.2" },
     { "GEOSEARCHSTORE",
-    "destination source [FROMMEMBER member] [FROMLONLAT longitude latitude] [BYRADIUS radius m|km|ft|mi] [BYBOX width height m|km|ft|mi] [ASC|DESC] [COUNT count] [WITHCOORD] [WITHDIST] [WITHHASH] [STOREDIST]",
+    "destination source [FROMMEMBER member] [FROMLONLAT longitude latitude] [BYRADIUS radius m|km|ft|mi] [BYBOX width height m|km|ft|mi] [ASC|DESC] [COUNT count [ANY]] [WITHCOORD] [WITHDIST] [WITHHASH] [STOREDIST]",
     "Query a sorted set representing a geospatial index to fetch members inside an area of a box or a circle, and store the result in another key.",
     13,
     "6.2" },
@@ -524,8 +534,8 @@ struct commandHelp {
     5,
     "2.0.0" },
     { "HELLO",
-    "protover [AUTH username password] [SETNAME clientname]",
-    "switch Redis protocol",
+    "[protover [AUTH username password] [SETNAME clientname]]",
+    "Handshake with Redis",
     8,
     "6.0.0" },
     { "HEXISTS",
@@ -684,8 +694,8 @@ struct commandHelp {
     9,
     "5.0.0" },
     { "LPOP",
-    "key",
-    "Remove and get the first element in a list",
+    "key [count]",
+    "Remove and get the first elements in a list",
     2,
     "1.0.0" },
     { "LPOS",
@@ -929,8 +939,8 @@ struct commandHelp {
     9,
     "2.8.12" },
     { "RPOP",
-    "key",
-    "Remove and get the last element in a list",
+    "key [count]",
+    "Remove and get the last elements in a list",
     2,
     "1.0.0" },
     { "RPOPLPUSH",
@@ -1189,10 +1199,15 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XADD",
-    "key [MAXLEN [=|~] length] [NOMKSTREAM] *|ID field value [field value ...]",
+    "key [NOMKSTREAM] [MAXLEN|MINID [=|~] threshold [LIMIT count]] *|ID field value [field value ...]",
     "Appends a new entry to a stream",
     14,
     "5.0.0" },
+    { "XAUTOCLAIM",
+    "key group consumer min-idle-time start [COUNT count] [justid]",
+    "Changes (or acquires) ownership of messages in a consumer group, as if the messages were delivered to the specified consumer.",
+    14,
+    "6.2.0" },
     { "XCLAIM",
     "key group consumer min-idle-time ID [ID ...] [IDLE ms] [TIME ms-unix-time] [RETRYCOUNT count] [force] [justid]",
     "Changes (or acquires) ownership of a message in a consumer group, as if the message was delivered to the specified consumer.",
@@ -1244,7 +1259,7 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XTRIM",
-    "key MAXLEN [=|~] length",
+    "key MAXLEN|MINID [=|~] threshold [LIMIT count]",
     "Trims the stream to (approximately if '~' is passed) a certain size",
     14,
     "5.0.0" },
@@ -1309,8 +1324,8 @@ struct commandHelp {
     4,
     "5.0.0" },
     { "ZRANGE",
-    "key start stop [WITHSCORES]",
-    "Return a range of members in a sorted set, by index",
+    "key min max [BYSCORE|BYLEX] [REV] [LIMIT offset count] [WITHSCORES]",
+    "Return a range of members in a sorted set",
     4,
     "1.2.0" },
     { "ZRANGEBYLEX",
@@ -1323,6 +1338,11 @@ struct commandHelp {
     "Return a range of members in a sorted set, by score",
     4,
     "1.0.5" },
+    { "ZRANGESTORE",
+    "dst src min max [BYSCORE|BYLEX] [REV] [LIMIT offset count]",
+    "Store a range of members from sorted set into another key",
+    4,
+    "6.2.0" },
     { "ZRANK",
     "key member",
     "Determine the index of a member in a sorted set",
diff --git a/src/version.h b/src/version.h
index 26505bfcb..d408dd3e8 100644
--- a/src/version.h
+++ b/src/version.h
@@ -1,2 +1,2 @@
-#define REDIS_VERSION "6.1.240"
-#define REDIS_VERSION_NUM 0x000601f0
+#define REDIS_VERSION "6.1.241"
+#define REDIS_VERSION_NUM 0x000601f1

From 593cde16bcac64fb52a36500377574e981c580f6 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Wed, 13 Jan 2021 17:38:02 +0800
Subject: [PATCH 0966/1098] Fix units in log message of copy-on-write (#8320)

---
 src/server.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index 0551eb3e4..90b9045ea 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5533,7 +5533,7 @@ void sendChildCOWInfo(int ptype, int on_exit, char *pname) {
     if (private_dirty) {
         serverLog(on_exit ? LL_NOTICE : LL_VERBOSE,
             "%s: %zu MB of memory used by copy-on-write",
-            pname, private_dirty);
+            pname, private_dirty/(1024*1024));
     }
 
     sendChildInfo(ptype, on_exit, private_dirty);

From 8a81ed1b5ab204fd33651bf448cd5e1230631fb7 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Wed, 13 Jan 2021 17:58:12 +0800
Subject: [PATCH 0967/1098] Fix use of  lookupKeyRead and lookupKeyWrite in
 zrangeGenericCommand, zunionInterDiffGenericCommand (#8316)

* Change zunionInterDiffGenericCommand to use lookupKeyRead if dstkey is null
* Change zrangeGenericCommand to use lookupKey Write if dstkey isn't null

ZRANGESTORE and UNION, ZINTER, ZDIFF are all new commands (6.2 RC1 and RC2).
In redis 6.0 the ZRANGE was using lookupKeyRead, and ZUNIONSTORE / ZINTERSTORE were using lookupKeyWrite.
So there bugs are introduced in 6.2 and will be resolved before it is released.

the implications of this bug are also not big:
The sole difference between LookupKeyRead and LookupKeyWrite is for command executed on a replica, which are not received from its master client. (for the master, and for the master client on the replica, these two functions behave the same)!
---
 src/t_zset.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/t_zset.c b/src/t_zset.c
index 3d63c41c6..eb4f274a3 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -2543,7 +2543,9 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in
     /* read keys to be used for input */
     src = zcalloc(sizeof(zsetopsrc) * setnum);
     for (i = 0, j = numkeysIndex+1; i < setnum; i++, j++) {
-        robj *obj = lookupKeyWrite(c->db,c->argv[j]);
+        robj *obj = dstkey ?
+            lookupKeyWrite(c->db,c->argv[j]) :
+            lookupKeyRead(c->db,c->argv[j]);
         if (obj != NULL) {
             if (obj->type != OBJ_ZSET && obj->type != OBJ_SET) {
                 zfree(src);
@@ -3605,11 +3607,16 @@ void zrangeGenericCommand(zrange_result_handler *handler, int argc_start, int st
     }
 
     /* Step 3: Lookup the key and get the range. */
-    if (((zobj = lookupKeyReadOrReply(c, key, shared.emptyarray)) == NULL)
-        || checkType(c, zobj, OBJ_ZSET)) {
+    zobj = handler->dstkey ?
+        lookupKeyWrite(c->db,key) :
+        lookupKeyRead(c->db,key);
+    if (zobj == NULL) {
+        addReply(c,shared.emptyarray);
         goto cleanup;
     }
 
+    if (checkType(c,zobj,OBJ_ZSET)) goto cleanup;
+
     /* Step 4: Pass this to the command-specific handler. */
     switch (rangetype) {
     case ZRANGE_AUTO:

From f5577fdbd8898112427e5807edf102f9fcd9da33 Mon Sep 17 00:00:00 2001
From: "houzj.fnst" <63178771+sherlockcpp@users.noreply.github.com>
Date: Thu, 14 Jan 2021 04:00:55 +0800
Subject: [PATCH 0968/1098] remove some unnecessary checks (#7431)

Remove several checks that always evaluate to true.
---
 src/db.c         | 2 +-
 src/expire.c     | 2 +-
 src/networking.c | 2 +-
 src/redis-cli.c  | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/db.c b/src/db.c
index 5d63566a7..13126988f 100644
--- a/src/db.c
+++ b/src/db.c
@@ -951,7 +951,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) {
         int filter = 0;
 
         /* Filter element if it does not match the pattern. */
-        if (!filter && use_pattern) {
+        if (use_pattern) {
             if (sdsEncodedObject(kobj)) {
                 if (!stringmatchlen(pat, patlen, kobj->ptr, sdslen(kobj->ptr), 0))
                     filter = 1;
diff --git a/src/expire.c b/src/expire.c
index 275a735a7..d2945524c 100644
--- a/src/expire.c
+++ b/src/expire.c
@@ -224,7 +224,7 @@ void activeExpireCycle(int type) {
             /* When there are less than 1% filled slots, sampling the key
              * space is expensive, so stop here waiting for better times...
              * The dictionary will be resized asap. */
-            if (num && slots > DICT_HT_INITIAL_SIZE &&
+            if (slots > DICT_HT_INITIAL_SIZE &&
                 (num*100/slots < 1)) break;
 
             /* The main collection cycle. Sample random keys among keys
diff --git a/src/networking.c b/src/networking.c
index e624dd8f9..f1c606236 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1707,7 +1707,7 @@ int processInlineBuffer(client *c) {
     }
 
     /* Handle the \r\n case. */
-    if (newline && newline != c->querybuf+c->qb_pos && *(newline-1) == '\r')
+    if (newline != c->querybuf+c->qb_pos && *(newline-1) == '\r')
         newline--, linefeed_chars++;
 
     /* Split the input buffer up to the \r\n */
diff --git a/src/redis-cli.c b/src/redis-cli.c
index 31d2360c9..2e8984c71 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -5301,7 +5301,7 @@ static clusterManagerNode *clusterNodeForResharding(char *id,
         clusterManagerLogErr(invalid_node_msg, id);
         *raise_err = 1;
         return NULL;
-    } else if (node != NULL && target != NULL) {
+    } else if (target != NULL) {
         if (!strcmp(node->name, target->name)) {
             clusterManagerLogErr( "*** It is not possible to use "
                                   "the target node as "

From 9cb9f98d2f0d9114ac4528b2f9434a2fd2edfd60 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuan21@baidu.com>
Date: Thu, 14 Jan 2021 04:36:03 +0800
Subject: [PATCH 0969/1098] Optimize performance of clusterGenNodesDescription
 for large clusters (#8182)

Optimize the performance of clusterGenNodesDescription by only checking slot ownership of each slot once, instead of checking each slot for each node.
---
 src/cluster.c                                 | 91 +++++++++++++++----
 src/cluster.h                                 |  1 +
 .../cluster/tests/18-cluster-nodes-slots.tcl  | 62 +++++++++++++
 3 files changed, 134 insertions(+), 20 deletions(-)
 create mode 100644 tests/cluster/tests/18-cluster-nodes-slots.tcl

diff --git a/src/cluster.c b/src/cluster.c
index 78c36e8d1..2a06ef6dc 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -779,6 +779,7 @@ clusterNode *createClusterNode(char *nodename, int flags) {
     node->configEpoch = 0;
     node->flags = flags;
     memset(node->slots,0,sizeof(node->slots));
+    node->slots_info = NULL;
     node->numslots = 0;
     node->numslaves = 0;
     node->slaves = NULL;
@@ -4144,8 +4145,8 @@ sds clusterGenNodeDescription(clusterNode *node) {
     sds ci;
 
     /* Node coordinates */
-    ci = sdscatprintf(sdsempty(),"%.40s %s:%d@%d ",
-        node->name,
+    ci = sdscatlen(sdsempty(),node->name,CLUSTER_NAMELEN);
+    ci = sdscatfmt(ci," %s:%i@%i ",
         node->ip,
         node->port,
         node->cport);
@@ -4154,40 +4155,46 @@ sds clusterGenNodeDescription(clusterNode *node) {
     ci = representClusterNodeFlags(ci, node->flags);
 
     /* Slave of... or just "-" */
+    ci = sdscatlen(ci," ",1);
     if (node->slaveof)
-        ci = sdscatprintf(ci," %.40s ",node->slaveof->name);
+        ci = sdscatlen(ci,node->slaveof->name,CLUSTER_NAMELEN);
     else
-        ci = sdscatlen(ci," - ",3);
+        ci = sdscatlen(ci,"-",1);
 
     unsigned long long nodeEpoch = node->configEpoch;
     if (nodeIsSlave(node) && node->slaveof) {
         nodeEpoch = node->slaveof->configEpoch;
     }
     /* Latency from the POV of this node, config epoch, link status */
-    ci = sdscatprintf(ci,"%lld %lld %llu %s",
+    ci = sdscatfmt(ci," %I %I %U %s",
         (long long) node->ping_sent,
         (long long) node->pong_received,
         nodeEpoch,
         (node->link || node->flags & CLUSTER_NODE_MYSELF) ?
                     "connected" : "disconnected");
 
-    /* Slots served by this instance */
-    start = -1;
-    for (j = 0; j < CLUSTER_SLOTS; j++) {
-        int bit;
+    /* Slots served by this instance. If we already have slots info,
+     * append it diretly, otherwise, generate slots only if it has. */
+    if (node->slots_info) {
+        ci = sdscatsds(ci, node->slots_info);
+    } else if (node->numslots > 0) {
+        start = -1;
+        for (j = 0; j < CLUSTER_SLOTS; j++) {
+            int bit;
 
-        if ((bit = clusterNodeGetSlotBit(node,j)) != 0) {
-            if (start == -1) start = j;
-        }
-        if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) {
-            if (bit && j == CLUSTER_SLOTS-1) j++;
-
-            if (start == j-1) {
-                ci = sdscatprintf(ci," %d",start);
-            } else {
-                ci = sdscatprintf(ci," %d-%d",start,j-1);
+            if ((bit = clusterNodeGetSlotBit(node,j)) != 0) {
+                if (start == -1) start = j;
+            }
+            if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) {
+                if (bit && j == CLUSTER_SLOTS-1) j++;
+
+                if (start == j-1) {
+                    ci = sdscatfmt(ci," %i",start);
+                } else {
+                    ci = sdscatfmt(ci," %i-%i",start,j-1);
+                }
+                start = -1;
             }
-            start = -1;
         }
     }
 
@@ -4208,6 +4215,41 @@ sds clusterGenNodeDescription(clusterNode *node) {
     return ci;
 }
 
+/* Generate the slot topology for all nodes and store the string representation
+ * in the slots_info struct on the node. This is used to improve the efficiency
+ * of clusterGenNodesDescription() because it removes looping of the slot space
+ * for generating the slot info for each node individually. */
+void clusterGenNodesSlotsInfo(int filter) {
+    clusterNode *n = NULL;
+    int start = -1;
+
+    for (int i = 0; i <= CLUSTER_SLOTS; i++) {
+        /* Find start node and slot id. */
+        if (n == NULL) {
+            if (i == CLUSTER_SLOTS) break;
+            n = server.cluster->slots[i];
+            start = i;
+            continue;
+        }
+
+        /* Generate slots info when occur different node with start
+         * or end of slot. */
+        if (i == CLUSTER_SLOTS || n != server.cluster->slots[i]) {
+            if (!(n->flags & filter)) {
+                if (n->slots_info == NULL) n->slots_info = sdsempty();
+                if (start == i-1) {
+                    n->slots_info = sdscatfmt(n->slots_info," %i",start);
+                } else {
+                    n->slots_info = sdscatfmt(n->slots_info," %i-%i",start,i-1);
+                }
+            }
+            if (i == CLUSTER_SLOTS) break;
+            n = server.cluster->slots[i];
+            start = i;
+        }
+    }
+}
+
 /* Generate a csv-alike representation of the nodes we are aware of,
  * including the "myself" node, and return an SDS string containing the
  * representation (it is up to the caller to free it).
@@ -4225,6 +4267,9 @@ sds clusterGenNodesDescription(int filter) {
     dictIterator *di;
     dictEntry *de;
 
+    /* Generate all nodes slots info firstly. */
+    clusterGenNodesSlotsInfo(filter);
+
     di = dictGetSafeIterator(server.cluster->nodes);
     while((de = dictNext(di)) != NULL) {
         clusterNode *node = dictGetVal(de);
@@ -4234,6 +4279,12 @@ sds clusterGenNodesDescription(int filter) {
         ci = sdscatsds(ci,ni);
         sdsfree(ni);
         ci = sdscatlen(ci,"\n",1);
+
+        /* Release slots info. */
+        if (node->slots_info) {
+            sdsfree(node->slots_info);
+            node->slots_info = NULL;
+        }
     }
     dictReleaseIterator(di);
     return ci;
diff --git a/src/cluster.h b/src/cluster.h
index d58f350ce..716c0d49c 100644
--- a/src/cluster.h
+++ b/src/cluster.h
@@ -118,6 +118,7 @@ typedef struct clusterNode {
     int flags;      /* CLUSTER_NODE_... */
     uint64_t configEpoch; /* Last configEpoch observed for this node */
     unsigned char slots[CLUSTER_SLOTS/8]; /* slots handled by this node */
+    sds slots_info; /* Slots info represented by string. */
     int numslots;   /* Number of slots handled by this node */
     int numslaves;  /* Number of slave nodes, if this is a master */
     struct clusterNode **slaves; /* pointers to slave nodes */
diff --git a/tests/cluster/tests/18-cluster-nodes-slots.tcl b/tests/cluster/tests/18-cluster-nodes-slots.tcl
new file mode 100644
index 000000000..ca0b3ce0d
--- /dev/null
+++ b/tests/cluster/tests/18-cluster-nodes-slots.tcl
@@ -0,0 +1,62 @@
+# Optimize CLUSTER NODES command by generating all nodes slot topology firstly
+
+source "../tests/includes/init-tests.tcl"
+
+proc cluster_allocate_with_continuous_slots {n} {
+    set slot 16383
+    set avg [expr ($slot+1) / $n]
+    while {$slot >= 0} {
+        set node [expr $slot/$avg >= $n ? $n-1 : $slot/$avg]
+        lappend slots_$node $slot
+        incr slot -1
+    }
+    for {set j 0} {$j < $n} {incr j} {
+        R $j cluster addslots {*}[set slots_${j}]
+    }
+}
+
+proc cluster_create_with_continuous_slots {masters slaves} {
+    cluster_allocate_with_continuous_slots $masters
+    if {$slaves} {
+        cluster_allocate_slaves $masters $slaves
+    }
+    assert_cluster_state ok
+}
+
+test "Create a 2 nodes cluster" {
+    cluster_create_with_continuous_slots 2 2
+}
+
+test "Cluster should start ok" {
+    assert_cluster_state ok
+}
+
+set master1 [Rn 0]
+set master2 [Rn 1]
+
+test "Continuous slots distribution" {
+    assert_match "* 0-8191*" [$master1 CLUSTER NODES]
+    assert_match "* 8192-16383*" [$master2 CLUSTER NODES]
+
+    $master1 CLUSTER DELSLOTS 4096
+    assert_match "* 0-4095 4097-8191*" [$master1 CLUSTER NODES]
+
+    $master2 CLUSTER DELSLOTS 12288
+    assert_match "* 8192-12287 12289-16383*" [$master2 CLUSTER NODES]
+}
+
+test "Discontinuous slots distribution" {
+    # Remove middle slots
+    $master1 CLUSTER DELSLOTS 4092 4094
+    assert_match "* 0-4091 4093 4095 4097-8191*" [$master1 CLUSTER NODES]
+    $master2 CLUSTER DELSLOTS 12284 12286
+    assert_match "* 8192-12283 12285 12287 12289-16383*" [$master2 CLUSTER NODES]
+
+    # Remove head slots
+    $master1 CLUSTER DELSLOTS 0 2
+    assert_match "* 1 3-4091 4093 4095 4097-8191*" [$master1 CLUSTER NODES]
+
+    # Remove tail slots
+    $master2 CLUSTER DELSLOTS 16380 16382 16383
+    assert_match "* 8192-12283 12285 12287 12289-16379 16381*" [$master2 CLUSTER NODES]
+}

From 108afbc0bbde9153e4cb08bf50a724c4feb9626e Mon Sep 17 00:00:00 2001
From: charsyam <charsyam@naver.com>
Date: Thu, 14 Jan 2021 09:44:04 +0900
Subject: [PATCH 0970/1098] Replace many calling zrealloc to one zmalloc in
 sentinelResetMasterAndChangeAddress (#8319)

Replace many calling zrealloc to one zmalloc in sentinelResetMasterAndChangeAddress
---
 src/sentinel.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/sentinel.c b/src/sentinel.c
index 02260feb7..a374977de 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -1523,14 +1523,17 @@ int sentinelResetMasterAndChangeAddress(sentinelRedisInstance *master, char *ip,
     newaddr = createSentinelAddr(ip,port);
     if (newaddr == NULL) return C_ERR;
 
-    /* Make a list of slaves to add back after the reset.
-     * Don't include the one having the address we are switching to. */
+    /* There can be only 0 or 1 slave that has the newaddr.
+     * and It can add old master 1 more slave. 
+     * so It allocates dictSize(master->slaves) + 1          */
+    slaves = zmalloc(sizeof(sentinelAddr*)*(dictSize(master->slaves) + 1));
+    
+    /* Don't include the one having the address we are switching to. */
     di = dictGetIterator(master->slaves);
     while((de = dictNext(di)) != NULL) {
         sentinelRedisInstance *slave = dictGetVal(de);
 
         if (sentinelAddrIsEqual(slave->addr,newaddr)) continue;
-        slaves = zrealloc(slaves,sizeof(sentinelAddr*)*(numslaves+1));
         slaves[numslaves++] = createSentinelAddr(slave->addr->ip,
                                                  slave->addr->port);
     }
@@ -1540,7 +1543,6 @@ int sentinelResetMasterAndChangeAddress(sentinelRedisInstance *master, char *ip,
      * as a slave as well, so that we'll be able to sense / reconfigure
      * the old master. */
     if (!sentinelAddrIsEqual(newaddr,master->addr)) {
-        slaves = zrealloc(slaves,sizeof(sentinelAddr*)*(numslaves+1));
         slaves[numslaves++] = createSentinelAddr(master->addr->ip,
                                                  master->addr->port);
     }

From ebf20b83b2cfcaf326f037a8274848662a3a9603 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= <viktor.soderqvist@est.tech>
Date: Wed, 13 Jan 2021 15:14:51 +0100
Subject: [PATCH 0971/1098] Modules API reference formatting fixes

Fixes markdown formatting errors and some functions not showing
up in the generated documentation at all.

Ruby script (gendoc.rb) fixes:

* Modified automatic instertion of backquotes:
  * Don't add backquotes around names which are already preceded by a
    backquote. Fixes for example \`RedisModule_Reply\*\` which turning
    into \`\`RedisModule_Reply\`\*\` messes up the formatting.
  * Add backquotes around types such as RedisModuleString (in addition
    to function names `RedisModule_[A-z()]*` and macro names
    `REDISMODULE_[A-z]*`).
  * Require 4 spaces indentation for disabling automatic backquotes, i.e.
    code blocks. Fixes continuations of list items (indented 2 spaces).
* More permissive extraction of doc comments:
  * Allow doc comments starting with `/**`.
  * Make space before `*` on each line optional.
  * Make space after `/*` and `/**` optional (needed when appearing on
    its own line).

Markdown fixes in module.c:

* Fix code blocks not indented enough (4 spaces needed).
* Add black line before code blocks and lists where missing (needed).
* Enclose special markdown characters `_*^<>` in backticks to prevent them
  from messing up formatting.
* Lists with `1)` changed to `1.` for proper markdown lists.
* Remove excessive indentation which causes text to be unintentionally
  rendered as code blocks.
* Other minor formatting fixes.

Other fixes in module.c:

* Remove blank lines between doc comment and function definition. A blank
  line here makes the Ruby script exclude the function in docs.
---
 src/module.c          | 501 ++++++++++++++++++++++--------------------
 src/modules/gendoc.rb |  15 +-
 2 files changed, 266 insertions(+), 250 deletions(-)

diff --git a/src/module.c b/src/module.c
index bf186f8b7..c8dd237ad 100644
--- a/src/module.c
+++ b/src/module.c
@@ -486,9 +486,9 @@ void *RM_PoolAlloc(RedisModuleCtx *ctx, size_t bytes) {
  * the value of the specified type. The function fails and returns
  * REDISMODULE_ERR if:
  *
- * 1) The key is not open for writing.
- * 2) The key is not empty.
- * 3) The specified type is unknown.
+ * 1. The key is not open for writing.
+ * 2. The key is not empty.
+ * 3. The specified type is unknown.
  */
 int moduleCreateEmptyKey(RedisModuleKey *key, int type) {
     robj *obj;
@@ -933,9 +933,9 @@ int RM_SignalModifiedKey(RedisModuleCtx *ctx, RedisModuleString *keyname) {
  * keys, call replies and Redis string objects once the command returns. In most
  * cases this eliminates the need of calling the following functions:
  *
- * 1) RedisModule_CloseKey()
- * 2) RedisModule_FreeCallReply()
- * 3) RedisModule_FreeString()
+ * 1. RedisModule_CloseKey()
+ * 2. RedisModule_FreeCallReply()
+ * 3. RedisModule_FreeString()
  *
  * These functions can still be used with automatic memory management enabled,
  * to optimize loops that make numerous allocations for example. */
@@ -1139,9 +1139,9 @@ void RM_FreeString(RedisModuleCtx *ctx, RedisModuleString *str) {
  * Normally you want to call this function when, at the same time
  * the following conditions are true:
  *
- * 1) You have automatic memory management enabled.
- * 2) You want to create string objects.
- * 3) Those string objects you create need to live *after* the callback
+ * 1. You have automatic memory management enabled.
+ * 2. You want to create string objects.
+ * 3. Those string objects you create need to live *after* the callback
  *    function(for example a command implementation) creating them returns.
  *
  * Usually you want this in order to store the created string object
@@ -1188,7 +1188,7 @@ void RM_RetainString(RedisModuleCtx *ctx, RedisModuleString *str) {
 * returned RedisModuleString.
 * 
 * It is possible to call this function with a NULL context.
- */
+*/
 RedisModuleString* RM_HoldString(RedisModuleCtx *ctx, RedisModuleString *str) {
     if (str->refcount == OBJ_STATIC_REFCOUNT) {
         return RM_CreateStringFromString(ctx, str);
@@ -1742,7 +1742,7 @@ int RM_ReplicateVerbatim(RedisModuleCtx *ctx) {
  * 2. The ID increases monotonically. Clients connecting to the server later
  *    are guaranteed to get IDs greater than any past ID previously seen.
  *
- * Valid IDs are from 1 to 2^64-1. If 0 is returned it means there is no way
+ * Valid IDs are from 1 to 2^64 - 1. If 0 is returned it means there is no way
  * to fetch the ID in the context the function was currently called.
  *
  * After obtaining the ID, it is possible to check if the command execution
@@ -2376,9 +2376,10 @@ int RM_ListPush(RedisModuleKey *key, int where, RedisModuleString *ele) {
  * that the user should be free with RM_FreeString() or by enabling
  * automatic memory. 'where' specifies if the element should be popped from
  * head or tail. The command returns NULL if:
- * 1) The list is empty.
- * 2) The key was not open for writing.
- * 3) The key is not a list. */
+ *
+ * 1. The list is empty.
+ * 2. The key was not open for writing.
+ * 3. The key is not a list. */
 RedisModuleString *RM_ListPop(RedisModuleKey *key, int where) {
     if (!(key->mode & REDISMODULE_WRITE) ||
         key->value == NULL ||
@@ -2610,8 +2611,8 @@ int zsetInitScoreRange(RedisModuleKey *key, double min, double max, int minex, i
  * The range is specified according to the two double values 'min' and 'max'.
  * Both can be infinite using the following two macros:
  *
- * REDISMODULE_POSITIVE_INFINITE for positive infinite value
- * REDISMODULE_NEGATIVE_INFINITE for negative infinite value
+ * * REDISMODULE_POSITIVE_INFINITE for positive infinite value
+ * * REDISMODULE_NEGATIVE_INFINITE for negative infinite value
  *
  * 'minex' and 'maxex' parameters, if true, respectively setup a range
  * where the min and max value are exclusive (not included) instead of
@@ -2970,7 +2971,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) {
  *
  *      RedisModuleString *first, *second;
  *      RedisModule_HashGet(mykey,REDISMODULE_HASH_NONE,argv[1],&first,
- *                      argv[2],&second,NULL);
+ *                          argv[2],&second,NULL);
  *
  * As with RedisModule_HashSet() the behavior of the command can be specified
  * passing flags different than REDISMODULE_HASH_NONE:
@@ -3348,29 +3349,31 @@ fmterr:
  * * **cmdname**: The Redis command to call.
  * * **fmt**: A format specifier string for the command's arguments. Each
  *   of the arguments should be specified by a valid type specification:
- *   b    The argument is a buffer and is immediately followed by another
+ *
+ *   * b: The argument is a buffer and is immediately followed by another
  *        argument that is the buffer's length.
- *   c    The argument is a pointer to a plain C string (null-terminated).
- *   l    The argument is long long integer.
- *   s    The argument is a RedisModuleString.
- *   v    The argument(s) is a vector of RedisModuleString.
+ *   * c: The argument is a pointer to a plain C string (null-terminated).
+ *   * l: The argument is long long integer.
+ *   * s: The argument is a RedisModuleString.
+ *   * v: The argument(s) is a vector of RedisModuleString.
  *
  *   The format specifier can also include modifiers:
- *   !    Sends the Redis command and its arguments to replicas and AOF.
- *   A    Suppress AOF propagation, send only to replicas (requires `!`).
- *   R    Suppress replicas propagation, send only to AOF (requires `!`).
+ *
+ *   * !: Sends the Redis command and its arguments to replicas and AOF.
+ *   * A: Suppress AOF propagation, send only to replicas (requires `!`).
+ *   * R: Suppress replicas propagation, send only to AOF (requires `!`).
  * * **...**: The actual arguments to the Redis command.
  *
  * On success a RedisModuleCallReply object is returned, otherwise
  * NULL is returned and errno is set to the following values:
  *
- * EBADF: wrong format specifier.
- * EINVAL: wrong command arity.
- * ENOENT: command does not exist.
- * EPERM:  operation in Cluster instance with key in non local slot.
- * EROFS:  operation in Cluster instance when a write command is sent
- *         in a readonly state.
- * ENETDOWN: operation in Cluster instance when cluster is down.
+ * * EBADF: wrong format specifier.
+ * * EINVAL: wrong command arity.
+ * * ENOENT: command does not exist.
+ * * EPERM: operation in Cluster instance with key in non local slot.
+ * * EROFS: operation in Cluster instance when a write command is sent
+ *          in a readonly state.
+ * * ENETDOWN: operation in Cluster instance when cluster is down.
  *
  * Example code fragment:
  * 
@@ -3686,7 +3689,7 @@ robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, robj *value) {
  *   that should be populated with the methods callbacks and structure
  *   version, like in the following example:
  *
- *      RedisModuleTypeMethods tm = {
+ *       RedisModuleTypeMethods tm = {
  *          .version = REDISMODULE_TYPE_METHOD_VERSION,
  *          .rdb_load = myType_RDBLoadCallBack,
  *          .rdb_save = myType_RDBSaveCallBack,
@@ -3702,7 +3705,7 @@ robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, robj *value) {
  *          .unlink = myType_UnlinkCallBack,
  *          .copy = myType_CopyCallback,
  *          .defrag = myType_DefragCallback
- *      }
+ *       }
  *
  * * **rdb_load**: A callback function pointer that loads data from RDB files.
  * * **rdb_save**: A callback function pointer that saves data to RDB files.
@@ -5103,9 +5106,9 @@ void moduleReleaseGIL(void) {
  *
  * The subscriber signature is:
  *
- *   int (*RedisModuleNotificationFunc) (RedisModuleCtx *ctx, int type,
- *                                       const char *event,
- *                                       RedisModuleString *key);
+ *     int (*RedisModuleNotificationFunc) (RedisModuleCtx *ctx, int type,
+ *                                         const char *event,
+ *                                         RedisModuleString *key);
  *
  * `type` is the event type bit, that must match the mask given at registration
  * time. The event string is the actual command being executed, and key is the
@@ -5369,28 +5372,27 @@ size_t RM_GetClusterSize(void) {
     return dictSize(server.cluster->nodes);
 }
 
+clusterNode *clusterLookupNode(const char *name); /* We need access to internals */
+
 /* Populate the specified info for the node having as ID the specified 'id',
  * then returns REDISMODULE_OK. Otherwise if the node ID does not exist from
  * the POV of this local node, REDISMODULE_ERR is returned.
  *
- * The arguments ip, master_id, port and flags can be NULL in case we don't
- * need to populate back certain info. If an ip and master_id (only populated
+ * The arguments `ip`, `master_id`, `port` and `flags` can be NULL in case we don't
+ * need to populate back certain info. If an `ip` and `master_id` (only populated
  * if the instance is a slave) are specified, they point to buffers holding
- * at least REDISMODULE_NODE_ID_LEN bytes. The strings written back as ip
- * and master_id are not null terminated.
+ * at least REDISMODULE_NODE_ID_LEN bytes. The strings written back as `ip`
+ * and `master_id` are not null terminated.
  *
  * The list of flags reported is the following:
  *
- * * REDISMODULE_NODE_MYSELF        This node
- * * REDISMODULE_NODE_MASTER        The node is a master
- * * REDISMODULE_NODE_SLAVE         The node is a replica
- * * REDISMODULE_NODE_PFAIL         We see the node as failing
- * * REDISMODULE_NODE_FAIL          The cluster agrees the node is failing
- * * REDISMODULE_NODE_NOFAILOVER    The slave is configured to never failover
+ * * REDISMODULE_NODE_MYSELF:       This node
+ * * REDISMODULE_NODE_MASTER:       The node is a master
+ * * REDISMODULE_NODE_SLAVE:        The node is a replica
+ * * REDISMODULE_NODE_PFAIL:        We see the node as failing
+ * * REDISMODULE_NODE_FAIL:         The cluster agrees the node is failing
+ * * REDISMODULE_NODE_NOFAILOVER:   The slave is configured to never failover
  */
-
-clusterNode *clusterLookupNode(const char *name); /* We need access to internals */
-
 int RM_GetClusterNodeInfo(RedisModuleCtx *ctx, const char *id, char *ip, char *master_id, int *port, int *flags) {
     UNUSED(ctx);
 
@@ -5434,18 +5436,18 @@ int RM_GetClusterNodeInfo(RedisModuleCtx *ctx, const char *id, char *ip, char *m
  * a different distributed system, but still want to use the Redis Cluster
  * message bus. Flags that can be set:
  *
- *  CLUSTER_MODULE_FLAG_NO_FAILOVER
- *  CLUSTER_MODULE_FLAG_NO_REDIRECTION
+ * * CLUSTER_MODULE_FLAG_NO_FAILOVER
+ * * CLUSTER_MODULE_FLAG_NO_REDIRECTION
  *
  * With the following effects:
  *
- *  NO_FAILOVER: prevent Redis Cluster slaves to failover a failing master.
- *               Also disables the replica migration feature.
+ * * NO_FAILOVER: prevent Redis Cluster slaves to failover a failing master.
+ *                Also disables the replica migration feature.
  *
- *  NO_REDIRECTION: Every node will accept any key, without trying to perform
- *                  partitioning according to the user Redis Cluster algorithm.
- *                  Slots informations will still be propagated across the
- *                  cluster, but without effects. */
+ * * NO_REDIRECTION: Every node will accept any key, without trying to perform
+ *                   partitioning according to the user Redis Cluster algorithm.
+ *                   Slots informations will still be propagated across the
+ *                   cluster, but without effects. */
 void RM_SetClusterFlags(RedisModuleCtx *ctx, uint64_t flags) {
     UNUSED(ctx);
     if (flags & REDISMODULE_CLUSTER_FLAG_NO_FAILOVER)
@@ -5964,15 +5966,15 @@ int RM_DictDel(RedisModuleDict *d, RedisModuleString *key, void *oldval) {
  * comparison operator to use in order to seek the first element. The
  * operators available are:
  *
- * "^"   -- Seek the first (lexicographically smaller) key.
- * "$"   -- Seek the last  (lexicographically biffer) key.
- * ">"   -- Seek the first element greater than the specified key.
- * ">="  -- Seek the first element greater or equal than the specified key.
- * "<"   -- Seek the first element smaller than the specified key.
- * "<="  -- Seek the first element smaller or equal than the specified key.
- * "=="  -- Seek the first element matching exactly the specified key.
+ * * `^`   -- Seek the first (lexicographically smaller) key.
+ * * `$`   -- Seek the last  (lexicographically biffer) key.
+ * * `>`   -- Seek the first element greater than the specified key.
+ * * `>=`  -- Seek the first element greater or equal than the specified key.
+ * * `<`   -- Seek the first element smaller than the specified key.
+ * * `<=`  -- Seek the first element smaller or equal than the specified key.
+ * * `==`  -- Seek the first element matching exactly the specified key.
  *
- * Note that for "^" and "$" the passed key is not used, and the user may
+ * Note that for `^` and `$` the passed key is not used, and the user may
  * just pass NULL with a length of 0.
  *
  * If the element to start the iteration cannot be seeked based on the
@@ -6017,11 +6019,11 @@ int RM_DictIteratorReseek(RedisModuleDictIter *di, const char *op, RedisModuleSt
     return RM_DictIteratorReseekC(di,op,key->ptr,sdslen(key->ptr));
 }
 
-/* Return the current item of the dictionary iterator 'di' and steps to the
+/* Return the current item of the dictionary iterator `di` and steps to the
  * next element. If the iterator already yield the last element and there
  * are no other elements to return, NULL is returned, otherwise a pointer
- * to a string representing the key is provided, and the '*keylen' length
- * is set by reference (if keylen is not NULL). The '*dataptr', if not NULL
+ * to a string representing the key is provided, and the `*keylen` length
+ * is set by reference (if keylen is not NULL). The `*dataptr`, if not NULL
  * is set to the value of the pointer stored at the returned key as auxiliary
  * data (as set by the RedisModule_DictSet API).
  *
@@ -6035,7 +6037,7 @@ int RM_DictIteratorReseek(RedisModuleDictIter *di, const char *op, RedisModuleSt
  *      }
  *
  * The returned pointer is of type void because sometimes it makes sense
- * to cast it to a char* sometimes to an unsigned char* depending on the
+ * to cast it to a `char*` sometimes to an unsigned `char*` depending on the
  * fact it contains or not binary data, so this API ends being more
  * comfortable to use.
  *
@@ -6119,8 +6121,8 @@ int RM_DictCompare(RedisModuleDictIter *di, const char *op, RedisModuleString *k
 int RM_InfoEndDictField(RedisModuleInfoCtx *ctx);
 
 /* Used to start a new section, before adding any fields. the section name will
- * be prefixed by "<modulename>_" and must only include A-Z,a-z,0-9.
- * NULL or empty string indicates the default section (only <modulename>) is used.
+ * be prefixed by `<modulename>_` and must only include A-Z,a-z,0-9.
+ * NULL or empty string indicates the default section (only `<modulename>`) is used.
  * When return value is REDISMODULE_ERR, the section should and will be skipped. */
 int RM_InfoAddSection(RedisModuleInfoCtx *ctx, char *name) {
     sds full_name = sdsdup(ctx->module->name);
@@ -6180,8 +6182,8 @@ int RM_InfoEndDictField(RedisModuleInfoCtx *ctx) {
 }
 
 /* Used by RedisModuleInfoFunc to add info fields.
- * Each field will be automatically prefixed by "<modulename>_".
- * Field names or values must not include \r\n of ":" */
+ * Each field will be automatically prefixed by `<modulename>_`.
+ * Field names or values must not include `\r\n` or `:`. */
 int RM_InfoAddFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value) {
     if (!ctx->in_section)
         return REDISMODULE_ERR;
@@ -6200,6 +6202,7 @@ int RM_InfoAddFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleStrin
     return REDISMODULE_OK;
 }
 
+/* See RedisModule_InfoAddFieldString(). */
 int RM_InfoAddFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) {
     if (!ctx->in_section)
         return REDISMODULE_ERR;
@@ -6218,6 +6221,7 @@ int RM_InfoAddFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) {
     return REDISMODULE_OK;
 }
 
+/* See RedisModule_InfoAddFieldString(). */
 int RM_InfoAddFieldDouble(RedisModuleInfoCtx *ctx, char *field, double value) {
     if (!ctx->in_section)
         return REDISMODULE_ERR;
@@ -6236,6 +6240,7 @@ int RM_InfoAddFieldDouble(RedisModuleInfoCtx *ctx, char *field, double value) {
     return REDISMODULE_OK;
 }
 
+/* See RedisModule_InfoAddFieldString(). */
 int RM_InfoAddFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long value) {
     if (!ctx->in_section)
         return REDISMODULE_ERR;
@@ -6254,6 +6259,7 @@ int RM_InfoAddFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long valu
     return REDISMODULE_OK;
 }
 
+/* See RedisModule_InfoAddFieldString(). */
 int RM_InfoAddFieldULongLong(RedisModuleInfoCtx *ctx, char *field, unsigned long long value) {
     if (!ctx->in_section)
         return REDISMODULE_ERR;
@@ -6272,6 +6278,8 @@ int RM_InfoAddFieldULongLong(RedisModuleInfoCtx *ctx, char *field, unsigned long
     return REDISMODULE_OK;
 }
 
+/* Registers callback for the INFO command. The callback should add INFO fields
+ * by calling the `RedisModule_InfoAddField*()` functions. */
 int RM_RegisterInfoFunc(RedisModuleCtx *ctx, RedisModuleInfoFunc cb) {
     ctx->module->info_cb = cb;
     return REDISMODULE_OK;
@@ -6711,7 +6719,6 @@ const RedisModuleString *RM_CommandFilterArgGet(RedisModuleCommandFilterCtx *fct
  * after the filter context is destroyed, so it must not be auto-memory
  * allocated, freed or used elsewhere.
  */
-
 int RM_CommandFilterArgInsert(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg)
 {
     int i;
@@ -6733,7 +6740,6 @@ int RM_CommandFilterArgInsert(RedisModuleCommandFilterCtx *fctx, int pos, RedisM
  * filter context is destroyed, so it must not be auto-memory allocated, freed
  * or used elsewhere.
  */
-
 int RM_CommandFilterArgReplace(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg)
 {
     if (pos < 0 || pos >= fctx->argc) return REDISMODULE_ERR;
@@ -6774,10 +6780,10 @@ size_t RM_MallocSize(void* ptr){
 /* Return the a number between 0 to 1 indicating the amount of memory
  * currently used, relative to the Redis "maxmemory" configuration.
  *
- * 0 - No memory limit configured.
- * Between 0 and 1 - The percentage of the memory used normalized in 0-1 range.
- * Exactly 1 - Memory limit reached.
- * Greater 1 - More memory used than the configured limit.
+ * * 0 - No memory limit configured.
+ * * Between 0 and 1 - The percentage of the memory used normalized in 0-1 range.
+ * * Exactly 1 - Memory limit reached.
+ * * Greater 1 - More memory used than the configured limit.
  */
 float RM_GetUsedMemoryRatio(){
     float level;
@@ -6840,21 +6846,22 @@ void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) {
  * the selected db.
  *
  * Callback for scan implementation.
- * void scan_callback(RedisModuleCtx *ctx, RedisModuleString *keyname,
- *                    RedisModuleKey *key, void *privdata);
- * ctx - the redis module context provided to for the scan.
- * keyname - owned by the caller and need to be retained if used after this
- * function.
  *
- * key - holds info on the key and value, it is provided as best effort, in
- * some cases it might be NULL, in which case the user should (can) use
- * RedisModule_OpenKey (and CloseKey too).
- * when it is provided, it is owned by the caller and will be free when the
- * callback returns.
+ *     void scan_callback(RedisModuleCtx *ctx, RedisModuleString *keyname,
+ *                        RedisModuleKey *key, void *privdata);
  *
- * privdata - the user data provided to RedisModule_Scan.
+ * - `ctx`: the redis module context provided to for the scan.
+ * - `keyname`: owned by the caller and need to be retained if used after this
+ *   function.
+ * - `key`: holds info on the key and value, it is provided as best effort, in
+ *   some cases it might be NULL, in which case the user should (can) use
+ *   RedisModule_OpenKey() (and CloseKey too).
+ *   when it is provided, it is owned by the caller and will be free when the
+ *   callback returns.
+ * - `privdata`: the user data provided to RedisModule_Scan().
  *
  * The way it should be used:
+ *
  *      RedisModuleCursor *c = RedisModule_ScanCursorCreate();
  *      while(RedisModule_Scan(ctx, c, callback, privateData));
  *      RedisModule_ScanCursorDestroy(c);
@@ -6938,7 +6945,9 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) {
 /* Scan api that allows a module to scan the elements in a hash, set or sorted set key
  *
  * Callback for scan implementation.
- * void scan_callback(RedisModuleKey *key, RedisModuleString* field, RedisModuleString* value, void *privdata);
+ *
+ *     void scan_callback(RedisModuleKey *key, RedisModuleString* field, RedisModuleString* value, void *privdata);
+ *
  * - key - the redis key context provided to for the scan.
  * - field - field name, owned by the caller and need to be retained if used
  *   after this function.
@@ -6947,6 +6956,7 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) {
  * - privdata - the user data provided to RedisModule_ScanKey.
  *
  * The way it should be used:
+ *
  *      RedisModuleCursor *c = RedisModule_ScanCursorCreate();
  *      RedisModuleKey *key = RedisModule_OpenKey(...)
  *      while(RedisModule_ScanKey(key, c, callback, privateData));
@@ -6955,6 +6965,7 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) {
  *
  * It is also possible to use this API from another thread while the lock is acquired during
  * the actuall call to RM_ScanKey, and re-opening the key each time:
+ *
  *      RedisModuleCursor *c = RedisModule_ScanCursorCreate();
  *      RedisModule_ThreadSafeContextLock(ctx);
  *      RedisModuleKey *key = RedisModule_OpenKey(...)
@@ -7159,10 +7170,10 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) {
  *
  * The callback must be of this type:
  *
- *  int (*RedisModuleEventCallback)(RedisModuleCtx *ctx,
- *                                  RedisModuleEvent eid,
- *                                  uint64_t subevent,
- *                                  void *data);
+ *     int (*RedisModuleEventCallback)(RedisModuleCtx *ctx,
+ *                                     RedisModuleEvent eid,
+ *                                     uint64_t subevent,
+ *                                     void *data);
  *
  * The 'ctx' is a normal Redis module context that the callback can use in
  * order to call other modules APIs. The 'eid' is the event itself, this
@@ -7176,200 +7187,200 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) {
  *
  * Here is a list of events you can use as 'eid' and related sub events:
  *
- *      RedisModuleEvent_ReplicationRoleChanged
+ * * RedisModuleEvent_ReplicationRoleChanged:
  *
- *          This event is called when the instance switches from master
- *          to replica or the other way around, however the event is
- *          also called when the replica remains a replica but starts to
- *          replicate with a different master.
+ *   This event is called when the instance switches from master
+ *   to replica or the other way around, however the event is
+ *   also called when the replica remains a replica but starts to
+ *   replicate with a different master.
  *
- *          The following sub events are available:
+ *   The following sub events are available:
  *
- *              REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_MASTER
- *              REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_REPLICA
+ *   * REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_MASTER
+ *   * REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_REPLICA
  *
- *          The 'data' field can be casted by the callback to a
- *          RedisModuleReplicationInfo structure with the following fields:
+ *   The 'data' field can be casted by the callback to a
+ *   RedisModuleReplicationInfo structure with the following fields:
  *
- *              int master; // true if master, false if replica
- *              char *masterhost; // master instance hostname for NOW_REPLICA
- *              int masterport; // master instance port for NOW_REPLICA
- *              char *replid1; // Main replication ID
- *              char *replid2; // Secondary replication ID
- *              uint64_t repl1_offset; // Main replication offset
- *              uint64_t repl2_offset; // Offset of replid2 validity
+ *       int master; // true if master, false if replica
+ *       char *masterhost; // master instance hostname for NOW_REPLICA
+ *       int masterport; // master instance port for NOW_REPLICA
+ *       char *replid1; // Main replication ID
+ *       char *replid2; // Secondary replication ID
+ *       uint64_t repl1_offset; // Main replication offset
+ *       uint64_t repl2_offset; // Offset of replid2 validity
  *
- *      RedisModuleEvent_Persistence
+ * * RedisModuleEvent_Persistence
  *
- *          This event is called when RDB saving or AOF rewriting starts
- *          and ends. The following sub events are available:
+ *   This event is called when RDB saving or AOF rewriting starts
+ *   and ends. The following sub events are available:
  *
- *              REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START
- *              REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START
- *              REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START
- *              REDISMODULE_SUBEVENT_PERSISTENCE_ENDED
- *              REDISMODULE_SUBEVENT_PERSISTENCE_FAILED
+ *   * REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START
+ *   * REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START
+ *   * REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START
+ *   * REDISMODULE_SUBEVENT_PERSISTENCE_ENDED
+ *   * REDISMODULE_SUBEVENT_PERSISTENCE_FAILED
  *
- *          The above events are triggered not just when the user calls the
- *          relevant commands like BGSAVE, but also when a saving operation
- *          or AOF rewriting occurs because of internal server triggers.
- *          The SYNC_RDB_START sub events are happening in the forground due to
- *          SAVE command, FLUSHALL, or server shutdown, and the other RDB and
- *          AOF sub events are executed in a background fork child, so any
- *          action the module takes can only affect the generated AOF or RDB,
- *          but will not be reflected in the parent process and affect connected
- *          clients and commands. Also note that the AOF_START sub event may end
- *          up saving RDB content in case of an AOF with rdb-preamble.
+ *   The above events are triggered not just when the user calls the
+ *   relevant commands like BGSAVE, but also when a saving operation
+ *   or AOF rewriting occurs because of internal server triggers.
+ *   The SYNC_RDB_START sub events are happening in the forground due to
+ *   SAVE command, FLUSHALL, or server shutdown, and the other RDB and
+ *   AOF sub events are executed in a background fork child, so any
+ *   action the module takes can only affect the generated AOF or RDB,
+ *   but will not be reflected in the parent process and affect connected
+ *   clients and commands. Also note that the AOF_START sub event may end
+ *   up saving RDB content in case of an AOF with rdb-preamble.
  *
- *      RedisModuleEvent_FlushDB
+ * * RedisModuleEvent_FlushDB
  *
- *          The FLUSHALL, FLUSHDB or an internal flush (for instance
- *          because of replication, after the replica synchronization)
- *          happened. The following sub events are available:
+ *   The FLUSHALL, FLUSHDB or an internal flush (for instance
+ *   because of replication, after the replica synchronization)
+ *   happened. The following sub events are available:
  *
- *              REDISMODULE_SUBEVENT_FLUSHDB_START
- *              REDISMODULE_SUBEVENT_FLUSHDB_END
+ *   * REDISMODULE_SUBEVENT_FLUSHDB_START
+ *   * REDISMODULE_SUBEVENT_FLUSHDB_END
  *
- *          The data pointer can be casted to a RedisModuleFlushInfo
- *          structure with the following fields:
+ *   The data pointer can be casted to a RedisModuleFlushInfo
+ *   structure with the following fields:
  *
- *              int32_t async;  // True if the flush is done in a thread.
- *                                 See for instance FLUSHALL ASYNC.
- *                                 In this case the END callback is invoked
- *                                 immediately after the database is put
- *                                 in the free list of the thread.
- *              int32_t dbnum;  // Flushed database number, -1 for all the DBs
- *                                 in the case of the FLUSHALL operation.
+ *       int32_t async;  // True if the flush is done in a thread.
+ *                       // See for instance FLUSHALL ASYNC.
+ *                       // In this case the END callback is invoked
+ *                       // immediately after the database is put
+ *                       // in the free list of the thread.
+ *       int32_t dbnum;  // Flushed database number, -1 for all the DBs
+ *                       // in the case of the FLUSHALL operation.
  *
- *          The start event is called *before* the operation is initated, thus
- *          allowing the callback to call DBSIZE or other operation on the
- *          yet-to-free keyspace.
+ *   The start event is called *before* the operation is initated, thus
+ *   allowing the callback to call DBSIZE or other operation on the
+ *   yet-to-free keyspace.
  *
- *      RedisModuleEvent_Loading
+ * * RedisModuleEvent_Loading
  *
- *          Called on loading operations: at startup when the server is
- *          started, but also after a first synchronization when the
- *          replica is loading the RDB file from the master.
- *          The following sub events are available:
+ *   Called on loading operations: at startup when the server is
+ *   started, but also after a first synchronization when the
+ *   replica is loading the RDB file from the master.
+ *   The following sub events are available:
  *
- *              REDISMODULE_SUBEVENT_LOADING_RDB_START
- *              REDISMODULE_SUBEVENT_LOADING_AOF_START
- *              REDISMODULE_SUBEVENT_LOADING_REPL_START
- *              REDISMODULE_SUBEVENT_LOADING_ENDED
- *              REDISMODULE_SUBEVENT_LOADING_FAILED
+ *   * REDISMODULE_SUBEVENT_LOADING_RDB_START
+ *   * REDISMODULE_SUBEVENT_LOADING_AOF_START
+ *   * REDISMODULE_SUBEVENT_LOADING_REPL_START
+ *   * REDISMODULE_SUBEVENT_LOADING_ENDED
+ *   * REDISMODULE_SUBEVENT_LOADING_FAILED
  *
- *          Note that AOF loading may start with an RDB data in case of
- *          rdb-preamble, in which case you'll only receive an AOF_START event.
+ *   Note that AOF loading may start with an RDB data in case of
+ *   rdb-preamble, in which case you'll only receive an AOF_START event.
  *
  *
- *      RedisModuleEvent_ClientChange
+ * * RedisModuleEvent_ClientChange
  *
- *          Called when a client connects or disconnects.
- *          The data pointer can be casted to a RedisModuleClientInfo
- *          structure, documented in RedisModule_GetClientInfoById().
- *          The following sub events are available:
+ *   Called when a client connects or disconnects.
+ *   The data pointer can be casted to a RedisModuleClientInfo
+ *   structure, documented in RedisModule_GetClientInfoById().
+ *   The following sub events are available:
  *
- *              REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED
- *              REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED
+ *   * REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED
+ *   * REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED
  *
- *      RedisModuleEvent_Shutdown
+ * * RedisModuleEvent_Shutdown
  *
- *          The server is shutting down. No subevents are available.
+ *   The server is shutting down. No subevents are available.
  *
- *  RedisModuleEvent_ReplicaChange
+ * * RedisModuleEvent_ReplicaChange
  *
- *          This event is called when the instance (that can be both a
- *          master or a replica) get a new online replica, or lose a
- *          replica since it gets disconnected.
- *          The following sub events are available:
+ *   This event is called when the instance (that can be both a
+ *   master or a replica) get a new online replica, or lose a
+ *   replica since it gets disconnected.
+ *   The following sub events are available:
  *
- *              REDISMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE
- *              REDISMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE
+ *   * REDISMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE
+ *   * REDISMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE
  *
- *          No additional information is available so far: future versions
- *          of Redis will have an API in order to enumerate the replicas
- *          connected and their state.
+ *   No additional information is available so far: future versions
+ *   of Redis will have an API in order to enumerate the replicas
+ *   connected and their state.
  *
- *  RedisModuleEvent_CronLoop
+ * * RedisModuleEvent_CronLoop
  *
- *          This event is called every time Redis calls the serverCron()
- *          function in order to do certain bookkeeping. Modules that are
- *          required to do operations from time to time may use this callback.
- *          Normally Redis calls this function 10 times per second, but
- *          this changes depending on the "hz" configuration.
- *          No sub events are available.
+ *   This event is called every time Redis calls the serverCron()
+ *   function in order to do certain bookkeeping. Modules that are
+ *   required to do operations from time to time may use this callback.
+ *   Normally Redis calls this function 10 times per second, but
+ *   this changes depending on the "hz" configuration.
+ *   No sub events are available.
  *
- *          The data pointer can be casted to a RedisModuleCronLoop
- *          structure with the following fields:
+ *   The data pointer can be casted to a RedisModuleCronLoop
+ *   structure with the following fields:
  *
- *              int32_t hz;  // Approximate number of events per second.
+ *       int32_t hz;  // Approximate number of events per second.
  *
- *  RedisModuleEvent_MasterLinkChange
+ * * RedisModuleEvent_MasterLinkChange
  *
- *          This is called for replicas in order to notify when the
- *          replication link becomes functional (up) with our master,
- *          or when it goes down. Note that the link is not considered
- *          up when we just connected to the master, but only if the
- *          replication is happening correctly.
- *          The following sub events are available:
+ *   This is called for replicas in order to notify when the
+ *   replication link becomes functional (up) with our master,
+ *   or when it goes down. Note that the link is not considered
+ *   up when we just connected to the master, but only if the
+ *   replication is happening correctly.
+ *   The following sub events are available:
  *
- *              REDISMODULE_SUBEVENT_MASTER_LINK_UP
- *              REDISMODULE_SUBEVENT_MASTER_LINK_DOWN
+ *   * REDISMODULE_SUBEVENT_MASTER_LINK_UP
+ *   * REDISMODULE_SUBEVENT_MASTER_LINK_DOWN
  *
- *  RedisModuleEvent_ModuleChange
+ * * RedisModuleEvent_ModuleChange
  *
- *          This event is called when a new module is loaded or one is unloaded.
- *          The following sub events are available:
+ *   This event is called when a new module is loaded or one is unloaded.
+ *   The following sub events are available:
  *
- *              REDISMODULE_SUBEVENT_MODULE_LOADED
- *              REDISMODULE_SUBEVENT_MODULE_UNLOADED
+ *   * REDISMODULE_SUBEVENT_MODULE_LOADED
+ *   * REDISMODULE_SUBEVENT_MODULE_UNLOADED
  *
- *          The data pointer can be casted to a RedisModuleModuleChange
- *          structure with the following fields:
+ *   The data pointer can be casted to a RedisModuleModuleChange
+ *   structure with the following fields:
  *
- *              const char* module_name;  // Name of module loaded or unloaded.
- *              int32_t module_version;  // Module version.
+ *       const char* module_name;  // Name of module loaded or unloaded.
+ *       int32_t module_version;  // Module version.
  *
- *  RedisModuleEvent_LoadingProgress
+ * * RedisModuleEvent_LoadingProgress
  *
- *          This event is called repeatedly called while an RDB or AOF file
- *          is being loaded.
- *          The following sub events are availble:
+ *   This event is called repeatedly called while an RDB or AOF file
+ *   is being loaded.
+ *   The following sub events are availble:
  *
- *              REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB
- *              REDISMODULE_SUBEVENT_LOADING_PROGRESS_AOF
+ *   * REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB
+ *   * REDISMODULE_SUBEVENT_LOADING_PROGRESS_AOF
  *
- *          The data pointer can be casted to a RedisModuleLoadingProgress
- *          structure with the following fields:
+ *   The data pointer can be casted to a RedisModuleLoadingProgress
+ *   structure with the following fields:
  *
- *              int32_t hz;  // Approximate number of events per second.
- *              int32_t progress;  // Approximate progress between 0 and 1024,
- *                                    or -1 if unknown.
+ *       int32_t hz;  // Approximate number of events per second.
+ *       int32_t progress;  // Approximate progress between 0 and 1024,
+ *                             or -1 if unknown.
  *
- *      RedisModuleEvent_SwapDB
+ * * RedisModuleEvent_SwapDB
  *
- *          This event is called when a SWAPDB command has been successfully
- *          Executed. 
- *          For this event call currently there is no subevents available.
+ *   This event is called when a SWAPDB command has been successfully
+ *   Executed.
+ *   For this event call currently there is no subevents available.
  *
- *          The data pointer can be casted to a RedisModuleSwapDbInfo
- *          structure with the following fields:
+ *   The data pointer can be casted to a RedisModuleSwapDbInfo
+ *   structure with the following fields:
  *
- *             int32_t dbnum_first;    // Swap Db first dbnum 
- *             int32_t dbnum_second;   // Swap Db second dbnum 
+ *       int32_t dbnum_first;    // Swap Db first dbnum
+ *       int32_t dbnum_second;   // Swap Db second dbnum
  *
- *      RedisModuleEvent_ReplBackup
+ * * RedisModuleEvent_ReplBackup
  *
- *          Called when diskless-repl-load config is set to swapdb,
- *          And redis needs to backup the the current database for the
- *          possibility to be restored later. A module with global data and
- *          maybe with aux_load and aux_save callbacks may need to use this
- *          notification to backup / restore / discard its globals.
- *          The following sub events are available:
+ *   Called when diskless-repl-load config is set to swapdb,
+ *   And redis needs to backup the the current database for the
+ *   possibility to be restored later. A module with global data and
+ *   maybe with aux_load and aux_save callbacks may need to use this
+ *   notification to backup / restore / discard its globals.
+ *   The following sub events are available:
  *
- *              REDISMODULE_SUBEVENT_REPL_BACKUP_CREATE
- *              REDISMODULE_SUBEVENT_REPL_BACKUP_RESTORE
- *              REDISMODULE_SUBEVENT_REPL_BACKUP_DISCARD
+ *   * REDISMODULE_SUBEVENT_REPL_BACKUP_CREATE
+ *   * REDISMODULE_SUBEVENT_REPL_BACKUP_RESTORE
+ *   * REDISMODULE_SUBEVENT_REPL_BACKUP_DISCARD
  *
  *
  * The function returns REDISMODULE_OK if the module was successfully subscribed
@@ -8064,7 +8075,8 @@ int RM_GetLFU(RedisModuleKey *key, long long *lfu_freq) {
  * the module can check if a certain set of flags are supported
  * by the redis server version in use.
  * Example:
- *        int supportedFlags = RM_GetContextFlagsAll()
+ *
+ *        int supportedFlags = RM_GetContextFlagsAll();
  *        if (supportedFlags & REDISMODULE_CTX_FLAGS_MULTI) {
  *              // REDISMODULE_CTX_FLAGS_MULTI is supported
  *        } else{
@@ -8080,7 +8092,8 @@ int RM_GetContextFlagsAll() {
  * the module can check if a certain set of flags are supported
  * by the redis server version in use.
  * Example:
- *        int supportedFlags = RM_GetKeyspaceNotificationFlagsAll()
+ *
+ *        int supportedFlags = RM_GetKeyspaceNotificationFlagsAll();
  *        if (supportedFlags & REDISMODULE_NOTIFY_LOADED) {
  *              // REDISMODULE_NOTIFY_LOADED is supported
  *        } else{
@@ -8150,8 +8163,8 @@ int RM_ModuleTypeReplaceValue(RedisModuleKey *key, moduleType *mt, void *new_val
  * an error condition. Error conditions are indicated by setting errno
  * as folllows:
  *
- *  ENOENT: Specified command does not exist.
- *  EINVAL: Invalid command arity specified.
+ * * ENOENT: Specified command does not exist.
+ * * EINVAL: Invalid command arity specified.
  *
  * NOTE: The returned array is not a Redis Module object so it does not
  * get automatically freed even when auto-memory is used. The caller
@@ -8247,11 +8260,11 @@ int RM_DefragShouldStop(RedisModuleDefragCtx *ctx) {
  * data type.
  *
  * This behavior is reserved to cases where late defrag is performed. Late
- * defrag is selected for keys that implement the free_effort callback and
- * return a free_effort value that is larger than the defrag
+ * defrag is selected for keys that implement the `free_effort` callback and
+ * return a `free_effort` value that is larger than the defrag
  * 'active-defrag-max-scan-fields' configuration directive.
  *
- * Smaller keys, keys that do not implement free_effort or the global
+ * Smaller keys, keys that do not implement `free_effort` or the global
  * defrag callback are not called in late-defrag mode. In those cases, a
  * call to this function will return REDISMODULE_ERR.
  *
diff --git a/src/modules/gendoc.rb b/src/modules/gendoc.rb
index ee6572884..249c8b6ea 100644
--- a/src/modules/gendoc.rb
+++ b/src/modules/gendoc.rb
@@ -4,16 +4,18 @@
 # Convert the C comment to markdown
 def markdown(s)
     s = s.gsub(/\*\/$/,"")
-    s = s.gsub(/^ \* {0,1}/,"")
-    s = s.gsub(/^\/\* /,"")
+    s = s.gsub(/^ ?\* ?/,"")
+    s = s.gsub(/^\/\*\*? ?/,"")
     s.chop! while s[-1] == "\n" || s[-1] == " "
     lines = s.split("\n")
     newlines = []
+    # Backquote function, macro and type names, except if already backquoted and
+    # in code blocks indented by 4 spaces.
     lines.each{|l|
-        if l[0] != ' '
-            l = l.gsub(/RM_[A-z()]+/){|x| "`#{x}`"}
-            l = l.gsub(/RedisModule_[A-z()]+/){|x| "`#{x}`"}
-            l = l.gsub(/REDISMODULE_[A-z]+/){|x| "`#{x}`"}
+        if not l.start_with?('    ')
+            l = l.gsub(/(?<!`)RM_[A-z()]+/){|x| "`#{x}`"}
+            l = l.gsub(/(?<!`)RedisModule[A-z()]+/){|x| "`#{x}`"}
+            l = l.gsub(/(?<!`)REDISMODULE_[A-z]+/){|x| "`#{x}`"}
         end
         newlines << l
     }
@@ -41,6 +43,7 @@ def docufy(src,i)
 end
 
 puts "# Modules API reference\n\n"
+puts "<!-- This file is generated from module.c using gendoc.rb -->\n\n"
 src = File.open("../module.c").to_a
 src.each_with_index{|line,i|
     if line =~ /RM_/ && line[0] != ' ' && line[0] != '#' && line[0] != '/'

From fcb3dfe56d3e2e0732c6787fe11789252eaf9ba2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= <viktor.soderqvist@est.tech>
Date: Thu, 14 Jan 2021 20:34:00 +0100
Subject: [PATCH 0972/1098] Rename non-API RM-prefixed functions to hide them
 from API docs

The prefix is changed from `RM_` to `module` on the following
internal functions, to prevent them from appearing in the API docs:

    RM_LogRaw -> moduleLogRaw
    RM_FreeCallReplyRec -> moduleFreeCallReplyRec
    RM_ZsetAddFlagsToCoreFlags -> moduleZsetAddFlagsToCoreFlags
    RM_ZsetAddFlagsFromCoreFlags -> moduleZsetAddFlagsFromCoreFlags
---
 src/module.c | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/module.c b/src/module.c
index c8dd237ad..fd3f90d84 100644
--- a/src/module.c
+++ b/src/module.c
@@ -2399,7 +2399,7 @@ RedisModuleString *RM_ListPop(RedisModuleKey *key, int where) {
 
 /* Conversion from/to public flags of the Modules API and our private flags,
  * so that we have everything decoupled. */
-int RM_ZsetAddFlagsToCoreFlags(int flags) {
+int moduleZsetAddFlagsToCoreFlags(int flags) {
     int retflags = 0;
     if (flags & REDISMODULE_ZADD_XX) retflags |= ZADD_XX;
     if (flags & REDISMODULE_ZADD_NX) retflags |= ZADD_NX;
@@ -2409,7 +2409,7 @@ int RM_ZsetAddFlagsToCoreFlags(int flags) {
 }
 
 /* See previous function comment. */
-int RM_ZsetAddFlagsFromCoreFlags(int flags) {
+int moduleZsetAddFlagsFromCoreFlags(int flags) {
     int retflags = 0;
     if (flags & ZADD_ADDED) retflags |= REDISMODULE_ZADD_ADDED;
     if (flags & ZADD_UPDATED) retflags |= REDISMODULE_ZADD_UPDATED;
@@ -2454,12 +2454,12 @@ int RM_ZsetAdd(RedisModuleKey *key, double score, RedisModuleString *ele, int *f
     if (!(key->mode & REDISMODULE_WRITE)) return REDISMODULE_ERR;
     if (key->value && key->value->type != OBJ_ZSET) return REDISMODULE_ERR;
     if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_ZSET);
-    if (flagsptr) flags = RM_ZsetAddFlagsToCoreFlags(*flagsptr);
+    if (flagsptr) flags = moduleZsetAddFlagsToCoreFlags(*flagsptr);
     if (zsetAdd(key->value,score,ele->ptr,&flags,NULL) == 0) {
         if (flagsptr) *flagsptr = 0;
         return REDISMODULE_ERR;
     }
-    if (flagsptr) *flagsptr = RM_ZsetAddFlagsFromCoreFlags(flags);
+    if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(flags);
     return REDISMODULE_OK;
 }
 
@@ -2481,7 +2481,7 @@ int RM_ZsetIncrby(RedisModuleKey *key, double score, RedisModuleString *ele, int
     if (!(key->mode & REDISMODULE_WRITE)) return REDISMODULE_ERR;
     if (key->value && key->value->type != OBJ_ZSET) return REDISMODULE_ERR;
     if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_ZSET);
-    if (flagsptr) flags = RM_ZsetAddFlagsToCoreFlags(*flagsptr);
+    if (flagsptr) flags = moduleZsetAddFlagsToCoreFlags(*flagsptr);
     flags |= ZADD_INCR;
     if (zsetAdd(key->value,score,ele->ptr,&flags,newscore) == 0) {
         if (flagsptr) *flagsptr = 0;
@@ -2492,7 +2492,7 @@ int RM_ZsetIncrby(RedisModuleKey *key, double score, RedisModuleString *ele, int
         *flagsptr = 0;
         return REDISMODULE_ERR;
     }
-    if (flagsptr) *flagsptr = RM_ZsetAddFlagsFromCoreFlags(flags);
+    if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(flags);
     return REDISMODULE_OK;
 }
 
@@ -3163,9 +3163,8 @@ void moduleParseCallReply_Array(RedisModuleCallReply *reply) {
     reply->type = REDISMODULE_REPLY_ARRAY;
 }
 
-/* Free a Call reply and all the nested replies it contains if it's an
- * array. */
-void RM_FreeCallReply_Rec(RedisModuleCallReply *reply, int freenested){
+/* Recursive free reply function. */
+void moduleFreeCallReplyRec(RedisModuleCallReply *reply, int freenested){
     /* Don't free nested replies by default: the user must always free the
      * toplevel reply. However be gentle and don't crash if the module
      * misuses the API. */
@@ -3175,7 +3174,7 @@ void RM_FreeCallReply_Rec(RedisModuleCallReply *reply, int freenested){
         if (reply->type == REDISMODULE_REPLY_ARRAY) {
             size_t j;
             for (j = 0; j < reply->len; j++)
-                RM_FreeCallReply_Rec(reply->val.array+j,1);
+                moduleFreeCallReplyRec(reply->val.array+j,1);
             zfree(reply->val.array);
         }
     }
@@ -3190,13 +3189,14 @@ void RM_FreeCallReply_Rec(RedisModuleCallReply *reply, int freenested){
     }
 }
 
-/* Wrapper for the recursive free reply function. This is needed in order
- * to have the first level function to return on nested replies, but only
- * if called by the module API. */
+/* Free a Call reply and all the nested replies it contains if it's an
+ * array. */
 void RM_FreeCallReply(RedisModuleCallReply *reply) {
-
+    /* This is a wrapper for the recursive free reply function. This is needed
+     * in order to have the first level function to return on nested replies,
+     * but only if called by the module API. */
     RedisModuleCtx *ctx = reply->ctx;
-    RM_FreeCallReply_Rec(reply,0);
+    moduleFreeCallReplyRec(reply,0);
     autoMemoryFreed(ctx,REDISMODULE_AM_REPLY,reply);
 }
 
@@ -4386,7 +4386,7 @@ const RedisModuleString *RM_GetKeyNameFromModuleKey(RedisModuleKey *key) {
  *      RM_LogIOError()
  *
  */
-void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_list ap) {
+void moduleLogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_list ap) {
     char msg[LOG_MAX_LEN];
     size_t name_len;
     int level;
@@ -4425,7 +4425,7 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li
 void RM_Log(RedisModuleCtx *ctx, const char *levelstr, const char *fmt, ...) {
     va_list ap;
     va_start(ap, fmt);
-    RM_LogRaw(ctx? ctx->module: NULL,levelstr,fmt,ap);
+    moduleLogRaw(ctx? ctx->module: NULL,levelstr,fmt,ap);
     va_end(ap);
 }
 
@@ -4437,7 +4437,7 @@ void RM_Log(RedisModuleCtx *ctx, const char *levelstr, const char *fmt, ...) {
 void RM_LogIOError(RedisModuleIO *io, const char *levelstr, const char *fmt, ...) {
     va_list ap;
     va_start(ap, fmt);
-    RM_LogRaw(io->type->module,levelstr,fmt,ap);
+    moduleLogRaw(io->type->module,levelstr,fmt,ap);
     va_end(ap);
 }
 

From 294f93af97a6964c28f8d9cabbccc5da8950f9b0 Mon Sep 17 00:00:00 2001
From: Yang Bodong <bodong.ybd@alibaba-inc.com>
Date: Fri, 15 Jan 2021 21:32:58 +0800
Subject: [PATCH 0973/1098] Add lazyfree-lazy-user-flush config to control
 default behavior of FLUSH[ALL|DB], SCRIPT FLUSH (#8258)

* Adds ASYNC and SYNC arguments to SCRIPT FLUSH
* Adds SYNC argument to FLUSHDB and FLUSHALL
* Adds new config to control the default behavior of FLUSHDB, FLUSHALL and SCRIPT FLUASH.

the new behavior is as follows:
* FLUSH[ALL|DB],SCRIPT FLUSH: Determine sync or async according to the
  value of lazyfree-lazy-user-flush.
* FLUSH[ALL|DB],SCRIPT FLUSH ASYNC: Always flushes the database in an async manner.
* FLUSH[ALL|DB],SCRIPT FLUSH SYNC: Always flushes the database in a sync manner.
---
 redis.conf               |  7 +++++++
 src/config.c             |  1 +
 src/db.c                 | 20 +++++++++++---------
 src/lazyfree.c           | 18 ++++++++++++++++++
 src/scripting.c          | 32 +++++++++++++++++++++++++-------
 src/server.h             |  2 ++
 tests/unit/scripting.tcl |  9 +++++++++
 7 files changed, 73 insertions(+), 16 deletions(-)

diff --git a/redis.conf b/redis.conf
index a5062fda9..5f9eed7fe 100644
--- a/redis.conf
+++ b/redis.conf
@@ -1089,6 +1089,13 @@ replica-lazy-flush no
 
 lazyfree-lazy-user-del no
 
+# FLUSHDB, FLUSHALL, and SCRIPT FLUSH support both asynchronous and synchronous
+# deletion, which can be controlled by passing the [SYNC|ASYNC] flags into the
+# commands. When neither flag is passed, this directive will be used to determine
+# if the data should be deleted asynchronously.
+
+lazyfree-lazy-user-flush no
+
 ################################ THREADED I/O #################################
 
 # Redis is mostly single threaded, however there are certain threaded
diff --git a/src/config.c b/src/config.c
index 2e109dbae..2442ace8e 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2385,6 +2385,7 @@ standardConfig configs[] = {
     createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, 0, NULL, NULL),
     createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, 0, NULL, NULL),
     createBoolConfig("lazyfree-lazy-user-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_user_del , 0, NULL, NULL),
+    createBoolConfig("lazyfree-lazy-user-flush", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_user_flush , 0, NULL, NULL),
     createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, 0, NULL, NULL),
     createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, 0, NULL, NULL),
     createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, 0, NULL, NULL),
diff --git a/src/db.c b/src/db.c
index 13126988f..29fef13fb 100644
--- a/src/db.c
+++ b/src/db.c
@@ -595,21 +595,23 @@ void signalFlushedDb(int dbid, int async) {
 /* Return the set of flags to use for the emptyDb() call for FLUSHALL
  * and FLUSHDB commands.
  *
- * Currently the command just attempts to parse the "ASYNC" option. It
- * also checks if the command arity is wrong.
+ * sync: flushes the database in an sync manner.
+ * async: flushes the database in an async manner.
+ * no option: determine sync or async according to the value of lazyfree-lazy-user-flush.
  *
  * On success C_OK is returned and the flags are stored in *flags, otherwise
  * C_ERR is returned and the function sends an error to the client. */
 int getFlushCommandFlags(client *c, int *flags) {
     /* Parse the optional ASYNC option. */
-    if (c->argc > 1) {
-        if (c->argc > 2 || strcasecmp(c->argv[1]->ptr,"async")) {
-            addReplyErrorObject(c,shared.syntaxerr);
-            return C_ERR;
-        }
-        *flags = EMPTYDB_ASYNC;
-    } else {
+    if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"sync")) {
         *flags = EMPTYDB_NO_FLAGS;
+    } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"async")) {
+        *flags = EMPTYDB_ASYNC;
+    } else if (c->argc == 1) {
+        *flags = server.lazyfree_lazy_user_flush ? EMPTYDB_ASYNC : EMPTYDB_NO_FLAGS;
+    } else {
+        addReplyErrorObject(c,shared.syntaxerr);
+        return C_ERR;
     }
     return C_OK;
 }
diff --git a/src/lazyfree.c b/src/lazyfree.c
index 8b9f0e2dc..f18b2027f 100644
--- a/src/lazyfree.c
+++ b/src/lazyfree.c
@@ -49,6 +49,14 @@ void lazyFreeTrackingTable(void *args[]) {
     atomicIncr(lazyfreed_objects,len);
 }
 
+void lazyFreeLuaScripts(void *args[]) {
+    dict *lua_scripts = args[0];
+    long long len = dictSize(lua_scripts);
+    dictRelease(lua_scripts);
+    atomicDecr(lazyfree_objects,len);
+    atomicIncr(lazyfreed_objects,len);
+}
+
 /* Return the number of currently pending objects to free. */
 size_t lazyfreeGetPendingObjectsCount(void) {
     size_t aux;
@@ -212,3 +220,13 @@ void freeTrackingRadixTreeAsync(rax *tracking) {
     atomicIncr(lazyfree_objects,tracking->numele);
     bioCreateLazyFreeJob(lazyFreeTrackingTable,1,tracking);
 }
+
+/* Free lua_scripts dict, if the dict is huge enough, free it in async way. */
+void freeLuaScriptsAsync(dict *lua_scripts) {
+    if (dictSize(lua_scripts) > LAZYFREE_THRESHOLD) {
+        atomicIncr(lazyfree_objects,dictSize(lua_scripts));
+        bioCreateLazyFreeJob(lazyFreeLuaScripts,1,lua_scripts);
+    } else {
+        dictRelease(lua_scripts);
+    }
+}
diff --git a/src/scripting.c b/src/scripting.c
index 75604e4d8..41469ee2e 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -1282,14 +1282,17 @@ void scriptingInit(int setup) {
 
 /* Release resources related to Lua scripting.
  * This function is used in order to reset the scripting environment. */
-void scriptingRelease(void) {
-    dictRelease(server.lua_scripts);
+void scriptingRelease(int async) {
+    if (async)
+        freeLuaScriptsAsync(server.lua_scripts);
+    else
+        dictRelease(server.lua_scripts);
     server.lua_scripts_mem = 0;
     lua_close(server.lua);
 }
 
-void scriptingReset(void) {
-    scriptingRelease();
+void scriptingReset(int async) {
+    scriptingRelease(async);
     scriptingInit(0);
 }
 
@@ -1711,8 +1714,12 @@ void scriptCommand(client *c) {
 "    Set the debug mode for subsequent scripts executed.",
 "EXISTS <sha1> [<sha1> ...]",
 "    Return information about the existence of the scripts in the script cache.",
-"FLUSH",
+"FLUSH [ASYNC|SYNC]",
 "    Flush the Lua scripts cache. Very dangerous on replicas.",
+"    When called without the optional mode argument, the behavior is determined by the",
+"    lazyfree-lazy-user-flush configuration directive. Valid modes are:",
+"    * ASYNC: Asynchronously flush the scripts cache.",
+"    * SYNC: Synchronously flush the scripts cache.",
 "KILL",
 "    Kill the currently executing Lua script.",
 "LOAD <script>",
@@ -1720,8 +1727,19 @@ void scriptCommand(client *c) {
 NULL
         };
         addReplyHelp(c, help);
-    } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"flush")) {
-        scriptingReset();
+    } else if (c->argc >= 2 && !strcasecmp(c->argv[1]->ptr,"flush")) {
+        int async = 0;
+        if (c->argc == 3 && !strcasecmp(c->argv[2]->ptr,"sync")) {
+            async = 0;
+        } else if (c->argc == 3 && !strcasecmp(c->argv[2]->ptr,"async")) {
+            async = 1;
+        } else if (c->argc == 2) {
+            async = server.lazyfree_lazy_user_flush ? 1 : 0;
+        } else {
+            addReplyError(c,"SCRIPT FLUSH only support SYNC|ASYNC option");
+            return;
+        }
+        scriptingReset(async);
         addReply(c,shared.ok);
         replicationScriptCacheFlush();
         server.dirty++; /* Propagating this command is a good idea. */
diff --git a/src/server.h b/src/server.h
index eb967a042..0de7b3a1e 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1530,6 +1530,7 @@ struct redisServer {
     int lazyfree_lazy_expire;
     int lazyfree_lazy_server_del;
     int lazyfree_lazy_user_del;
+    int lazyfree_lazy_user_flush;
     /* Latency monitor */
     long long latency_monitor_threshold;
     dict *latency_events;
@@ -2344,6 +2345,7 @@ int ldbRemoveChild(pid_t pid);
 void ldbKillForkedSessions(void);
 int ldbPendingChildren(void);
 sds luaCreateFunction(client *c, lua_State *lua, robj *body);
+void freeLuaScriptsAsync(dict *lua_scripts);
 
 /* Blocked clients */
 void processUnblockedClients(void);
diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl
index 6fd152594..3aa3c0fba 100644
--- a/tests/unit/scripting.tcl
+++ b/tests/unit/scripting.tcl
@@ -330,6 +330,15 @@ start_server {tags {"scripting"}} {
         set e
     } {NOSCRIPT*}
 
+    test {SCRIPTING FLUSH ASYNC} {
+        for {set j 0} {$j < 100} {incr j} {
+            r script load "return $j"
+        }
+        assert { [string match "*number_of_cached_scripts:100*" [r info Memory]] }
+        r script flush async
+        assert { [string match "*number_of_cached_scripts:0*" [r info Memory]] }
+    }
+
     test {SCRIPT EXISTS - can detect already defined scripts?} {
         r eval "return 1+1" 0
         r script exists a27e7e8a43702b7046d4f6a7ccf5b60cef6b9bd9 a27e7e8a43702b7046d4f6a7ccf5b60cef6b9bda

From f5cf1e46a4812f0616f0ce763417b8c84f035493 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Sun, 17 Jan 2021 08:43:21 -0500
Subject: [PATCH 0974/1098] [Sentinel] more specific config errors for
 duplicate known-replica and known-sentinel (#8332)

Previously invalid configuration errors were not very specific and in some cases hard to understand.

Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
---
 src/sentinel.c | 37 ++++++++++++++++++++++++++++++-------
 1 file changed, 30 insertions(+), 7 deletions(-)

diff --git a/src/sentinel.c b/src/sentinel.c
index a374977de..089282b71 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -1642,6 +1642,33 @@ char *sentinelInstanceMapCommand(sentinelRedisInstance *ri, char *command) {
 }
 
 /* ============================ Config handling ============================= */
+
+/* Generalise handling create instance error. Use SRI_MASTER, SRI_SLAVE or
+ * SRI_SENTINEL as a role value. */
+char *sentinelCheckCreateInstanceErrors(int role) {
+    switch(errno) {
+    case EBUSY:
+        switch (role) {
+        case SRI_MASTER:
+            return "Duplicate master name.";
+        case SRI_SLAVE:
+            return "Duplicate hostname and port for replica.";
+        case SRI_SENTINEL:
+            return "Duplicate runid for sentinel.";
+        default:
+            serverAssert(0);
+            break;
+        }
+        break;
+    case ENOENT:
+        return "Can't resolve instance hostname.";
+    case EINVAL:
+        return "Invalid port number.";
+    default:
+        return "Unknown Error for creating instances.";
+    }
+}
+
 char *sentinelHandleConfiguration(char **argv, int argc) {
     sentinelRedisInstance *ri;
 
@@ -1653,11 +1680,7 @@ char *sentinelHandleConfiguration(char **argv, int argc) {
         if (createSentinelRedisInstance(argv[1],SRI_MASTER,argv[2],
                                         atoi(argv[3]),quorum,NULL) == NULL)
         {
-            switch(errno) {
-            case EBUSY: return "Duplicated master name.";
-            case ENOENT: return "Can't resolve master instance hostname.";
-            case EINVAL: return "Invalid port number";
-            }
+            return sentinelCheckCreateInstanceErrors(SRI_MASTER);
         }
     } else if (!strcasecmp(argv[0],"down-after-milliseconds") && argc == 3) {
         /* down-after-milliseconds <name> <milliseconds> */
@@ -1739,7 +1762,7 @@ char *sentinelHandleConfiguration(char **argv, int argc) {
         if ((slave = createSentinelRedisInstance(NULL,SRI_SLAVE,argv[2],
                     atoi(argv[3]), ri->quorum, ri)) == NULL)
         {
-            return "Wrong hostname or port for replica.";
+            return sentinelCheckCreateInstanceErrors(SRI_SLAVE);
         }
     } else if (!strcasecmp(argv[0],"known-sentinel") &&
                (argc == 4 || argc == 5)) {
@@ -1752,7 +1775,7 @@ char *sentinelHandleConfiguration(char **argv, int argc) {
             if ((si = createSentinelRedisInstance(argv[4],SRI_SENTINEL,argv[2],
                         atoi(argv[3]), ri->quorum, ri)) == NULL)
             {
-                return "Wrong hostname or port for sentinel.";
+                return sentinelCheckCreateInstanceErrors(SRI_SENTINEL);
             }
             si->runid = sdsnew(argv[4]);
             sentinelTryConnectionSharing(si);

From 522d93607a46e28b035c675fe066778c453e84ba Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 17 Jan 2021 15:48:48 +0200
Subject: [PATCH 0975/1098] Add io-thread daily CI tests. (#8232)

This adds basic coverage to IO threads by running the cluster and few selected Redis test suite tests with the IO threads enabled.

Also provides some necessary additional improvements to the test suite:

* Add --config to sentinel/cluster tests for arbitrary configuration.
* Fix --tags whitelisting which was broken.
* Add a `network` tag to some tests that are more network intensive. This is work in progress and more tests should be properly tagged in the future.
---
 .github/workflows/daily.yml           | 17 ++++++++
 tests/instances.tcl                   |  9 +++++
 tests/integration/rdb.tcl             |  4 ++
 tests/integration/redis-benchmark.tcl |  2 +-
 tests/integration/replication-4.tcl   |  2 +-
 tests/integration/replication.tcl     |  2 +-
 tests/support/server.tcl              | 56 ++++++++++++++++++++-------
 tests/unit/limits.tcl                 |  2 +-
 tests/unit/pause.tcl                  |  2 +-
 tests/unit/protocol.tcl               |  2 +-
 tests/unit/pubsub.tcl                 |  2 +-
 tests/unit/scan.tcl                   |  2 +-
 tests/unit/tracking.tcl               |  2 +-
 tests/unit/wait.tcl                   |  2 +-
 14 files changed, 81 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 028c44e0c..cfbf24f87 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -99,6 +99,23 @@ jobs:
         ./runtest-cluster --tls
         ./runtest-cluster
 
+  test-ubuntu-io-threads:
+    runs-on: ubuntu-latest
+    if: github.repository == 'redis/redis'
+    timeout-minutes: 14400
+    steps:
+    - uses: actions/checkout@v2
+    - name: make
+      run: |
+        make
+    - name: test
+      run: |
+        sudo apt-get install tcl8.5 tcl-tls
+        ./runtest --config io-threads 4 --config io-threads-do-reads yes --accurate --verbose --tags network
+    - name: cluster tests
+      run: |
+        ./runtest-cluster --config io-threads 4 --config io-threads-do-reads yes
+
   test-valgrind:
     runs-on: ubuntu-latest
     if: github.repository == 'redis/redis'
diff --git a/tests/instances.tcl b/tests/instances.tcl
index a9cc01008..18eb1a402 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -24,6 +24,7 @@ set ::simulate_error 0
 set ::failed 0
 set ::sentinel_instances {}
 set ::redis_instances {}
+set ::global_config {}
 set ::sentinel_base_port 20000
 set ::redis_base_port 30000
 set ::redis_port_count 1024
@@ -92,6 +93,9 @@ proc spawn_instance {type base_port count {conf {}}} {
         foreach directive $conf {
             puts $cfg $directive
         }
+        dict for {name val} $::global_config {
+            puts $cfg "$name $val"
+        }
         close $cfg
 
         # Finally exec it and remember the pid for later cleanup.
@@ -239,6 +243,10 @@ proc parse_options {} {
                 -certfile "$::tlsdir/client.crt" \
                 -keyfile "$::tlsdir/client.key"
             set ::tls 1
+        } elseif {$opt eq {--config}} {
+            set val2 [lindex $::argv [expr $j+2]]
+            dict set ::global_config $val $val2
+            incr j 2
         } elseif {$opt eq "--help"} {
             puts "--single <pattern>      Only runs tests specified by pattern."
             puts "--dont-clean            Keep log files on exit."
@@ -246,6 +254,7 @@ proc parse_options {} {
             puts "--fail                  Simulate a test failure."
             puts "--valgrind              Run with valgrind."
             puts "--tls                   Run tests in TLS mode."
+            puts "--config <k> <v>        Extra config argument(s)."
             puts "--help                  Shows this help."
             exit 0
         } else {
diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl
index 99495b2b7..a89221197 100644
--- a/tests/integration/rdb.tcl
+++ b/tests/integration/rdb.tcl
@@ -1,3 +1,5 @@
+tags {"rdb"} {
+
 set server_path [tmpdir "server.rdb-encoding-test"]
 
 # Copy RDB with different encodings in server path
@@ -289,3 +291,5 @@ start_server {overrides {save ""}} {
     }
 }
 } ;# system_name
+
+} ;# tags
diff --git a/tests/integration/redis-benchmark.tcl b/tests/integration/redis-benchmark.tcl
index 5a4f09952..3684d7c3b 100644
--- a/tests/integration/redis-benchmark.tcl
+++ b/tests/integration/redis-benchmark.tcl
@@ -5,7 +5,7 @@ proc cmdstat {cmd} {
     return [cmdrstat $cmd r]
 }
 
-start_server {tags {"benchmark"}} {
+start_server {tags {"benchmark network"}} {
     start_server {} {
         set master_host [srv 0 host]
         set master_port [srv 0 port]
diff --git a/tests/integration/replication-4.tcl b/tests/integration/replication-4.tcl
index 8071c4f97..c867001b8 100644
--- a/tests/integration/replication-4.tcl
+++ b/tests/integration/replication-4.tcl
@@ -1,4 +1,4 @@
-start_server {tags {"repl"}} {
+start_server {tags {"repl network"}} {
     start_server {} {
 
         set master [srv -1 client]
diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl
index 6c437ba71..8d09c68c1 100644
--- a/tests/integration/replication.tcl
+++ b/tests/integration/replication.tcl
@@ -5,7 +5,7 @@ proc log_file_matches {log pattern} {
     string match $pattern $content
 }
 
-start_server {tags {"repl"}} {
+start_server {tags {"repl network"}} {
     set slave [srv 0 client]
     set slave_host [srv 0 host]
     set slave_port [srv 0 port]
diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index 77ba31d84..0d36d46be 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -152,20 +152,48 @@ proc server_is_up {host port retrynum} {
     return 0
 }
 
+# Check if current ::tags match requested tags. If ::allowtags are used,
+# there must be some intersection. If ::denytags are used, no intersection
+# is allowed. Returns 1 if tags are acceptable or 0 otherwise, in which
+# case err_return names a return variable for the message to be logged.
+proc tags_acceptable {err_return} {
+    upvar $err_return err
+
+    # If tags are whitelisted, make sure there's match
+    if {[llength $::allowtags] > 0} {
+        set matched 0
+        foreach tag $::allowtags {
+            if {[lsearch $::tags $tag] >= 0} {
+                incr matched
+            }
+        }
+        if {$matched < 1} {
+            set err "Tag: none of the tags allowed"
+            return 0
+        }
+    }
+
+    foreach tag $::denytags {
+        if {[lsearch $::tags $tag] >= 0} {
+            set err "Tag: $tag denied"
+            return 0
+        }
+    }
+
+    return 1
+}
+
 # doesn't really belong here, but highly coupled to code in start_server
 proc tags {tags code} {
     # If we 'tags' contain multiple tags, quoted and seperated by spaces,
     # we want to get rid of the quotes in order to have a proper list
     set tags [string map { \" "" } $tags]
     set ::tags [concat $::tags $tags]
-    # We skip unwanted tags
-    foreach tag $::denytags {
-        if {[lsearch $::tags $tag] >= 0} {
-            incr ::num_aborted
-            send_data_packet $::test_server_fd ignore "Tag: $tag"
-            set ::tags [lrange $::tags 0 end-[llength $tags]]
-            return
-        }
+    if {![tags_acceptable err]} {
+        incr ::num_aborted
+        send_data_packet $::test_server_fd ignore $err
+        set ::tags [lrange $::tags 0 end-[llength $tags]]
+        return
     }
     uplevel 1 $code
     set ::tags [lrange $::tags 0 end-[llength $tags]]
@@ -267,13 +295,11 @@ proc start_server {options {code undefined}} {
     }
 
     # We skip unwanted tags
-    foreach tag $::denytags {
-        if {[lsearch $::tags $tag] >= 0} {
-            incr ::num_aborted
-            send_data_packet $::test_server_fd ignore "Tag: $tag"
-            set ::tags [lrange $::tags 0 end-[llength $tags]]
-            return
-        }
+    if {![tags_acceptable err]} {
+        incr ::num_aborted
+        send_data_packet $::test_server_fd ignore $err
+        set ::tags [lrange $::tags 0 end-[llength $tags]]
+        return
     }
 
     # If we are running against an external server, we just push the
diff --git a/tests/unit/limits.tcl b/tests/unit/limits.tcl
index 38ba76208..51122e8f5 100644
--- a/tests/unit/limits.tcl
+++ b/tests/unit/limits.tcl
@@ -1,4 +1,4 @@
-start_server {tags {"limits"} overrides {maxclients 10}} {
+start_server {tags {"limits network"} overrides {maxclients 10}} {
     if {$::tls} {
         set expected_code "*I/O error*"
     } else {
diff --git a/tests/unit/pause.tcl b/tests/unit/pause.tcl
index 9f5cfd607..67b684d36 100644
--- a/tests/unit/pause.tcl
+++ b/tests/unit/pause.tcl
@@ -1,4 +1,4 @@
-start_server {tags {"pause"}} {
+start_server {tags {"pause network"}} {
     test "Test read commands are not blocked by client pause" {
         r client PAUSE 100000000 WRITE
         set rd [redis_deferring_client]
diff --git a/tests/unit/protocol.tcl b/tests/unit/protocol.tcl
index 4dfdc6f59..442c23de6 100644
--- a/tests/unit/protocol.tcl
+++ b/tests/unit/protocol.tcl
@@ -1,4 +1,4 @@
-start_server {tags {"protocol"}} {
+start_server {tags {"protocol network"}} {
     test "Handle an empty query" {
         reconnect
         r write "\r\n"
diff --git a/tests/unit/pubsub.tcl b/tests/unit/pubsub.tcl
index 9c7a43bf0..966565ae1 100644
--- a/tests/unit/pubsub.tcl
+++ b/tests/unit/pubsub.tcl
@@ -1,4 +1,4 @@
-start_server {tags {"pubsub"}} {
+start_server {tags {"pubsub network"}} {
     proc __consume_subscribe_messages {client type channels} {
         set numsub -1
         set counts {}
diff --git a/tests/unit/scan.tcl b/tests/unit/scan.tcl
index 9f9ff4df2..3981a2234 100644
--- a/tests/unit/scan.tcl
+++ b/tests/unit/scan.tcl
@@ -1,4 +1,4 @@
-start_server {tags {"scan"}} {
+start_server {tags {"scan network"}} {
     test "SCAN basic" {
         r flushdb
         r debug populate 1000
diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index 88cf9dc42..7aaca47ca 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -1,4 +1,4 @@
-start_server {tags {"tracking"}} {
+start_server {tags {"tracking network"}} {
     # Create a deferred client we'll use to redirect invalidation
     # messages to.
     set rd_redirection [redis_deferring_client]
diff --git a/tests/unit/wait.tcl b/tests/unit/wait.tcl
index 0a4965c20..78c3d8202 100644
--- a/tests/unit/wait.tcl
+++ b/tests/unit/wait.tcl
@@ -1,6 +1,6 @@
 source tests/support/cli.tcl
 
-start_server {tags {"wait"}} {
+start_server {tags {"wait network"}} {
 start_server {} {
     set slave [srv 0 client]
     set slave_host [srv 0 host]

From 366a16ff0585fbc60df630b9ae238cfedeaaf81a Mon Sep 17 00:00:00 2001
From: Raghav Muddur <r@nmvk.com>
Date: Mon, 18 Jan 2021 01:37:05 -0800
Subject: [PATCH 0976/1098] Update getTimeZone to long (#8346)

---
 src/util.c | 4 ++--
 src/util.h | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/util.c b/src/util.c
index eca212e57..3243fa51e 100644
--- a/src/util.c
+++ b/src/util.c
@@ -749,7 +749,7 @@ sds getAbsolutePath(char *filename) {
  * Gets the proper timezone in a more portable fashion
  * i.e timezone variables are linux specific.
  */
-unsigned long getTimeZone(void) {
+long getTimeZone(void) {
 #if defined(__linux__) || defined(__sun)
     return timezone;
 #else
@@ -758,7 +758,7 @@ unsigned long getTimeZone(void) {
 
     gettimeofday(&tv, &tz);
 
-    return tz.tz_minuteswest * 60UL;
+    return tz.tz_minuteswest * 60L;
 #endif
 }
 
diff --git a/src/util.h b/src/util.h
index e9ad0ee4d..feaa82924 100644
--- a/src/util.h
+++ b/src/util.h
@@ -60,7 +60,7 @@ int string2d(const char *s, size_t slen, double *dp);
 int d2string(char *buf, size_t len, double value);
 int ld2string(char *buf, size_t len, long double value, ld2string_mode mode);
 sds getAbsolutePath(char *filename);
-unsigned long getTimeZone(void);
+long getTimeZone(void);
 int pathIsBaseName(char *path);
 
 #ifdef REDIS_TEST

From 5198b513d3001fb6399823909cac90ef8c3381e8 Mon Sep 17 00:00:00 2001
From: Andy Pan <panjf2000@gmail.com>
Date: Tue, 19 Jan 2021 17:36:21 +0800
Subject: [PATCH 0977/1098] Add a precheck before the actual call to fcntl
 (#8360)

Don't bother to call fcntl if the flags are not gonna be changed.
---
 src/anet.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/anet.c b/src/anet.c
index 7a0a1b1ed..4e2e6be88 100644
--- a/src/anet.c
+++ b/src/anet.c
@@ -69,6 +69,11 @@ int anetSetBlock(char *err, int fd, int non_block) {
         return ANET_ERR;
     }
 
+    /* Check if this flag has been set or unset, if so, 
+     * then there is no need to call fcntl to set/unset it again. */
+    if (!!(flags & O_NONBLOCK) == !!non_block)
+        return ANET_OK;
+
     if (non_block)
         flags |= O_NONBLOCK;
     else

From 4985c11bd6ef67a8eb58716f6c2b6b3d96e8538e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= <viktor.soderqvist@est.tech>
Date: Tue, 19 Jan 2021 12:15:33 +0100
Subject: [PATCH 0978/1098] Bugfix: Make modules blocked on keys unblock on
 commands like LPUSH (#8356)

This was a regression from #7625 (only in 6.2 RC2).

This makes it possible again to implement blocking list and zset
commands using the modules API.

This commit also includes a test case for the reverse: A module
unblocks a client blocked on BLPOP by inserting elements using
RedisModule_ListPush(). This already works, but it was untested.
---
 src/blocked.c                        | 14 ++++-
 tests/modules/blockonkeys.c          | 78 ++++++++++++++++++++++++++++
 tests/unit/moduleapi/blockonkeys.tcl | 30 +++++++++++
 3 files changed, 120 insertions(+), 2 deletions(-)

diff --git a/src/blocked.c b/src/blocked.c
index 46935c79f..1d69ff902 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -684,10 +684,20 @@ static int getBlockedTypeByType(int type) {
 void signalKeyAsReady(redisDb *db, robj *key, int type) {
     readyList *rl;
 
-    /* If no clients are blocked on this type, just return */
+    /* Quick returns. */
     int btype = getBlockedTypeByType(type);
-    if (btype == BLOCKED_NONE || !server.blocked_clients_by_type[btype])
+    if (btype == BLOCKED_NONE) {
+        /* The type can never block. */
         return;
+    }
+    if (!server.blocked_clients_by_type[btype] &&
+        !server.blocked_clients_by_type[BLOCKED_MODULE]) {
+        /* No clients block on this type. Note: Blocked modules are represented
+         * by BLOCKED_MODULE, even if the intention is to wake up by normal
+         * types (list, zset, stream), so we need to check that there are no
+         * blocked modules before we do a quick return here. */
+        return;
+    }
 
     /* No clients blocking for this key? No need to queue it. */
     if (dictFind(db->blocking_keys,key) == NULL) return;
diff --git a/tests/modules/blockonkeys.c b/tests/modules/blockonkeys.c
index 94f31d455..6f157342c 100644
--- a/tests/modules/blockonkeys.c
+++ b/tests/modules/blockonkeys.c
@@ -298,6 +298,76 @@ int fsl_getall(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     return REDISMODULE_OK;
 }
 
+/* Callback for blockonkeys_popall */
+int blockonkeys_popall_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argc);
+    RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE);
+    if (RedisModule_KeyType(key) == REDISMODULE_KEYTYPE_LIST) {
+        RedisModuleString *elem;
+        long len = 0;
+        RedisModule_ReplyWithArray(ctx, REDISMODULE_POSTPONED_ARRAY_LEN);
+        while ((elem = RedisModule_ListPop(key, REDISMODULE_LIST_HEAD)) != NULL) {
+            len++;
+            RedisModule_ReplyWithString(ctx, elem);
+            RedisModule_FreeString(ctx, elem);
+        }
+        RedisModule_ReplySetArrayLength(ctx, len);
+    } else {
+        RedisModule_ReplyWithError(ctx, "ERR Not a list");
+    }
+    RedisModule_CloseKey(key);
+    return REDISMODULE_OK;
+}
+
+int blockonkeys_popall_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+    return RedisModule_ReplyWithError(ctx, "ERR Timeout");
+}
+
+/* BLOCKONKEYS.POPALL key
+ *
+ * Blocks on an empty key for up to 3 seconds. When unblocked by a list
+ * operation like LPUSH, all the elements are popped and returned. Fails with an
+ * error on timeout. */
+int blockonkeys_popall(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 2)
+        return RedisModule_WrongArity(ctx);
+
+    RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_READ);
+    if (RedisModule_KeyType(key) == REDISMODULE_KEYTYPE_EMPTY) {
+        RedisModule_BlockClientOnKeys(ctx, blockonkeys_popall_reply_callback,
+                                      blockonkeys_popall_timeout_callback,
+                                      NULL, 3000, &argv[1], 1, NULL);
+    } else {
+        RedisModule_ReplyWithError(ctx, "ERR Key not empty");
+    }
+    RedisModule_CloseKey(key);
+    return REDISMODULE_OK;
+}
+
+/* A module equivalent of LPUSH */
+int blockonkeys_lpush(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc < 3)
+        return RedisModule_WrongArity(ctx);
+
+    RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE);
+    if (RedisModule_KeyType(key) != REDISMODULE_KEYTYPE_EMPTY &&
+        RedisModule_KeyType(key) != REDISMODULE_KEYTYPE_LIST) {
+        RedisModule_ReplyWithError(ctx, REDISMODULE_ERRORMSG_WRONGTYPE);
+    } else {
+        for (int i = 2; i < argc; i++) {
+            if (RedisModule_ListPush(key, REDISMODULE_LIST_HEAD,
+                                     argv[i]) != REDISMODULE_OK) {
+                RedisModule_CloseKey(key);
+                return RedisModule_ReplyWithError(ctx, "ERR Push failed");
+            }
+        }
+    }
+    RedisModule_CloseKey(key);
+    return RedisModule_ReplyWithSimpleString(ctx, "OK");
+}
+
 int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
@@ -334,5 +404,13 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     if (RedisModule_CreateCommand(ctx,"fsl.getall",fsl_getall,"",0,0,0) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
 
+    if (RedisModule_CreateCommand(ctx, "blockonkeys.popall", blockonkeys_popall,
+                                  "", 1, 1, 1) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx, "blockonkeys.lpush", blockonkeys_lpush,
+                                  "", 1, 1, 1) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
     return REDISMODULE_OK;
 }
diff --git a/tests/unit/moduleapi/blockonkeys.tcl b/tests/unit/moduleapi/blockonkeys.tcl
index 5e5d93da3..0f6612164 100644
--- a/tests/unit/moduleapi/blockonkeys.tcl
+++ b/tests/unit/moduleapi/blockonkeys.tcl
@@ -185,4 +185,34 @@ start_server {tags {"modules"}} {
         r fsl.push k 34
         assert_equal {34} [$rd read]
     }
+
+    test {Module client blocked on keys woken up by LPUSH} {
+        r del k
+        set rd [redis_deferring_client]
+        $rd blockonkeys.popall k
+        # wait until client is actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Client is not blocked"
+        }
+        r lpush k 42 squirrel banana
+        assert_equal {banana squirrel 42} [$rd read]
+        $rd close
+    }
+
+    test {Module client unblocks BLPOP} {
+        r del k
+        set rd [redis_deferring_client]
+        $rd blpop k 3
+        # wait until client is actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Client is not blocked"
+        }
+        r blockonkeys.lpush k 42
+        assert_equal {k 42} [$rd read]
+        $rd close
+    }
 }

From d42ea9e88ac5a9cc03e69be905bae81e88327d35 Mon Sep 17 00:00:00 2001
From: filipe oliveira <filipecosta.90@gmail.com>
Date: Tue, 19 Jan 2021 15:24:49 +0000
Subject: [PATCH 0979/1098] Removing unnecessary runtime tio_debug checks
 (#8250)

These statements were dead code.
---
 src/networking.c | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index f1c606236..6d5973563 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -3355,8 +3355,6 @@ void processEventsWhileBlocked(void) {
  * Threaded I/O
  * ========================================================================== */
 
-int tio_debug = 0;
-
 #define IO_THREADS_MAX_NUM 128
 #define IO_THREADS_OP_READ 0
 #define IO_THREADS_OP_WRITE 1
@@ -3407,8 +3405,6 @@ void *IOThreadMain(void *myid) {
 
         serverAssert(getIOPendingCount(id) != 0);
 
-        if (tio_debug) printf("[%ld] %d to handle\n", id, (int)listLength(io_threads_list[id]));
-
         /* Process: note that the main thread will never touch our list
          * before we drop the pending count to 0. */
         listIter li;
@@ -3426,8 +3422,6 @@ void *IOThreadMain(void *myid) {
         }
         listEmpty(io_threads_list[id]);
         setIOPendingCount(id, 0);
-
-        if (tio_debug) printf("[%ld] Done\n", id);
     }
 }
 
@@ -3482,8 +3476,6 @@ void killIOThreads(void) {
 }
 
 void startThreadedIO(void) {
-    if (tio_debug) { printf("S"); fflush(stdout); }
-    if (tio_debug) printf("--- STARTING THREADED IO ---\n");
     serverAssert(server.io_threads_active == 0);
     for (int j = 1; j < server.io_threads_num; j++)
         pthread_mutex_unlock(&io_threads_mutex[j]);
@@ -3494,10 +3486,6 @@ void stopThreadedIO(void) {
     /* We may have still clients with pending reads when this function
      * is called: handle them before stopping the threads. */
     handleClientsWithPendingReadsUsingThreads();
-    if (tio_debug) { printf("E"); fflush(stdout); }
-    if (tio_debug) printf("--- STOPPING THREADED IO [R%d] [W%d] ---\n",
-        (int) listLength(server.clients_pending_read),
-        (int) listLength(server.clients_pending_write));
     serverAssert(server.io_threads_active == 1);
     for (int j = 1; j < server.io_threads_num; j++)
         pthread_mutex_lock(&io_threads_mutex[j]);
@@ -3540,8 +3528,6 @@ int handleClientsWithPendingWritesUsingThreads(void) {
     /* Start threads if needed. */
     if (!server.io_threads_active) startThreadedIO();
 
-    if (tio_debug) printf("%d TOTAL WRITE pending clients\n", processed);
-
     /* Distribute the clients across N different lists. */
     listIter li;
     listNode *ln;
@@ -3586,7 +3572,6 @@ int handleClientsWithPendingWritesUsingThreads(void) {
             pending += getIOPendingCount(j);
         if (pending == 0) break;
     }
-    if (tio_debug) printf("I/O WRITE All threads finshed\n");
 
     /* Run the list of clients again to install the write handler where
      * needed. */
@@ -3639,8 +3624,6 @@ int handleClientsWithPendingReadsUsingThreads(void) {
     int processed = listLength(server.clients_pending_read);
     if (processed == 0) return 0;
 
-    if (tio_debug) printf("%d TOTAL READ pending clients\n", processed);
-
     /* Distribute the clients across N different lists. */
     listIter li;
     listNode *ln;
@@ -3676,7 +3659,6 @@ int handleClientsWithPendingReadsUsingThreads(void) {
             pending += getIOPendingCount(j);
         if (pending == 0) break;
     }
-    if (tio_debug) printf("I/O READ All threads finshed\n");
 
     /* Run the list of clients again to process the new buffers. */
     while(listLength(server.clients_pending_read)) {

From a29aec9abb8b4ffd030be9e4c8875ddd8ec1357c Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 19 Jan 2021 18:49:26 +0200
Subject: [PATCH 0980/1098] Add tests to make sure that relative EXPIRE is
 propagated to replicas (#8357)

This commit adds tests to make sure that relative and absolute expire commands
are propagated as is to replicas and stop any future attempt to change that without
a proper discussion. see #8327 and #5171

Additionally it slightly improve the AOF test that tests the opposite (always
propagating absolute times), by covering more commands, and shaving 2
seconds from the test time.
---
 tests/unit/expire.tcl | 71 +++++++++++++++++++++++++++++++++++++------
 1 file changed, 61 insertions(+), 10 deletions(-)

diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl
index 8bcdc16b7..e71018073 100644
--- a/tests/unit/expire.tcl
+++ b/tests/unit/expire.tcl
@@ -209,19 +209,70 @@ start_server {tags {"expire"}} {
         set e
     } {*not an integer*}
 
-    test {SET - use EX/PX option, TTL should not be reseted after loadaof} {
-        r config set appendonly yes
-        r set foo bar EX 100
-        after 2000
-        r debug loadaof
-        set ttl [r ttl foo]
-        assert {$ttl <= 98 && $ttl > 90}
+    test {EXPIRE and SET EX/PX option, TTL should not be reset after loadaof} {
+        # This test makes sure that expire times are propagated as absolute
+        # times to the AOF file and not as relative time, so that when the AOF
+        # is reloaded the TTLs are not being shifted forward to the future.
+        # We want the time to logically pass when the server is restarted!
 
-        r set foo bar PX 100000
+        r config set appendonly yes
+        r set foo1 bar EX 100
+        r set foo2 bar PX 100000
+        r set foo3 bar
+        r set foo4 bar
+        r expire foo3 100
+        r pexpire foo4 100000
+        r setex foo5 100 bar
+        r psetex foo6 100000 bar
         after 2000
         r debug loadaof
-        set ttl [r ttl foo]
-        assert {$ttl <= 98 && $ttl > 90}
+        assert_range [r ttl foo1] 90 98
+        assert_range [r ttl foo2] 90 98
+        assert_range [r ttl foo3] 90 98
+        assert_range [r ttl foo4] 90 98
+        assert_range [r ttl foo5] 90 98
+        assert_range [r ttl foo6] 90 98
+    }
+
+    test {EXPIRE relative and absolute propagation to replicas} {
+        # Make sure that relative and absolute expire commands are propagated
+        # "as is" to replicas.
+        # We want replicas to honor the same high level contract of expires that
+        # the master has, that is, we want the time to be counted logically
+        # starting from the moment the write was received. This usually provides
+        # the most coherent behavior from the point of view of the external
+        # users, with TTLs that are similar from the POV of the external observer.
+        #
+        # This test is here to stop some innocent / eager optimization or cleanup
+        # from doing the wrong thing without proper discussion, see:
+        # https://github.com/redis/redis/pull/5171#issuecomment-409553266
+
+        set repl [attach_to_replication_stream]
+        r set foo1 bar ex 100
+        r set foo1 bar px 100000
+        r setex foo1 100 bar
+        r psetex foo1 100000 bar
+        r set foo2 bar
+        r expire foo2 100
+        r pexpire foo2 100000
+        r set foo3 bar
+        r expireat foo3 [expr [clock seconds]+100]
+        r pexpireat foo3 [expr [clock seconds]*1000+100000]
+        r expireat foo3 [expr [clock seconds]-100]
+        assert_replication_stream $repl {
+            {select *}
+            {set foo1 bar ex 100}
+            {set foo1 bar px 100000}
+            {setex foo1 100 bar}
+            {psetex foo1 100000 bar}
+            {set foo2 bar}
+            {expire foo2 100}
+            {pexpire foo2 100000}
+            {set foo3 bar}
+            {expireat foo3 *}
+            {pexpireat foo3 *}
+            {del foo3}
+        }
     }
 
     test {SET command will remove expire} {

From aaf71b380ed5ef8d5d63f8a60733c35202c5b838 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Tue, 19 Jan 2021 20:31:30 +0200
Subject: [PATCH 0981/1098] Fix description of the save conf directive (#8337)

The line that said:

Note: you can disable saving completely by commenting out all "save" lines

was incorrect.

Also, note that the save values used in the default redis.conf file are different than the server's defaults, so this PR also addresses this and introduces a potential behavior change for some users.
---
 redis.conf | 45 ++++++++++++++++++++++-----------------------
 1 file changed, 22 insertions(+), 23 deletions(-)

diff --git a/redis.conf b/redis.conf
index 5f9eed7fe..bb0d70644 100644
--- a/redis.conf
+++ b/redis.conf
@@ -326,30 +326,29 @@ databases 16
 always-show-logo no
 
 ################################ SNAPSHOTTING  ################################
-#
-# Save the DB on disk:
-#
-#   save <seconds> <changes>
-#
-#   Will save the DB if both the given number of seconds and the given
-#   number of write operations against the DB occurred.
-#
-#   In the example below the behavior will be to save:
-#   after 900 sec (15 min) if at least 1 key changed
-#   after 300 sec (5 min) if at least 10 keys changed
-#   after 60 sec if at least 10000 keys changed
-#
-#   Note: you can disable saving completely by commenting out all "save" lines.
-#
-#   It is also possible to remove all the previously configured save
-#   points by adding a save directive with a single empty string argument
-#   like in the following example:
-#
-#   save ""
 
-save 900 1
-save 300 10
-save 60 10000
+# Save the DB to disk.
+#
+# save <seconds> <changes>
+#
+# Redis will save the DB if both the given number of seconds and the given
+# number of write operations against the DB occurred.
+#
+# Snapshotting can be completely disabled with a single empty string argument
+# as in following example:
+#
+# save ""
+#
+# Unless specified otherwise, by default Redis will save the DB:
+#   * After 3600 seconds (an hour) if at least 1 key changed
+#   * After 300 seconds (5 minutes) if at least 10 keys changed
+#   * After 60 seconds if at least 10000 keys changed
+#
+# You can set these explicitly by uncommenting the three following lines.
+#
+# save 3600 1
+# save 300 100
+# save 60 10000
 
 # By default Redis will stop accepting writes if RDB snapshots are enabled
 # (at least one save point) and the latest background save failed.

From fb66e2e24943018961321d13e46ee2ab66de882a Mon Sep 17 00:00:00 2001
From: Andy Pan <panjf2000@gmail.com>
Date: Wed, 20 Jan 2021 04:57:30 +0800
Subject: [PATCH 0982/1098] Use FD_CLOEXEC in Sentinel, so that FDs don't leak
 to the scripts it runs (#8242)

Sentinel uses execve to run scripts, so it needs to use FD_CLOEXEC
on all file descriptors, so that they're not accessible by the script it runs.

This commit includes a change to the sentinel tests, which verifies no
FDs are left opened when the script is executed.
---
 src/ae.c                                     |  1 +
 src/ae_epoll.c                               |  1 +
 src/ae_evport.c                              |  1 +
 src/ae_kqueue.c                              |  1 +
 src/anet.c                                   | 23 ++++++++++++++++++++
 src/anet.h                                   |  1 +
 src/cluster.c                                |  2 +-
 src/module.c                                 |  5 +++++
 src/networking.c                             |  3 +++
 src/sentinel.c                               |  3 ++-
 src/server.c                                 |  6 +++--
 tests/instances.tcl                          | 14 +++++++++++-
 tests/sentinel/run.tcl                       |  2 +-
 tests/sentinel/tests/includes/init-tests.tcl |  2 ++
 tests/sentinel/tests/includes/notify.sh      | 20 +++++++++++++++++
 15 files changed, 79 insertions(+), 6 deletions(-)
 create mode 100755 tests/sentinel/tests/includes/notify.sh

diff --git a/src/ae.c b/src/ae.c
index 1c3a4e091..283f51438 100644
--- a/src/ae.c
+++ b/src/ae.c
@@ -31,6 +31,7 @@
  */
 
 #include "ae.h"
+#include "anet.h"
 
 #include <stdio.h>
 #include <sys/time.h>
diff --git a/src/ae_epoll.c b/src/ae_epoll.c
index fa197297e..07ca8ca41 100644
--- a/src/ae_epoll.c
+++ b/src/ae_epoll.c
@@ -51,6 +51,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) {
         zfree(state);
         return -1;
     }
+    anetCloexec(state->epfd);
     eventLoop->apidata = state;
     return 0;
 }
diff --git a/src/ae_evport.c b/src/ae_evport.c
index 4e254b602..7a0b03aea 100644
--- a/src/ae_evport.c
+++ b/src/ae_evport.c
@@ -82,6 +82,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) {
         zfree(state);
         return -1;
     }
+    anetCloexec(state->portfd);
 
     state->npending = 0;
 
diff --git a/src/ae_kqueue.c b/src/ae_kqueue.c
index 6796f4ceb..b146f2519 100644
--- a/src/ae_kqueue.c
+++ b/src/ae_kqueue.c
@@ -53,6 +53,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) {
         zfree(state);
         return -1;
     }
+    anetCloexec(state->kqfd);
     eventLoop->apidata = state;
     return 0;
 }
diff --git a/src/anet.c b/src/anet.c
index 4e2e6be88..f2c39b200 100644
--- a/src/anet.c
+++ b/src/anet.c
@@ -94,6 +94,29 @@ int anetBlock(char *err, int fd) {
     return anetSetBlock(err,fd,0);
 }
 
+/* Enable the FD_CLOEXEC on the given fd to avoid fd leaks. 
+ * This function should be invoked for fd's on specific places 
+ * where fork + execve system calls are called. */
+int anetCloexec(int fd) {
+    int r;
+    int flags;
+
+    do {
+        r = fcntl(fd, F_GETFD);
+    } while (r == -1 && errno == EINTR);
+
+    if (r == -1 || (r & FD_CLOEXEC))
+        return r;
+
+    flags = r | FD_CLOEXEC;
+
+    do {
+        r = fcntl(fd, F_SETFD, flags);
+    } while (r == -1 && errno == EINTR);
+
+    return r;
+}
+
 /* Set TCP keep alive option to detect dead peers. The interval option
  * is only used for Linux as we are using Linux-specific APIs to set
  * the probe send time, interval, and count. */
diff --git a/src/anet.h b/src/anet.h
index fbf41cd17..dc3cbeb32 100644
--- a/src/anet.h
+++ b/src/anet.h
@@ -70,6 +70,7 @@ int anetUnixAccept(char *err, int serversock);
 int anetWrite(int fd, char *buf, int count);
 int anetNonBlock(char *err, int fd);
 int anetBlock(char *err, int fd);
+int anetCloexec(int fd);
 int anetEnableTcpNoDelay(char *err, int fd);
 int anetDisableTcpNoDelay(char *err, int fd);
 int anetTcpKeepAlive(char *err, int fd);
diff --git a/src/cluster.c b/src/cluster.c
index 2a06ef6dc..2b5acec90 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -398,7 +398,7 @@ int clusterLockConfig(char *filename) {
     /* To lock it, we need to open the file in a way it is created if
      * it does not exist, otherwise there is a race condition with other
      * processes. */
-    int fd = open(filename,O_WRONLY|O_CREAT,0644);
+    int fd = open(filename,O_WRONLY|O_CREAT|O_CLOEXEC,0644);
     if (fd == -1) {
         serverLog(LL_WARNING,
             "Can't open %s in order to acquire a lock: %s",
diff --git a/src/module.c b/src/module.c
index fd3f90d84..a6a1c1df7 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7670,6 +7670,11 @@ void moduleInitModulesSystem(void) {
     anetNonBlock(NULL,server.module_blocked_pipe[0]);
     anetNonBlock(NULL,server.module_blocked_pipe[1]);
 
+    /* Enable close-on-exec flag on pipes in case of the fork-exec system calls in
+     * sentinels or redis servers. */
+    anetCloexec(server.module_blocked_pipe[0]);
+    anetCloexec(server.module_blocked_pipe[1]);
+
     /* Create the timers radix tree. */
     Timers = raxNew();
 
diff --git a/src/networking.c b/src/networking.c
index 6d5973563..26e4017c5 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -1104,6 +1104,7 @@ void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
                     "Accepting client connection: %s", server.neterr);
             return;
         }
+        anetCloexec(cfd);
         serverLog(LL_VERBOSE,"Accepted %s:%d", cip, cport);
         acceptCommonHandler(connCreateAcceptedSocket(cfd),0,cip);
     }
@@ -1124,6 +1125,7 @@ void acceptTLSHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
                     "Accepting client connection: %s", server.neterr);
             return;
         }
+        anetCloexec(cfd);
         serverLog(LL_VERBOSE,"Accepted %s:%d", cip, cport);
         acceptCommonHandler(connCreateAcceptedTLS(cfd, server.tls_auth_clients),0,cip);
     }
@@ -1143,6 +1145,7 @@ void acceptUnixHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
                     "Accepting client connection: %s", server.neterr);
             return;
         }
+        anetCloexec(cfd);
         serverLog(LL_VERBOSE,"Accepted connection to %s", server.unixsocket);
         acceptCommonHandler(connCreateAcceptedSocket(cfd),CLIENT_UNIX_SOCKET,NULL);
     }
diff --git a/src/sentinel.c b/src/sentinel.c
index 089282b71..26c129385 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -2135,6 +2135,7 @@ void sentinelReconnectInstance(sentinelRedisInstance *ri) {
                 link->cc->errstr);
             instanceLinkCloseConnection(link,link->cc);
         } else {
+            anetCloexec(link->cc->c.fd);
             link->pending_commands = 0;
             link->cc_conn_time = mstime();
             link->cc->data = link;
@@ -2162,7 +2163,7 @@ void sentinelReconnectInstance(sentinelRedisInstance *ri) {
             instanceLinkCloseConnection(link,link->pc);
         } else {
             int retval;
-
+            anetCloexec(link->pc->c.fd);
             link->pc_conn_time = mstime();
             link->pc->data = link;
             redisAeAttach(server.el,link->pc);
diff --git a/src/server.c b/src/server.c
index 90b9045ea..f3d75ce04 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2957,6 +2957,7 @@ int listenToPort(int port, int *fds, int *count) {
             return C_ERR;
         }
         anetNonBlock(NULL,fds[*count]);
+        anetCloexec(fds[*count]);
         (*count)++;
     }
     return C_OK;
@@ -3095,6 +3096,7 @@ void initServer(void) {
             exit(1);
         }
         anetNonBlock(NULL,server.sofd);
+        anetCloexec(server.sofd);
     }
 
     /* Abort if there are no listening sockets at all. */
@@ -5470,7 +5472,7 @@ void setupChildSignalHandlers(void) {
  * of the parent process, e.g. fd(socket or flock) etc.
  * should close the resources not used by the child process, so that if the
  * parent restarts it can bind/lock despite the child possibly still running. */
-void closeClildUnusedResourceAfterFork() {
+void closeChildUnusedResourceAfterFork() {
     closeListeningSockets(0);
     if (server.cluster_enabled && server.cluster_config_file_lock_fd != -1)
         close(server.cluster_config_file_lock_fd);  /* don't care if this fails */
@@ -5497,7 +5499,7 @@ int redisFork(int purpose) {
         server.in_fork_child = purpose;
         setOOMScoreAdj(CONFIG_OOM_BGCHILD);
         setupChildSignalHandlers();
-        closeClildUnusedResourceAfterFork();
+        closeChildUnusedResourceAfterFork();
     } else {
         /* Parent */
         server.stat_total_forks++;
diff --git a/tests/instances.tcl b/tests/instances.tcl
index 18eb1a402..5b25bcc97 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -400,6 +400,11 @@ proc check_leaks instance_types {
 
 # Execute all the units inside the 'tests' directory.
 proc run_tests {} {
+    set sentinel_fd_leaks_file "sentinel_fd_leaks"
+    if { [file exists $sentinel_fd_leaks_file] } {
+        file delete $sentinel_fd_leaks_file
+    }
+
     set tests [lsort [glob ../tests/*]]
     foreach test $tests {
         if {$::run_matching ne {} && [string match $::run_matching $test] == 0} {
@@ -414,7 +419,14 @@ proc run_tests {} {
 
 # Print a message and exists with 0 / 1 according to zero or more failures.
 proc end_tests {} {
-    if {$::failed == 0} {
+    set sentinel_fd_leaks_file "sentinel_fd_leaks"
+    if { [file exists $sentinel_fd_leaks_file] } {
+        puts [colorstr red "WARNING: sentinel test(s) failed, there are leaked fds in sentinel:"] 
+        exec cat $sentinel_fd_leaks_file
+        exit 1
+    }
+
+    if {$::failed == 0 } {
         puts "GOOD! No errors."
         exit 0
     } else {
diff --git a/tests/sentinel/run.tcl b/tests/sentinel/run.tcl
index 996af906a..99362c193 100644
--- a/tests/sentinel/run.tcl
+++ b/tests/sentinel/run.tcl
@@ -10,7 +10,7 @@ set ::tlsdir "../../tls"
 
 proc main {} {
     parse_options
-    spawn_instance sentinel $::sentinel_base_port $::instances_count
+    spawn_instance sentinel $::sentinel_base_port $::instances_count [list "sentinel deny-scripts-reconfig no"]
     spawn_instance redis $::redis_base_port $::instances_count
     run_tests
     cleanup
diff --git a/tests/sentinel/tests/includes/init-tests.tcl b/tests/sentinel/tests/includes/init-tests.tcl
index 234f9c589..d6796fda6 100644
--- a/tests/sentinel/tests/includes/init-tests.tcl
+++ b/tests/sentinel/tests/includes/init-tests.tcl
@@ -37,6 +37,8 @@ test "(init) Sentinels can start monitoring a master" {
         S $id SENTINEL SET mymaster down-after-milliseconds 2000
         S $id SENTINEL SET mymaster failover-timeout 20000
         S $id SENTINEL SET mymaster parallel-syncs 10
+        S $id SENTINEL SET mymaster notification-script ../../tests/includes/notify.sh
+        S $id SENTINEL SET mymaster client-reconfig-script ../../tests/includes/notify.sh
     }
 }
 
diff --git a/tests/sentinel/tests/includes/notify.sh b/tests/sentinel/tests/includes/notify.sh
new file mode 100755
index 000000000..09a8addca
--- /dev/null
+++ b/tests/sentinel/tests/includes/notify.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+OS=`uname -s`
+if [ ${OS} == "Linux" ]
+then
+    exit 0
+fi
+
+# fd 3 is meant to catch the actual access to /proc/pid/fd, 
+# in case there's an fd leak by the sentinel,
+# it can take 3, but then the access to /proc will take another fd, and we'll catch that.
+leaked_fd_count=`ls /proc/self/fd | grep -vE '^[0|1|2|3]$' | wc -l`
+if [ $leaked_fd_count -gt 0 ]
+then
+    sentinel_fd_leaks_file="../sentinel_fd_leaks"
+    if [ ! -f $sentinel_fd_leaks_file ]
+    then
+        ls -l /proc/self/fd | cat >> $sentinel_fd_leaks_file
+    fi
+fi

From 1be29606c56bc747a66c99d6cf31110df51b65b4 Mon Sep 17 00:00:00 2001
From: Andy Pan <panjf2000@gmail.com>
Date: Wed, 20 Jan 2021 15:58:02 +0800
Subject: [PATCH 0983/1098] Fix sentinel FD leak test, not printing the list of
 leaks (#8363)

---
 tests/instances.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/instances.tcl b/tests/instances.tcl
index 5b25bcc97..4e583eeb3 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -422,7 +422,7 @@ proc end_tests {} {
     set sentinel_fd_leaks_file "sentinel_fd_leaks"
     if { [file exists $sentinel_fd_leaks_file] } {
         puts [colorstr red "WARNING: sentinel test(s) failed, there are leaked fds in sentinel:"] 
-        exec cat $sentinel_fd_leaks_file
+        puts [exec cat $sentinel_fd_leaks_file]
         exit 1
     }
 

From 6401920d705aedbfc43409f27c8d8e89bfb0e756 Mon Sep 17 00:00:00 2001
From: Andy Pan <panjf2000@gmail.com>
Date: Wed, 20 Jan 2021 16:17:20 +0800
Subject: [PATCH 0984/1098] Fix sentinel FD leak test, checking the wrong OS
 name (#8364)

---
 tests/sentinel/tests/includes/notify.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/sentinel/tests/includes/notify.sh b/tests/sentinel/tests/includes/notify.sh
index 09a8addca..2365f1d3a 100755
--- a/tests/sentinel/tests/includes/notify.sh
+++ b/tests/sentinel/tests/includes/notify.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 OS=`uname -s`
-if [ ${OS} == "Linux" ]
+if [ ${OS} != "Linux" ]
 then
     exit 0
 fi

From 16258f21d1d0e84e0cc4b5b0f93c78be6384658d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= <viktor.soderqvist@est.tech>
Date: Wed, 20 Jan 2021 10:47:06 +0100
Subject: [PATCH 0985/1098] More modules API ref formatting fixes (#8344)

Fix broken formatting in `RM_Call` and `RM_CreateDataType`,
`RM_SubscribeToServerEvent` (nested lists, etc. in list items).

Unhide docs of `RM_LoadDataTypeFromString` and
`RM_SaveDataTypeToString` by removing blank line between docs and
function.

Clarification added to `RM__Assert`: Recommentation to use the
`RedisModule_Assert` macro instead.

All names containing underscores (variable and macro names) are
wrapped in backticks (if not already wrapped in backticks). This
prevents underscore from being interpreted as italics in some
cases.

Names including a wildcard star, e.g. RM_Defrag*(), is wrapped in
backticks (and RM replaced by RedisModule in this case). This
prevents the * from being interpreted as an italics marker.

A list item with a sublist, a paragraph and another sublist is a
combination which seems impossible to achieve with RedCarped
markdown, so the one occurrence of this is rewritten.

Various trivial changes (typos, backticks, etc.).

Ruby script:

* Replace `RM_Xyz` with `RedisModule_Xyz` in docs. (RM is correct
  when refering to the C code but RedisModule is correct in the
  API docs.)
* Automatic backquotes around C functions like `malloc()`.
* Turn URLs into links. The link text is the URL itself.
* Don't add backticks inside bold (**...**)
---
 src/module.c          | 341 +++++++++++++++++++++---------------------
 src/modules/gendoc.rb |  18 ++-
 2 files changed, 183 insertions(+), 176 deletions(-)

diff --git a/src/module.c b/src/module.c
index a6a1c1df7..56a3ef98a 100644
--- a/src/module.c
+++ b/src/module.c
@@ -478,8 +478,8 @@ void *RM_PoolAlloc(RedisModuleCtx *ctx, size_t bytes) {
  * Helpers for modules API implementation
  * -------------------------------------------------------------------------- */
 
-/* Create an empty key of the specified type. 'kp' must point to a key object
- * opened for writing where the .value member is set to NULL because the
+/* Create an empty key of the specified type. `key` must point to a key object
+ * opened for writing where the `.value` member is set to NULL because the
  * key was found to be non existing.
  *
  * On success REDISMODULE_OK is returned and the key is populated with
@@ -1322,7 +1322,7 @@ int RM_StringAppendBuffer(RedisModuleCtx *ctx, RedisModuleString *str, const cha
  * -------------------------------------------------------------------------- */
 
 /* Send an error about the number of arguments given to the command,
- * citing the command name in the error message.
+ * citing the command name in the error message. Returns REDISMODULE_OK.
  *
  * Example:
  *
@@ -1394,7 +1394,7 @@ int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) {
     return REDISMODULE_OK;
 }
 
-/* Reply with a simple string (+... \r\n in RESP protocol). This replies
+/* Reply with a simple string (`+... \r\n` in RESP protocol). This replies
  * are suitable only when sending a small non-binary string with small
  * overhead, like "OK" or similar replies.
  *
@@ -3348,20 +3348,19 @@ fmterr:
  *
  * * **cmdname**: The Redis command to call.
  * * **fmt**: A format specifier string for the command's arguments. Each
- *   of the arguments should be specified by a valid type specification:
+ *   of the arguments should be specified by a valid type specification. The
+ *   format specifier can also contain the modifiers `!`, `A` and `R` which
+ *   don't have a corresponding argument.
  *
- *   * b: The argument is a buffer and is immediately followed by another
- *        argument that is the buffer's length.
- *   * c: The argument is a pointer to a plain C string (null-terminated).
- *   * l: The argument is long long integer.
- *   * s: The argument is a RedisModuleString.
- *   * v: The argument(s) is a vector of RedisModuleString.
- *
- *   The format specifier can also include modifiers:
- *
- *   * !: Sends the Redis command and its arguments to replicas and AOF.
- *   * A: Suppress AOF propagation, send only to replicas (requires `!`).
- *   * R: Suppress replicas propagation, send only to AOF (requires `!`).
+ *     * `b` -- The argument is a buffer and is immediately followed by another
+ *              argument that is the buffer's length.
+ *     * `c` -- The argument is a pointer to a plain C string (null-terminated).
+ *     * `l` -- The argument is long long integer.
+ *     * `s` -- The argument is a RedisModuleString.
+ *     * `v` -- The argument(s) is a vector of RedisModuleString.
+ *     * `!` -- Sends the Redis command and its arguments to replicas and AOF.
+ *     * `A` -- Suppress AOF propagation, send only to replicas (requires `!`).
+ *     * `R` -- Suppress replicas propagation, send only to AOF (requires `!`).
  * * **...**: The actual arguments to the Redis command.
  *
  * On success a RedisModuleCallReply object is returned, otherwise
@@ -3685,27 +3684,28 @@ robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, robj *value) {
  *   still load old data produced by an older version if the rdb_load
  *   callback is able to check the encver value and act accordingly.
  *   The encver must be a positive value between 0 and 1023.
+ *
  * * **typemethods_ptr** is a pointer to a RedisModuleTypeMethods structure
  *   that should be populated with the methods callbacks and structure
  *   version, like in the following example:
  *
- *       RedisModuleTypeMethods tm = {
- *          .version = REDISMODULE_TYPE_METHOD_VERSION,
- *          .rdb_load = myType_RDBLoadCallBack,
- *          .rdb_save = myType_RDBSaveCallBack,
- *          .aof_rewrite = myType_AOFRewriteCallBack,
- *          .free = myType_FreeCallBack,
+ *         RedisModuleTypeMethods tm = {
+ *             .version = REDISMODULE_TYPE_METHOD_VERSION,
+ *             .rdb_load = myType_RDBLoadCallBack,
+ *             .rdb_save = myType_RDBSaveCallBack,
+ *             .aof_rewrite = myType_AOFRewriteCallBack,
+ *             .free = myType_FreeCallBack,
  *
- *          // Optional fields
- *          .digest = myType_DigestCallBack,
- *          .mem_usage = myType_MemUsageCallBack,
- *          .aux_load = myType_AuxRDBLoadCallBack,
- *          .aux_save = myType_AuxRDBSaveCallBack,
- *          .free_effort = myType_FreeEffortCallBack,
- *          .unlink = myType_UnlinkCallBack,
- *          .copy = myType_CopyCallback,
- *          .defrag = myType_DefragCallback
- *       }
+ *             // Optional fields
+ *             .digest = myType_DigestCallBack,
+ *             .mem_usage = myType_MemUsageCallBack,
+ *             .aux_load = myType_AuxRDBLoadCallBack,
+ *             .aux_save = myType_AuxRDBSaveCallBack,
+ *             .free_effort = myType_FreeEffortCallBack,
+ *             .unlink = myType_UnlinkCallBack,
+ *             .copy = myType_CopyCallback,
+ *             .defrag = myType_DefragCallback
+ *         }
  *
  * * **rdb_load**: A callback function pointer that loads data from RDB files.
  * * **rdb_save**: A callback function pointer that saves data to RDB files.
@@ -3743,7 +3743,7 @@ robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, robj *value) {
  *   a time limit and provides cursor support is used only for keys that are determined
  *   to have significant internal complexity. To determine this, the defrag mechanism
  *   uses the free_effort callback and the 'active-defrag-max-scan-fields' config directive.
- *   NOTE: The value is passed as a void** and the function is expected to update the
+ *   NOTE: The value is passed as a `void**` and the function is expected to update the
  *   pointer if the top-level value pointer is defragmented and consequentially changes.
  *
  * Note: the module name "AAAAAAAAA" is reserved and produces an error, it
@@ -3903,7 +3903,7 @@ int moduleAllDatatypesHandleErrors() {
 }
 
 /* Returns true if any previous IO API failed.
- * for Load* APIs the REDISMODULE_OPTIONS_HANDLE_IO_ERRORS flag must be set with
+ * for `Load*` APIs the REDISMODULE_OPTIONS_HANDLE_IO_ERRORS flag must be set with
  * RedisModule_SetModuleOptions first. */
 int RM_IsIOError(RedisModuleIO *io) {
     return io->error;
@@ -3929,7 +3929,7 @@ saveerr:
 }
 
 /* Load an unsigned 64 bit value from the RDB file. This function should only
- * be called in the context of the rdb_load method of modules implementing
+ * be called in the context of the `rdb_load` method of modules implementing
  * new data types. */
 uint64_t RM_LoadUnsigned(RedisModuleIO *io) {
     if (io->error) return 0;
@@ -4245,7 +4245,6 @@ void RM_DigestEndSequence(RedisModuleDigest *md) {
  * If this is NOT done, Redis will handle corrupted (or just truncated) serialized
  * data by producing an error message and terminating the process.
  */
-
 void *RM_LoadDataTypeFromString(const RedisModuleString *str, const moduleType *mt) {
     rio payload;
     RedisModuleIO io;
@@ -4273,7 +4272,6 @@ void *RM_LoadDataTypeFromString(const RedisModuleString *str, const moduleType *
  * implement in order to allow a module to arbitrarily serialize/de-serialize
  * keys, similar to how the Redis 'DUMP' and 'RESTORE' commands are implemented.
  */
-
 RedisModuleString *RM_SaveDataTypeToString(RedisModuleCtx *ctx, void *data, const moduleType *mt) {
     rio payload;
     RedisModuleIO io;
@@ -4371,7 +4369,7 @@ const RedisModuleString *RM_GetKeyNameFromIO(RedisModuleIO *io) {
     return io->key;
 }
 
-/* Returns a RedisModuleString with the name of the key from RedisModuleKey */
+/* Returns a RedisModuleString with the name of the key from RedisModuleKey. */
 const RedisModuleString *RM_GetKeyNameFromModuleKey(RedisModuleKey *key) {
     return key ? key->key : NULL;
 }
@@ -4442,6 +4440,9 @@ void RM_LogIOError(RedisModuleIO *io, const char *levelstr, const char *fmt, ...
 }
 
 /* Redis-like assert function.
+ *
+ * The macro `RedisModule_Assert(expression)` is recommended, rather than
+ * calling this function directly.
  *
  * A failed assertion will shut down the server and produce logging information
  * that looks identical to information generated by Redis itself.
@@ -4676,7 +4677,7 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc
  * key, or a client in queue before this one can be served, modifying the key
  * as well and making it empty again. So when a client is blocked with
  * RedisModule_BlockClientOnKeys() the reply callback is not called after
- * RM_UnblockCLient() is called, but every time a key is signaled as ready:
+ * RM_UnblockClient() is called, but every time a key is signaled as ready:
  * if the reply callback can serve the client, it returns REDISMODULE_OK
  * and the client is unblocked, otherwise it will return REDISMODULE_ERR
  * and we'll try again later.
@@ -7189,199 +7190,197 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) {
  *
  * * RedisModuleEvent_ReplicationRoleChanged:
  *
- *   This event is called when the instance switches from master
- *   to replica or the other way around, however the event is
- *   also called when the replica remains a replica but starts to
- *   replicate with a different master.
+ *     This event is called when the instance switches from master
+ *     to replica or the other way around, however the event is
+ *     also called when the replica remains a replica but starts to
+ *     replicate with a different master.
  *
- *   The following sub events are available:
+ *     The following sub events are available:
  *
- *   * REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_MASTER
- *   * REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_REPLICA
+ *     * `REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_MASTER`
+ *     * `REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_REPLICA`
  *
- *   The 'data' field can be casted by the callback to a
- *   RedisModuleReplicationInfo structure with the following fields:
+ *     The 'data' field can be casted by the callback to a
+ *     `RedisModuleReplicationInfo` structure with the following fields:
  *
- *       int master; // true if master, false if replica
- *       char *masterhost; // master instance hostname for NOW_REPLICA
- *       int masterport; // master instance port for NOW_REPLICA
- *       char *replid1; // Main replication ID
- *       char *replid2; // Secondary replication ID
- *       uint64_t repl1_offset; // Main replication offset
- *       uint64_t repl2_offset; // Offset of replid2 validity
+ *         int master; // true if master, false if replica
+ *         char *masterhost; // master instance hostname for NOW_REPLICA
+ *         int masterport; // master instance port for NOW_REPLICA
+ *         char *replid1; // Main replication ID
+ *         char *replid2; // Secondary replication ID
+ *         uint64_t repl1_offset; // Main replication offset
+ *         uint64_t repl2_offset; // Offset of replid2 validity
  *
  * * RedisModuleEvent_Persistence
  *
- *   This event is called when RDB saving or AOF rewriting starts
- *   and ends. The following sub events are available:
+ *     This event is called when RDB saving or AOF rewriting starts
+ *     and ends. The following sub events are available:
  *
- *   * REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START
- *   * REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START
- *   * REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START
- *   * REDISMODULE_SUBEVENT_PERSISTENCE_ENDED
- *   * REDISMODULE_SUBEVENT_PERSISTENCE_FAILED
+ *     * `REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START`
+ *     * `REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START`
+ *     * `REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START`
+ *     * `REDISMODULE_SUBEVENT_PERSISTENCE_ENDED`
+ *     * `REDISMODULE_SUBEVENT_PERSISTENCE_FAILED`
  *
- *   The above events are triggered not just when the user calls the
- *   relevant commands like BGSAVE, but also when a saving operation
- *   or AOF rewriting occurs because of internal server triggers.
- *   The SYNC_RDB_START sub events are happening in the forground due to
- *   SAVE command, FLUSHALL, or server shutdown, and the other RDB and
- *   AOF sub events are executed in a background fork child, so any
- *   action the module takes can only affect the generated AOF or RDB,
- *   but will not be reflected in the parent process and affect connected
- *   clients and commands. Also note that the AOF_START sub event may end
- *   up saving RDB content in case of an AOF with rdb-preamble.
+ *     The above events are triggered not just when the user calls the
+ *     relevant commands like BGSAVE, but also when a saving operation
+ *     or AOF rewriting occurs because of internal server triggers.
+ *     The SYNC_RDB_START sub events are happening in the forground due to
+ *     SAVE command, FLUSHALL, or server shutdown, and the other RDB and
+ *     AOF sub events are executed in a background fork child, so any
+ *     action the module takes can only affect the generated AOF or RDB,
+ *     but will not be reflected in the parent process and affect connected
+ *     clients and commands. Also note that the AOF_START sub event may end
+ *     up saving RDB content in case of an AOF with rdb-preamble.
  *
  * * RedisModuleEvent_FlushDB
  *
- *   The FLUSHALL, FLUSHDB or an internal flush (for instance
- *   because of replication, after the replica synchronization)
- *   happened. The following sub events are available:
+ *     The FLUSHALL, FLUSHDB or an internal flush (for instance
+ *     because of replication, after the replica synchronization)
+ *     happened. The following sub events are available:
  *
- *   * REDISMODULE_SUBEVENT_FLUSHDB_START
- *   * REDISMODULE_SUBEVENT_FLUSHDB_END
+ *     * `REDISMODULE_SUBEVENT_FLUSHDB_START`
+ *     * `REDISMODULE_SUBEVENT_FLUSHDB_END`
  *
- *   The data pointer can be casted to a RedisModuleFlushInfo
- *   structure with the following fields:
+ *     The data pointer can be casted to a RedisModuleFlushInfo
+ *     structure with the following fields:
  *
- *       int32_t async;  // True if the flush is done in a thread.
- *                       // See for instance FLUSHALL ASYNC.
- *                       // In this case the END callback is invoked
- *                       // immediately after the database is put
- *                       // in the free list of the thread.
- *       int32_t dbnum;  // Flushed database number, -1 for all the DBs
- *                       // in the case of the FLUSHALL operation.
+ *         int32_t async;  // True if the flush is done in a thread.
+ *                         // See for instance FLUSHALL ASYNC.
+ *                         // In this case the END callback is invoked
+ *                         // immediately after the database is put
+ *                         // in the free list of the thread.
+ *         int32_t dbnum;  // Flushed database number, -1 for all the DBs
+ *                         // in the case of the FLUSHALL operation.
  *
- *   The start event is called *before* the operation is initated, thus
- *   allowing the callback to call DBSIZE or other operation on the
- *   yet-to-free keyspace.
+ *     The start event is called *before* the operation is initated, thus
+ *     allowing the callback to call DBSIZE or other operation on the
+ *     yet-to-free keyspace.
  *
  * * RedisModuleEvent_Loading
  *
- *   Called on loading operations: at startup when the server is
- *   started, but also after a first synchronization when the
- *   replica is loading the RDB file from the master.
- *   The following sub events are available:
+ *     Called on loading operations: at startup when the server is
+ *     started, but also after a first synchronization when the
+ *     replica is loading the RDB file from the master.
+ *     The following sub events are available:
  *
- *   * REDISMODULE_SUBEVENT_LOADING_RDB_START
- *   * REDISMODULE_SUBEVENT_LOADING_AOF_START
- *   * REDISMODULE_SUBEVENT_LOADING_REPL_START
- *   * REDISMODULE_SUBEVENT_LOADING_ENDED
- *   * REDISMODULE_SUBEVENT_LOADING_FAILED
- *
- *   Note that AOF loading may start with an RDB data in case of
- *   rdb-preamble, in which case you'll only receive an AOF_START event.
+ *     * `REDISMODULE_SUBEVENT_LOADING_RDB_START`
+ *     * `REDISMODULE_SUBEVENT_LOADING_AOF_START`
+ *     * `REDISMODULE_SUBEVENT_LOADING_REPL_START`
+ *     * `REDISMODULE_SUBEVENT_LOADING_ENDED`
+ *     * `REDISMODULE_SUBEVENT_LOADING_FAILED`
  *
+ *     Note that AOF loading may start with an RDB data in case of
+ *     rdb-preamble, in which case you'll only receive an AOF_START event.
  *
  * * RedisModuleEvent_ClientChange
  *
- *   Called when a client connects or disconnects.
- *   The data pointer can be casted to a RedisModuleClientInfo
- *   structure, documented in RedisModule_GetClientInfoById().
- *   The following sub events are available:
+ *     Called when a client connects or disconnects.
+ *     The data pointer can be casted to a RedisModuleClientInfo
+ *     structure, documented in RedisModule_GetClientInfoById().
+ *     The following sub events are available:
  *
- *   * REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED
- *   * REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED
+ *     * `REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED`
+ *     * `REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED`
  *
  * * RedisModuleEvent_Shutdown
  *
- *   The server is shutting down. No subevents are available.
+ *     The server is shutting down. No subevents are available.
  *
  * * RedisModuleEvent_ReplicaChange
  *
- *   This event is called when the instance (that can be both a
- *   master or a replica) get a new online replica, or lose a
- *   replica since it gets disconnected.
- *   The following sub events are available:
+ *     This event is called when the instance (that can be both a
+ *     master or a replica) get a new online replica, or lose a
+ *     replica since it gets disconnected.
+ *     The following sub events are available:
  *
- *   * REDISMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE
- *   * REDISMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE
+ *     * `REDISMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE`
+ *     * `REDISMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE`
  *
- *   No additional information is available so far: future versions
- *   of Redis will have an API in order to enumerate the replicas
- *   connected and their state.
+ *     No additional information is available so far: future versions
+ *     of Redis will have an API in order to enumerate the replicas
+ *     connected and their state.
  *
  * * RedisModuleEvent_CronLoop
  *
- *   This event is called every time Redis calls the serverCron()
- *   function in order to do certain bookkeeping. Modules that are
- *   required to do operations from time to time may use this callback.
- *   Normally Redis calls this function 10 times per second, but
- *   this changes depending on the "hz" configuration.
- *   No sub events are available.
+ *     This event is called every time Redis calls the serverCron()
+ *     function in order to do certain bookkeeping. Modules that are
+ *     required to do operations from time to time may use this callback.
+ *     Normally Redis calls this function 10 times per second, but
+ *     this changes depending on the "hz" configuration.
+ *     No sub events are available.
  *
- *   The data pointer can be casted to a RedisModuleCronLoop
- *   structure with the following fields:
+ *     The data pointer can be casted to a RedisModuleCronLoop
+ *     structure with the following fields:
  *
- *       int32_t hz;  // Approximate number of events per second.
+ *         int32_t hz;  // Approximate number of events per second.
  *
  * * RedisModuleEvent_MasterLinkChange
  *
- *   This is called for replicas in order to notify when the
- *   replication link becomes functional (up) with our master,
- *   or when it goes down. Note that the link is not considered
- *   up when we just connected to the master, but only if the
- *   replication is happening correctly.
- *   The following sub events are available:
+ *     This is called for replicas in order to notify when the
+ *     replication link becomes functional (up) with our master,
+ *     or when it goes down. Note that the link is not considered
+ *     up when we just connected to the master, but only if the
+ *     replication is happening correctly.
+ *     The following sub events are available:
  *
- *   * REDISMODULE_SUBEVENT_MASTER_LINK_UP
- *   * REDISMODULE_SUBEVENT_MASTER_LINK_DOWN
+ *     * `REDISMODULE_SUBEVENT_MASTER_LINK_UP`
+ *     * `REDISMODULE_SUBEVENT_MASTER_LINK_DOWN`
  *
  * * RedisModuleEvent_ModuleChange
  *
- *   This event is called when a new module is loaded or one is unloaded.
- *   The following sub events are available:
+ *     This event is called when a new module is loaded or one is unloaded.
+ *     The following sub events are available:
  *
- *   * REDISMODULE_SUBEVENT_MODULE_LOADED
- *   * REDISMODULE_SUBEVENT_MODULE_UNLOADED
+ *     * `REDISMODULE_SUBEVENT_MODULE_LOADED`
+ *     * `REDISMODULE_SUBEVENT_MODULE_UNLOADED`
  *
- *   The data pointer can be casted to a RedisModuleModuleChange
- *   structure with the following fields:
+ *     The data pointer can be casted to a RedisModuleModuleChange
+ *     structure with the following fields:
  *
- *       const char* module_name;  // Name of module loaded or unloaded.
- *       int32_t module_version;  // Module version.
+ *         const char* module_name;  // Name of module loaded or unloaded.
+ *         int32_t module_version;  // Module version.
  *
  * * RedisModuleEvent_LoadingProgress
  *
- *   This event is called repeatedly called while an RDB or AOF file
- *   is being loaded.
- *   The following sub events are availble:
+ *     This event is called repeatedly called while an RDB or AOF file
+ *     is being loaded.
+ *     The following sub events are availble:
  *
- *   * REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB
- *   * REDISMODULE_SUBEVENT_LOADING_PROGRESS_AOF
+ *     * `REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB`
+ *     * `REDISMODULE_SUBEVENT_LOADING_PROGRESS_AOF`
  *
- *   The data pointer can be casted to a RedisModuleLoadingProgress
- *   structure with the following fields:
+ *     The data pointer can be casted to a RedisModuleLoadingProgress
+ *     structure with the following fields:
  *
- *       int32_t hz;  // Approximate number of events per second.
- *       int32_t progress;  // Approximate progress between 0 and 1024,
- *                             or -1 if unknown.
+ *         int32_t hz;  // Approximate number of events per second.
+ *         int32_t progress;  // Approximate progress between 0 and 1024,
+ *                            // or -1 if unknown.
  *
  * * RedisModuleEvent_SwapDB
  *
- *   This event is called when a SWAPDB command has been successfully
- *   Executed.
- *   For this event call currently there is no subevents available.
+ *     This event is called when a SWAPDB command has been successfully
+ *     Executed.
+ *     For this event call currently there is no subevents available.
  *
- *   The data pointer can be casted to a RedisModuleSwapDbInfo
- *   structure with the following fields:
+ *     The data pointer can be casted to a RedisModuleSwapDbInfo
+ *     structure with the following fields:
  *
- *       int32_t dbnum_first;    // Swap Db first dbnum
- *       int32_t dbnum_second;   // Swap Db second dbnum
+ *         int32_t dbnum_first;    // Swap Db first dbnum
+ *         int32_t dbnum_second;   // Swap Db second dbnum
  *
  * * RedisModuleEvent_ReplBackup
  *
- *   Called when diskless-repl-load config is set to swapdb,
- *   And redis needs to backup the the current database for the
- *   possibility to be restored later. A module with global data and
- *   maybe with aux_load and aux_save callbacks may need to use this
- *   notification to backup / restore / discard its globals.
- *   The following sub events are available:
- *
- *   * REDISMODULE_SUBEVENT_REPL_BACKUP_CREATE
- *   * REDISMODULE_SUBEVENT_REPL_BACKUP_RESTORE
- *   * REDISMODULE_SUBEVENT_REPL_BACKUP_DISCARD
+ *     Called when diskless-repl-load config is set to swapdb,
+ *     And redis needs to backup the the current database for the
+ *     possibility to be restored later. A module with global data and
+ *     maybe with aux_load and aux_save callbacks may need to use this
+ *     notification to backup / restore / discard its globals.
+ *     The following sub events are available:
  *
+ *     * `REDISMODULE_SUBEVENT_REPL_BACKUP_CREATE`
+ *     * `REDISMODULE_SUBEVENT_REPL_BACKUP_RESTORE`
+ *     * `REDISMODULE_SUBEVENT_REPL_BACKUP_DISCARD`
  *
  * The function returns REDISMODULE_OK if the module was successfully subscribed
  * for the specified event. If the API is called from a wrong context or unsupported event
@@ -8291,7 +8290,7 @@ int RM_DefragCursorSet(RedisModuleDefragCtx *ctx, unsigned long cursor) {
 /* Fetch a cursor value that has been previously stored using RM_DefragCursorSet().
  *
  * If not called for a late defrag operation, REDISMODULE_ERR will be returned and
- * the cursor should be ignored. See DM_DefragCursorSet() for more details on
+ * the cursor should be ignored. See RM_DefragCursorSet() for more details on
  * defrag cursors.
  */
 int RM_DefragCursorGet(RedisModuleDefragCtx *ctx, unsigned long *cursor) {
diff --git a/src/modules/gendoc.rb b/src/modules/gendoc.rb
index 249c8b6ea..2fd2ec5d7 100644
--- a/src/modules/gendoc.rb
+++ b/src/modules/gendoc.rb
@@ -9,13 +9,21 @@ def markdown(s)
     s.chop! while s[-1] == "\n" || s[-1] == " "
     lines = s.split("\n")
     newlines = []
-    # Backquote function, macro and type names, except if already backquoted and
-    # in code blocks indented by 4 spaces.
+    # Fix some markdown, except in code blocks indented by 4 spaces.
     lines.each{|l|
         if not l.start_with?('    ')
-            l = l.gsub(/(?<!`)RM_[A-z()]+/){|x| "`#{x}`"}
-            l = l.gsub(/(?<!`)RedisModule[A-z()]+/){|x| "`#{x}`"}
-            l = l.gsub(/(?<!`)REDISMODULE_[A-z]+/){|x| "`#{x}`"}
+            # Rewrite RM_Xyz() to `RedisModule_Xyz()`. The () suffix is
+            # optional. Even RM_Xyz*() with * as wildcard is handled.
+            l = l.gsub(/(?<!`)RM_([A-z]+(?:\*?\(\))?)/, '`RedisModule_\1`')
+            # Add backquotes around RedisModule functions and type where missing.
+            l = l.gsub(/(?<!`)RedisModule[A-z]+(?:\*?\(\))?/){|x| "`#{x}`"}
+            # Add backquotes around c functions like malloc() where missing.
+            l = l.gsub(/(?<![`A-z])[a-z_]+\(\)/, '`\0`')
+            # Add backquotes around macro and var names containing underscores.
+            l = l.gsub(/(?<![`A-z\*])[A-Za-z]+_[A-Za-z0-9_]+/){|x| "`#{x}`"}
+            # Link URLs preceded by space (i.e. when not already linked)
+            l = l.gsub(/ (https?:\/\/[A-Za-z0-9_\/\.\-]+[A-Za-z0-9\/])/,
+                       ' [\1](\1)')
         end
         newlines << l
     }

From baf92f3f1c15cf188fafaf802202da1c54caa709 Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Wed, 20 Jan 2021 12:56:45 +0100
Subject: [PATCH 0986/1098] Fix firstkey,lastkey,step in COMMAND command for
 some commands (#8367)

The output for COMMAND command was wrong for some commands.
clients can use firstkey,lastkey,step to find (some) key name arguments, and the
"movablekeys" flag to know that they can't know all (or any) of the key name arguments.

These commands had the wrong output:
1. GEORADIUS*_RO used to have "movablekeys" (which it doesn't really need)
2. XREAD and XREADGROUP used to have (1,1,1). but that's completely wrong.
3. Z*STORE used to have (0,0,0) but it can at lest give the index of the dstkey (1,1,1)
---
 src/server.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/server.c b/src/server.c
index f3d75ce04..03e448f10 100644
--- a/src/server.c
+++ b/src/server.c
@@ -449,15 +449,15 @@ struct redisCommand redisCommandTable[] = {
 
     {"zunionstore",zunionstoreCommand,-4,
      "write use-memory @sortedset",
-     0,zunionInterDiffStoreGetKeys,0,0,0,0,0,0},
+     0,zunionInterDiffStoreGetKeys,1,1,1,0,0,0},
 
     {"zinterstore",zinterstoreCommand,-4,
      "write use-memory @sortedset",
-     0,zunionInterDiffStoreGetKeys,0,0,0,0,0,0},
+     0,zunionInterDiffStoreGetKeys,1,1,1,0,0,0},
 
     {"zdiffstore",zdiffstoreCommand,-4,
      "write use-memory @sortedset",
-     0,zunionInterDiffStoreGetKeys,0,0,0,0,0,0},
+     0,zunionInterDiffStoreGetKeys,1,1,1,0,0,0},
 
     {"zunion",zunionCommand,-3,
      "read-only @sortedset",
@@ -941,7 +941,7 @@ struct redisCommand redisCommandTable[] = {
 
     {"georadius_ro",georadiusroCommand,-6,
      "read-only @geo",
-     0,georadiusGetKeys,1,1,1,0,0,0},
+     0,NULL,1,1,1,0,0,0},
 
     {"georadiusbymember",georadiusbymemberCommand,-5,
      "write use-memory @geo",
@@ -949,7 +949,7 @@ struct redisCommand redisCommandTable[] = {
 
     {"georadiusbymember_ro",georadiusbymemberroCommand,-5,
      "read-only @geo",
-     0,georadiusGetKeys,1,1,1,0,0,0},
+     0,NULL,1,1,1,0,0,0},
 
     {"geohash",geohashCommand,-2,
      "read-only @geo",
@@ -1016,11 +1016,11 @@ struct redisCommand redisCommandTable[] = {
 
     {"xread",xreadCommand,-4,
      "read-only @stream @blocking",
-     0,xreadGetKeys,1,1,1,0,0,0},
+     0,xreadGetKeys,0,0,0,0,0,0},
 
     {"xreadgroup",xreadCommand,-7,
      "write @stream @blocking",
-     0,xreadGetKeys,1,1,1,0,0,0},
+     0,xreadGetKeys,0,0,0,0,0,0},
 
     {"xgroup",xgroupCommand,-2,
      "write use-memory @stream",

From 5a77d015bed9734064c6572d5425971e9d697c53 Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Wed, 20 Jan 2021 13:03:38 +0100
Subject: [PATCH 0987/1098] Fix misleading module test (#8366)

the test was misleading because the module would actually woke up on a wrong type and
re-blocked, while the test name suggests the module doesn't not wake up at all on a wrong type..

i changed the name of the test + added verification that indeed the module wakes up and gets
re-blocked after it understand it's the wrong type
---
 tests/modules/blockonkeys.c          | 1 +
 tests/unit/moduleapi/blockonkeys.tcl | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/modules/blockonkeys.c b/tests/modules/blockonkeys.c
index 6f157342c..6e4b5b79c 100644
--- a/tests/modules/blockonkeys.c
+++ b/tests/modules/blockonkeys.c
@@ -65,6 +65,7 @@ int get_fsl(RedisModuleCtx *ctx, RedisModuleString *keyname, int mode, int creat
         RedisModule_CloseKey(key);
         if (reply_on_failure)
             RedisModule_ReplyWithError(ctx, REDISMODULE_ERRORMSG_WRONGTYPE);
+        RedisModule_Call(ctx, "INCR", "c", "fsl_wrong_type");
         return 0;
     }
 
diff --git a/tests/unit/moduleapi/blockonkeys.tcl b/tests/unit/moduleapi/blockonkeys.tcl
index 0f6612164..00041674f 100644
--- a/tests/unit/moduleapi/blockonkeys.tcl
+++ b/tests/unit/moduleapi/blockonkeys.tcl
@@ -168,7 +168,7 @@ start_server {tags {"modules"}} {
         assert_error "*unblocked*" {$rd read}
     }
 
-    test {Module client blocked on keys does not wake up on wrong type} {
+    test {Module client re-blocked on keys after woke up on wrong type} {
         r del k
         set rd [redis_deferring_client]
         $rd fsl.bpop k 0
@@ -184,6 +184,7 @@ start_server {tags {"modules"}} {
         r del k
         r fsl.push k 34
         assert_equal {34} [$rd read]
+        assert_equal {1} [r get fsl_wrong_type] ;# first lpush caused one wrong-type wake-up
     }
 
     test {Module client blocked on keys woken up by LPUSH} {

From ac5f21d613bf4a4883e767389ac5651a4118a5af Mon Sep 17 00:00:00 2001
From: Guy Korland <gkorland@gmail.com>
Date: Wed, 20 Jan 2021 14:07:09 +0200
Subject: [PATCH 0988/1098] Add CI for FreeBSD (#8292)

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 .github/workflows/ci.yml    | 11 +++++++++++
 .github/workflows/daily.yml | 17 +++++++++++++++++
 src/server.c                |  2 +-
 3 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2582c53a4..2e1e7865c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -58,3 +58,14 @@ jobs:
       run: |
         yum -y install gcc make
         make REDIS_CFLAGS='-Werror'
+        
+  build-freebsd:
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: make
+      uses: vmactions/freebsd-vm@v0.1.0
+      with:
+        usesh: true
+        prepare: pkg install -y gmake
+        run: gmake
diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index cfbf24f87..8fb23bac4 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -203,3 +203,20 @@ jobs:
     - name: cluster tests
       run: ./runtest-cluster
 
+  test-freebsd:
+    runs-on: macos-latest
+    if: github.repository == 'redis/redis'
+    timeout-minutes: 14400
+    steps:
+    - uses: actions/checkout@v2
+    - name: test
+      uses: vmactions/freebsd-vm@v0.1.0
+      with:
+        usesh: true
+        prepare: pkg install -y gmake lang/tcl85
+        run: |
+          gmake
+          ./runtest --accurate --verbose --no-latency
+          MAKE=gmake ./runtest-moduleapi --verbose
+          ./runtest-sentinel
+          ./runtest-cluster
diff --git a/src/server.c b/src/server.c
index 03e448f10..1b8bc0c42 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4684,7 +4684,7 @@ sds genRedisInfoString(const char *section) {
             "aof_last_cow_size:%zu\r\n"
             "module_fork_in_progress:%d\r\n"
             "module_fork_last_cow_size:%zu\r\n",
-            server.loading,
+            (int)server.loading,
             server.stat_current_cow_bytes,
             server.dirty,
             server.child_type == CHILD_TYPE_RDB,

From b548ffabbecca073e241882c22192f682a086242 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 20 Jan 2021 21:57:24 +0200
Subject: [PATCH 0989/1098] CONFIG REWRITE should honor umask settings. (#8371)

Fixes a regression introduced due to a new (safer) way of rewriting configuration files. In the past the file was simply overwritten (same inode), but now Redis creates a new temporary file and later renames it over the old one.

The temp file typically gets created with 0600 permissions so we later fchmod it to fix that. Unlike open with O_CREAT, fchmod doesn't consider umask so we have to do that explicitly.

Fixes #8369
---
 src/config.c | 2 +-
 src/server.c | 6 ++++++
 src/server.h | 1 +
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/config.c b/src/config.c
index 2442ace8e..2f9721f58 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1683,7 +1683,7 @@ int rewriteConfigOverwriteFile(char *configfile, sds content) {
 
     if (fsync(fd))
         serverLog(LL_WARNING, "Could not sync tmp config file to disk (%s)", strerror(errno));
-    else if (fchmod(fd, 0644) == -1)
+    else if (fchmod(fd, 0644 & ~server.umask) == -1)
         serverLog(LL_WARNING, "Could not chmod config file (%s)", strerror(errno));
     else if (rename(tmp_conffile, configfile) == -1)
         serverLog(LL_WARNING, "Could not rename tmp config file (%s)", strerror(errno));
diff --git a/src/server.c b/src/server.c
index 1b8bc0c42..dc661b6e2 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5753,6 +5753,12 @@ int main(int argc, char **argv) {
     init_genrand64(((long long) tv.tv_sec * 1000000 + tv.tv_usec) ^ getpid());
     crc64_init();
 
+    /* Store umask value. Because umask(2) only offers a set-and-get API we have
+     * to reset it and restore it back. We do this early to avoid a potential
+     * race condition with threads that could be creating files or directories.
+     */
+    umask(server.umask = umask(0777));
+
     uint8_t hashseed[16];
     getRandomBytes(hashseed,sizeof(hashseed));
     dictSetHashFunctionSeed(hashseed);
diff --git a/src/server.h b/src/server.h
index 0de7b3a1e..13144ce32 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1124,6 +1124,7 @@ struct redisServer {
     int config_hz;              /* Configured HZ value. May be different than
                                    the actual 'hz' field value if dynamic-hz
                                    is enabled. */
+    mode_t umask;               /* The umask value of the process on startup */
     int hz;                     /* serverCron() calls frequency in hertz */
     int in_fork_child;          /* indication that this is a fork child */
     redisDb *db;

From 4f9de66f0eb50469767ed05a38c2eb091c9663e5 Mon Sep 17 00:00:00 2001
From: Simon Kotwicz <simonkotwicz@gmail.com>
Date: Wed, 20 Jan 2021 20:36:59 -0500
Subject: [PATCH 0990/1098] Fix typo in redis.conf doc (#8362)

Fix incorrect documentation about default save time
---
 redis.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/redis.conf b/redis.conf
index bb0d70644..a131b406f 100644
--- a/redis.conf
+++ b/redis.conf
@@ -341,7 +341,7 @@ always-show-logo no
 #
 # Unless specified otherwise, by default Redis will save the DB:
 #   * After 3600 seconds (an hour) if at least 1 key changed
-#   * After 300 seconds (5 minutes) if at least 10 keys changed
+#   * After 300 seconds (5 minutes) if at least 100 keys changed
 #   * After 60 seconds if at least 10000 keys changed
 #
 # You can set these explicitly by uncommenting the three following lines.

From 53cdf8a3d10285208e96980881dcac94314d8686 Mon Sep 17 00:00:00 2001
From: daemyungkang <60596794+daemyungkang@users.noreply.github.com>
Date: Thu, 21 Jan 2021 15:08:05 +0900
Subject: [PATCH 0991/1098] change log for invalid cluster port (#8343)

Update log message to correctly indicate cluster port range
---
 src/cluster.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 2b5acec90..97a25b0b3 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -509,8 +509,7 @@ void clusterInit(void) {
         serverLog(LL_WARNING, "Redis port number too high. "
                    "Cluster communication port is 10,000 port "
                    "numbers higher than your Redis port. "
-                   "Your Redis port number must be "
-                   "lower than 55535.");
+                   "Your Redis port number must be 55535 or less.");
         exit(1);
     }
     if (listenToPort(port+CLUSTER_PORT_INCR,

From 3f3cb9d09b815869764e74dd52af1994fed55fe6 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 21 Jan 2021 11:56:08 +0200
Subject: [PATCH 0992/1098] Better error message const correctness. (#8348)

---
 src/acl.c      | 10 +++----
 src/config.c   | 74 +++++++++++++++++++++++++-------------------------
 src/sentinel.c |  4 +--
 src/server.h   |  4 +--
 4 files changed, 46 insertions(+), 46 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 14d023cc3..1f07c292f 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1024,8 +1024,8 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
 
 /* Return a description of the error that occurred in ACLSetUser() according to
  * the errno value set by the function on error. */
-char *ACLSetUserStringError(void) {
-    char *errmsg = "Wrong format";
+const char *ACLSetUserStringError(void) {
+    const char *errmsg = "Wrong format";
     if (errno == ENOENT)
         errmsg = "Unknown command or category name in ACL";
     else if (errno == EINVAL)
@@ -1454,7 +1454,7 @@ int ACLLoadConfiguredUsers(void) {
         /* Load every rule defined for this user. */
         for (int j = 1; aclrules[j]; j++) {
             if (ACLSetUser(u,aclrules[j],sdslen(aclrules[j])) != C_OK) {
-                char *errmsg = ACLSetUserStringError();
+                const char *errmsg = ACLSetUserStringError();
                 serverLog(LL_WARNING,"Error loading ACL rule '%s' for "
                                      "the user named '%s': %s",
                           aclrules[j],aclrules[0],errmsg);
@@ -1587,7 +1587,7 @@ sds ACLLoadFromFile(const char *filename) {
         for (j = 2; j < argc; j++) {
             argv[j] = sdstrim(argv[j],"\t\r\n");
             if (ACLSetUser(fakeuser,argv[j],sdslen(argv[j])) != C_OK) {
-                char *errmsg = ACLSetUserStringError();
+                const char *errmsg = ACLSetUserStringError();
                 errors = sdscatprintf(errors,
                          "%s:%d: %s. ",
                          server.acl_filename, linenum, errmsg);
@@ -1908,7 +1908,7 @@ void aclCommand(client *c) {
 
         for (int j = 3; j < c->argc; j++) {
             if (ACLSetUser(tempu,c->argv[j]->ptr,sdslen(c->argv[j]->ptr)) != C_OK) {
-                char *errmsg = ACLSetUserStringError();
+                const char *errmsg = ACLSetUserStringError();
                 addReplyErrorFormat(c,
                     "Error in ACL SETUSER modifier '%s': %s",
                     (char*)c->argv[j]->ptr, errmsg);
diff --git a/src/config.c b/src/config.c
index 2f9721f58..ec8867342 100644
--- a/src/config.c
+++ b/src/config.c
@@ -153,15 +153,15 @@ int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT] = { 0, 200, 800 };
 typedef struct boolConfigData {
     int *config; /* The pointer to the server config this value is stored in */
     const int default_value; /* The default value of the config on rewrite */
-    int (*is_valid_fn)(int val, char **err); /* Optional function to check validity of new value (generic doc above) */
-    int (*update_fn)(int val, int prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */
+    int (*is_valid_fn)(int val, const char **err); /* Optional function to check validity of new value (generic doc above) */
+    int (*update_fn)(int val, int prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */
 } boolConfigData;
 
 typedef struct stringConfigData {
     char **config; /* Pointer to the server config this value is stored in. */
     const char *default_value; /* Default value of the config on rewrite. */
-    int (*is_valid_fn)(char* val, char **err); /* Optional function to check validity of new value (generic doc above) */
-    int (*update_fn)(char* val, char* prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */
+    int (*is_valid_fn)(char* val, const char **err); /* Optional function to check validity of new value (generic doc above) */
+    int (*update_fn)(char* val, char* prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */
     int convert_empty_to_null; /* Boolean indicating if empty strings should
                                   be stored as a NULL value. */
 } stringConfigData;
@@ -169,8 +169,8 @@ typedef struct stringConfigData {
 typedef struct sdsConfigData {
     sds *config; /* Pointer to the server config this value is stored in. */
     const char *default_value; /* Default value of the config on rewrite. */
-    int (*is_valid_fn)(sds val, char **err); /* Optional function to check validity of new value (generic doc above) */
-    int (*update_fn)(sds val, sds prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */
+    int (*is_valid_fn)(sds val, const char **err); /* Optional function to check validity of new value (generic doc above) */
+    int (*update_fn)(sds val, sds prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */
     int convert_empty_to_null; /* Boolean indicating if empty SDS strings should
                                   be stored as a NULL value. */
 } sdsConfigData;
@@ -179,8 +179,8 @@ typedef struct enumConfigData {
     int *config; /* The pointer to the server config this value is stored in */
     configEnum *enum_value; /* The underlying enum type this data represents */
     const int default_value; /* The default value of the config on rewrite */
-    int (*is_valid_fn)(int val, char **err); /* Optional function to check validity of new value (generic doc above) */
-    int (*update_fn)(int val, int prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */
+    int (*is_valid_fn)(int val, const char **err); /* Optional function to check validity of new value (generic doc above) */
+    int (*update_fn)(int val, int prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */
 } enumConfigData;
 
 typedef enum numericType {
@@ -214,8 +214,8 @@ typedef struct numericConfigData {
     long long lower_bound; /* The lower bound of this numeric value */
     long long upper_bound; /* The upper bound of this numeric value */
     const long long default_value; /* The default value of the config on rewrite */
-    int (*is_valid_fn)(long long val, char **err); /* Optional function to check validity of new value (generic doc above) */
-    int (*update_fn)(long long val, long long prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */
+    int (*is_valid_fn)(long long val, const char **err); /* Optional function to check validity of new value (generic doc above) */
+    int (*update_fn)(long long val, long long prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */
 } numericConfigData;
 
 typedef union typeData {
@@ -230,10 +230,10 @@ typedef struct typeInterface {
     /* Called on server start, to init the server with default value */
     void (*init)(typeData data);
     /* Called on server start, should return 1 on success, 0 on error and should set err */
-    int (*load)(typeData data, sds *argc, int argv, char **err);
+    int (*load)(typeData data, sds *argc, int argv, const char **err);
     /* Called on server startup and CONFIG SET, returns 1 on success, 0 on error
      * and can set a verbose err string, update is true when called from CONFIG SET */
-    int (*set)(typeData data, sds value, int update, char **err);
+    int (*set)(typeData data, sds value, int update, const char **err);
     /* Called on CONFIG GET, required to add output to the client */
     void (*get)(client *c, typeData data);
     /* Called on CONFIG REWRITE, required to rewrite the config state */
@@ -325,7 +325,7 @@ void queueLoadModule(sds path, sds *argv, int argc) {
  * server.oom_score_adj_values if valid.
  */
 
-static int updateOOMScoreAdjValues(sds *args, char **err, int apply) {
+static int updateOOMScoreAdjValues(sds *args, const char **err, int apply) {
     int i;
     int values[CONFIG_OOM_COUNT];
 
@@ -385,7 +385,7 @@ void initConfigValues() {
 }
 
 void loadServerConfigFromString(char *config) {
-    char *err = NULL;
+    const char *err = NULL;
     int linenum = 0, totlines, i;
     int slaveof_linenum = 0;
     sds *lines;
@@ -608,7 +608,7 @@ void loadServerConfigFromString(char *config) {
             int argc_err;
             if (ACLAppendUserForLoading(argv,argc,&argc_err) == C_ERR) {
                 char buf[1024];
-                char *errmsg = ACLSetUserStringError();
+                const char *errmsg = ACLSetUserStringError();
                 snprintf(buf,sizeof(buf),"Error in user declaration '%s': %s",
                     argv[argc_err],errmsg);
                 err = buf;
@@ -730,7 +730,7 @@ void configSetCommand(client *c) {
     robj *o;
     long long ll;
     int err;
-    char *errstr = NULL;
+    const char *errstr = NULL;
     serverAssertWithInfo(c,c->argv[2],sdsEncodedObject(c->argv[2]));
     serverAssertWithInfo(c,c->argv[3],sdsEncodedObject(c->argv[3]));
     o = c->argv[3];
@@ -1795,7 +1795,7 @@ static void boolConfigInit(typeData data) {
     *data.yesno.config = data.yesno.default_value;
 }
 
-static int boolConfigSet(typeData data, sds value, int update, char **err) {
+static int boolConfigSet(typeData data, sds value, int update, const char **err) {
     int yn = yesnotoi(value);
     if (yn == -1) {
         *err = "argument must be 'yes' or 'no'";
@@ -1836,7 +1836,7 @@ static void stringConfigInit(typeData data) {
     *data.string.config = (data.string.convert_empty_to_null && !data.string.default_value) ? NULL : zstrdup(data.string.default_value);
 }
 
-static int stringConfigSet(typeData data, sds value, int update, char **err) {
+static int stringConfigSet(typeData data, sds value, int update, const char **err) {
     if (data.string.is_valid_fn && !data.string.is_valid_fn(value, err))
         return 0;
     char *prev = *data.string.config;
@@ -1863,7 +1863,7 @@ static void sdsConfigInit(typeData data) {
     *data.sds.config = (data.sds.convert_empty_to_null && !data.sds.default_value) ? NULL: sdsnew(data.sds.default_value);
 }
 
-static int sdsConfigSet(typeData data, sds value, int update, char **err) {
+static int sdsConfigSet(typeData data, sds value, int update, const char **err) {
     if (data.sds.is_valid_fn && !data.sds.is_valid_fn(value, err))
         return 0;
     sds prev = *data.sds.config;
@@ -1922,7 +1922,7 @@ static void enumConfigInit(typeData data) {
     *data.enumd.config = data.enumd.default_value;
 }
 
-static int enumConfigSet(typeData data, sds value, int update, char **err) {
+static int enumConfigSet(typeData data, sds value, int update, const char **err) {
     int enumval = configEnumGetValue(data.enumd.enum_value, value);
     if (enumval == INT_MIN) {
         sds enumerr = sdsnew("argument must be one of the following: ");
@@ -2028,7 +2028,7 @@ static void numericConfigInit(typeData data) {
     SET_NUMERIC_TYPE(data.numeric.default_value)
 }
 
-static int numericBoundaryCheck(typeData data, long long ll, char **err) {
+static int numericBoundaryCheck(typeData data, long long ll, const char **err) {
     if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG ||
         data.numeric.numeric_type == NUMERIC_TYPE_UINT ||
         data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) {
@@ -2058,7 +2058,7 @@ static int numericBoundaryCheck(typeData data, long long ll, char **err) {
     return 1;
 }
 
-static int numericConfigSet(typeData data, sds value, int update, char **err) {
+static int numericConfigSet(typeData data, sds value, int update, const char **err) {
     long long ll, prev = 0;
     if (data.numeric.is_memory) {
         int memerr;
@@ -2196,7 +2196,7 @@ static void numericConfigRewrite(typeData data, const char *name, struct rewrite
     } \
 }
 
-static int isValidActiveDefrag(int val, char **err) {
+static int isValidActiveDefrag(int val, const char **err) {
 #ifndef HAVE_DEFRAG
     if (val) {
         *err = "Active defragmentation cannot be enabled: it "
@@ -2212,7 +2212,7 @@ static int isValidActiveDefrag(int val, char **err) {
     return 1;
 }
 
-static int isValidDBfilename(char *val, char **err) {
+static int isValidDBfilename(char *val, const char **err) {
     if (!pathIsBaseName(val)) {
         *err = "dbfilename can't be a path, just a filename";
         return 0;
@@ -2220,7 +2220,7 @@ static int isValidDBfilename(char *val, char **err) {
     return 1;
 }
 
-static int isValidAOFfilename(char *val, char **err) {
+static int isValidAOFfilename(char *val, const char **err) {
     if (!pathIsBaseName(val)) {
         *err = "appendfilename can't be a path, just a filename";
         return 0;
@@ -2228,7 +2228,7 @@ static int isValidAOFfilename(char *val, char **err) {
     return 1;
 }
 
-static int updateHZ(long long val, long long prev, char **err) {
+static int updateHZ(long long val, long long prev, const char **err) {
     UNUSED(prev);
     UNUSED(err);
     /* Hz is more a hint from the user, so we accept values out of range
@@ -2240,14 +2240,14 @@ static int updateHZ(long long val, long long prev, char **err) {
     return 1;
 }
 
-static int updateJemallocBgThread(int val, int prev, char **err) {
+static int updateJemallocBgThread(int val, int prev, const char **err) {
     UNUSED(prev);
     UNUSED(err);
     set_jemalloc_bg_thread(val);
     return 1;
 }
 
-static int updateReplBacklogSize(long long val, long long prev, char **err) {
+static int updateReplBacklogSize(long long val, long long prev, const char **err) {
     /* resizeReplicationBacklog sets server.repl_backlog_size, and relies on
      * being able to tell when the size changes, so restore prev before calling it. */
     UNUSED(err);
@@ -2256,7 +2256,7 @@ static int updateReplBacklogSize(long long val, long long prev, char **err) {
     return 1;
 }
 
-static int updateMaxmemory(long long val, long long prev, char **err) {
+static int updateMaxmemory(long long val, long long prev, const char **err) {
     UNUSED(prev);
     UNUSED(err);
     if (val) {
@@ -2269,7 +2269,7 @@ static int updateMaxmemory(long long val, long long prev, char **err) {
     return 1;
 }
 
-static int updateGoodSlaves(long long val, long long prev, char **err) {
+static int updateGoodSlaves(long long val, long long prev, const char **err) {
     UNUSED(val);
     UNUSED(prev);
     UNUSED(err);
@@ -2277,7 +2277,7 @@ static int updateGoodSlaves(long long val, long long prev, char **err) {
     return 1;
 }
 
-static int updateAppendonly(int val, int prev, char **err) {
+static int updateAppendonly(int val, int prev, const char **err) {
     UNUSED(prev);
     if (val == 0 && server.aof_state != AOF_OFF) {
         stopAppendOnly();
@@ -2290,7 +2290,7 @@ static int updateAppendonly(int val, int prev, char **err) {
     return 1;
 }
 
-static int updateSighandlerEnabled(int val, int prev, char **err) {
+static int updateSighandlerEnabled(int val, int prev, const char **err) {
     UNUSED(err);
     UNUSED(prev);
     if (val)
@@ -2300,7 +2300,7 @@ static int updateSighandlerEnabled(int val, int prev, char **err) {
     return 1;
 }
 
-static int updateMaxclients(long long val, long long prev, char **err) {
+static int updateMaxclients(long long val, long long prev, const char **err) {
     /* Try to check if the OS is capable of supporting so many FDs. */
     if (val > prev) {
         adjustOpenFilesLimit();
@@ -2328,7 +2328,7 @@ static int updateMaxclients(long long val, long long prev, char **err) {
     return 1;
 }
 
-static int updateOOMScoreAdj(int val, int prev, char **err) {
+static int updateOOMScoreAdj(int val, int prev, const char **err) {
     UNUSED(prev);
 
     if (val) {
@@ -2342,7 +2342,7 @@ static int updateOOMScoreAdj(int val, int prev, char **err) {
 }
 
 #ifdef USE_OPENSSL
-static int updateTlsCfg(char *val, char *prev, char **err) {
+static int updateTlsCfg(char *val, char *prev, const char **err) {
     UNUSED(val);
     UNUSED(prev);
     UNUSED(err);
@@ -2355,13 +2355,13 @@ static int updateTlsCfg(char *val, char *prev, char **err) {
     }
     return 1;
 }
-static int updateTlsCfgBool(int val, int prev, char **err) {
+static int updateTlsCfgBool(int val, int prev, const char **err) {
     UNUSED(val);
     UNUSED(prev);
     return updateTlsCfg(NULL, NULL, err);
 }
 
-static int updateTlsCfgInt(long long val, long long prev, char **err) {
+static int updateTlsCfgInt(long long val, long long prev, const char **err) {
     UNUSED(val);
     UNUSED(prev);
     return updateTlsCfg(NULL, NULL, err);
diff --git a/src/sentinel.c b/src/sentinel.c
index 26c129385..9da7732a0 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -1645,7 +1645,7 @@ char *sentinelInstanceMapCommand(sentinelRedisInstance *ri, char *command) {
 
 /* Generalise handling create instance error. Use SRI_MASTER, SRI_SLAVE or
  * SRI_SENTINEL as a role value. */
-char *sentinelCheckCreateInstanceErrors(int role) {
+const char *sentinelCheckCreateInstanceErrors(int role) {
     switch(errno) {
     case EBUSY:
         switch (role) {
@@ -1669,7 +1669,7 @@ char *sentinelCheckCreateInstanceErrors(int role) {
     }
 }
 
-char *sentinelHandleConfiguration(char **argv, int argc) {
+const char *sentinelHandleConfiguration(char **argv, int argc) {
     sentinelRedisInstance *ri;
 
     if (!strcasecmp(argv[0],"monitor") && argc == 5) {
diff --git a/src/server.h b/src/server.h
index 13144ce32..cb103aca8 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2044,7 +2044,7 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen);
 sds ACLDefaultUserFirstPassword(void);
 uint64_t ACLGetCommandCategoryFlagByName(const char *name);
 int ACLAppendUserForLoading(sds *argv, int argc, int *argc_err);
-char *ACLSetUserStringError(void);
+const char *ACLSetUserStringError(void);
 int ACLLoadConfiguredUsers(void);
 sds ACLDescribeUser(user *u);
 void ACLLoadUsersAtStartup(void);
@@ -2332,7 +2332,7 @@ int clusterSendModuleMessageToTarget(const char *target, uint64_t module_id, uin
 void initSentinelConfig(void);
 void initSentinel(void);
 void sentinelTimer(void);
-char *sentinelHandleConfiguration(char **argv, int argc);
+const char *sentinelHandleConfiguration(char **argv, int argc);
 void sentinelIsRunning(void);
 
 /* redis-check-rdb & aof */

From f28fccd2db929e7034874cf795af0433d8deb467 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 21 Jan 2021 19:55:42 +0200
Subject: [PATCH 0993/1098] Fix anetCloexec for Sentinel TLS conns. (#8377)

The flag should be set before TLS negotiation begins to avoid a race
condition where a fork+exec before it is completed ends up leaking the
file descriptor.
---
 src/sentinel.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/sentinel.c b/src/sentinel.c
index 9da7732a0..75030df78 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -2126,6 +2126,7 @@ void sentinelReconnectInstance(sentinelRedisInstance *ri) {
     /* Commands connection. */
     if (link->cc == NULL) {
         link->cc = redisAsyncConnectBind(ri->addr->ip,ri->addr->port,NET_FIRST_BIND_ADDR);
+        if (!link->cc->err) anetCloexec(link->cc->c.fd);
         if (!link->cc->err && server.tls_replication &&
                 (instanceLinkNegotiateTLS(link->cc) == C_ERR)) {
             sentinelEvent(LL_DEBUG,"-cmd-link-reconnection",ri,"%@ #Failed to initialize TLS");
@@ -2135,7 +2136,6 @@ void sentinelReconnectInstance(sentinelRedisInstance *ri) {
                 link->cc->errstr);
             instanceLinkCloseConnection(link,link->cc);
         } else {
-            anetCloexec(link->cc->c.fd);
             link->pending_commands = 0;
             link->cc_conn_time = mstime();
             link->cc->data = link;
@@ -2154,6 +2154,7 @@ void sentinelReconnectInstance(sentinelRedisInstance *ri) {
     /* Pub / Sub */
     if ((ri->flags & (SRI_MASTER|SRI_SLAVE)) && link->pc == NULL) {
         link->pc = redisAsyncConnectBind(ri->addr->ip,ri->addr->port,NET_FIRST_BIND_ADDR);
+        if (!link->pc->err) anetCloexec(link->pc->c.fd);
         if (!link->pc->err && server.tls_replication &&
                 (instanceLinkNegotiateTLS(link->pc) == C_ERR)) {
             sentinelEvent(LL_DEBUG,"-pubsub-link-reconnection",ri,"%@ #Failed to initialize TLS");
@@ -2163,7 +2164,6 @@ void sentinelReconnectInstance(sentinelRedisInstance *ri) {
             instanceLinkCloseConnection(link,link->pc);
         } else {
             int retval;
-            anetCloexec(link->pc->c.fd);
             link->pc_conn_time = mstime();
             link->pc->data = link;
             redisAeAttach(server.el,link->pc);

From 0232778f35e949bb5e0b84f7cc63b230ddbd1c6d Mon Sep 17 00:00:00 2001
From: Huang Zw <huang_zhw@126.com>
Date: Fri, 22 Jan 2021 15:37:25 +0800
Subject: [PATCH 0994/1098] In dbOverwrite moduleNotifyKeyUnlink should use old
 val (#8381)

The module notification was using the wrong value.
---
 src/db.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/db.c b/src/db.c
index 29fef13fb..3871753dd 100644
--- a/src/db.c
+++ b/src/db.c
@@ -226,7 +226,7 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) {
     /* Although the key is not really deleted from the database, we regard 
     overwrite as two steps of unlink+add, so we still need to call the unlink 
     callback of the module. */
-    moduleNotifyKeyUnlink(key,val);
+    moduleNotifyKeyUnlink(key,old);
     dictSetVal(db->dict, de, val);
 
     if (server.lazyfree_lazy_server_del) {

From 8449a5df87017361000919d372b520a897c2de07 Mon Sep 17 00:00:00 2001
From: Andy Pan <panjf2000@gmail.com>
Date: Fri, 22 Jan 2021 18:11:58 +0800
Subject: [PATCH 0995/1098] Sentinel tests, disable FD leak check, and print
 more details (#8376)

* Print more details about fd leaks
* temporarily prevent the leaks from failing the tests

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 tests/instances.tcl                     | 3 ++-
 tests/sentinel/tests/includes/notify.sh | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/instances.tcl b/tests/instances.tcl
index 4e583eeb3..e35aaefa0 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -423,7 +423,8 @@ proc end_tests {} {
     if { [file exists $sentinel_fd_leaks_file] } {
         puts [colorstr red "WARNING: sentinel test(s) failed, there are leaked fds in sentinel:"] 
         puts [exec cat $sentinel_fd_leaks_file]
-        exit 1
+        # temporarily disabling this error from failing the tests until leaks are fixed.
+        #exit 1
     }
 
     if {$::failed == 0 } {
diff --git a/tests/sentinel/tests/includes/notify.sh b/tests/sentinel/tests/includes/notify.sh
index 2365f1d3a..5de0eaf76 100755
--- a/tests/sentinel/tests/includes/notify.sh
+++ b/tests/sentinel/tests/includes/notify.sh
@@ -16,5 +16,6 @@ then
     if [ ! -f $sentinel_fd_leaks_file ]
     then
         ls -l /proc/self/fd | cat >> $sentinel_fd_leaks_file
+        lsof -p $$ | cat >> $sentinel_fd_leaks_file
     fi
 fi

From 9c1483100a1d0a9160951852d70e632fced179fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= <viktor.soderqvist@est.tech>
Date: Fri, 22 Jan 2021 15:19:37 +0100
Subject: [PATCH 0996/1098] Test that module can wake up module blocked on
 non-empty list key (#8382)

BLPOP and other blocking list commands can only block on empty keys
and LPUSH only wakes up clients when the list is created.

Using the module API, it's possible to block on a non-empty key.
Unblocking a client blocked on a non-empty list (or zset) can only
be done using RedisModule_SignalKeyAsReady(). This commit tests it.
---
 tests/modules/blockonkeys.c          | 89 +++++++++++++++++++++++++++-
 tests/unit/moduleapi/blockonkeys.tcl | 20 +++++++
 2 files changed, 108 insertions(+), 1 deletion(-)

diff --git a/tests/modules/blockonkeys.c b/tests/modules/blockonkeys.c
index 6e4b5b79c..2b7b614a9 100644
--- a/tests/modules/blockonkeys.c
+++ b/tests/modules/blockonkeys.c
@@ -2,6 +2,7 @@
 #include "redismodule.h"
 
 #include <string.h>
+#include <strings.h>
 #include <assert.h>
 #include <unistd.h>
 
@@ -347,7 +348,11 @@ int blockonkeys_popall(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     return REDISMODULE_OK;
 }
 
-/* A module equivalent of LPUSH */
+/* BLOCKONKEYS.LPUSH key val [val ..]
+ * BLOCKONKEYS.LPUSH_UNBLOCK key val [val ..]
+ *
+ * A module equivalent of LPUSH. If the name LPUSH_UNBLOCK is used,
+ * RM_SignalKeyAsReady() is also called. */
 int blockonkeys_lpush(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     if (argc < 3)
         return RedisModule_WrongArity(ctx);
@@ -366,9 +371,83 @@ int blockonkeys_lpush(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
         }
     }
     RedisModule_CloseKey(key);
+
+    /* signal key as ready if the command is lpush_unblock */
+    size_t len;
+    const char *str = RedisModule_StringPtrLen(argv[0], &len);
+    if (!strncasecmp(str, "blockonkeys.lpush_unblock", len)) {
+        RedisModule_SignalKeyAsReady(ctx, argv[1]);
+    }
     return RedisModule_ReplyWithSimpleString(ctx, "OK");
 }
 
+/* Callback for the BLOCKONKEYS.BLPOPN command */
+int blockonkeys_blpopn_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argc);
+    long long n;
+    RedisModule_StringToLongLong(argv[2], &n);
+    RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE);
+    int result;
+    if (RedisModule_KeyType(key) == REDISMODULE_KEYTYPE_LIST &&
+        RedisModule_ValueLength(key) >= (size_t)n) {
+        RedisModule_ReplyWithArray(ctx, n);
+        for (long i = 0; i < n; i++) {
+            RedisModuleString *elem = RedisModule_ListPop(key, REDISMODULE_LIST_HEAD);
+            RedisModule_ReplyWithString(ctx, elem);
+            RedisModule_FreeString(ctx, elem);
+        }
+        result = REDISMODULE_OK;
+    } else if (RedisModule_KeyType(key) == REDISMODULE_KEYTYPE_LIST ||
+               RedisModule_KeyType(key) == REDISMODULE_KEYTYPE_EMPTY) {
+        /* continue blocking */
+        result = REDISMODULE_ERR;
+    } else {
+        result = RedisModule_ReplyWithError(ctx, REDISMODULE_ERRORMSG_WRONGTYPE);
+    }
+    RedisModule_CloseKey(key);
+    return result;
+}
+
+int blockonkeys_blpopn_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+    return RedisModule_ReplyWithError(ctx, "ERR Timeout");
+}
+
+/* BLOCKONKEYS.BLPOPN key N
+ *
+ * Blocks until key has N elements and then pops them or fails after 3 seconds.
+ */
+int blockonkeys_blpopn(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc < 3) return RedisModule_WrongArity(ctx);
+
+    long long n;
+    if (RedisModule_StringToLongLong(argv[2], &n) != REDISMODULE_OK) {
+        return RedisModule_ReplyWithError(ctx, "ERR Invalid N");
+    }
+
+    RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE);
+    int keytype = RedisModule_KeyType(key);
+    if (keytype != REDISMODULE_KEYTYPE_EMPTY &&
+        keytype != REDISMODULE_KEYTYPE_LIST) {
+        RedisModule_ReplyWithError(ctx, REDISMODULE_ERRORMSG_WRONGTYPE);
+    } else if (keytype == REDISMODULE_KEYTYPE_LIST &&
+               RedisModule_ValueLength(key) >= (size_t)n) {
+        RedisModule_ReplyWithArray(ctx, n);
+        for (long i = 0; i < n; i++) {
+            RedisModuleString *elem = RedisModule_ListPop(key, REDISMODULE_LIST_HEAD);
+            RedisModule_ReplyWithString(ctx, elem);
+            RedisModule_FreeString(ctx, elem);
+        }
+    } else {
+        RedisModule_BlockClientOnKeys(ctx, blockonkeys_blpopn_reply_callback,
+                                      blockonkeys_blpopn_timeout_callback,
+                                      NULL, 3000, &argv[1], 1, NULL);
+    }
+    RedisModule_CloseKey(key);
+    return REDISMODULE_OK;
+}
+
 int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     REDISMODULE_NOT_USED(argv);
     REDISMODULE_NOT_USED(argc);
@@ -413,5 +492,13 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
                                   "", 1, 1, 1) == REDISMODULE_ERR)
         return REDISMODULE_ERR;
 
+    if (RedisModule_CreateCommand(ctx, "blockonkeys.lpush_unblock", blockonkeys_lpush,
+                                  "", 1, 1, 1) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx, "blockonkeys.blpopn", blockonkeys_blpopn,
+                                  "", 1, 1, 1) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
     return REDISMODULE_OK;
 }
diff --git a/tests/unit/moduleapi/blockonkeys.tcl b/tests/unit/moduleapi/blockonkeys.tcl
index 00041674f..75191b3c7 100644
--- a/tests/unit/moduleapi/blockonkeys.tcl
+++ b/tests/unit/moduleapi/blockonkeys.tcl
@@ -216,4 +216,24 @@ start_server {tags {"modules"}} {
         assert_equal {k 42} [$rd read]
         $rd close
     }
+
+    test {Module unblocks module blocked on non-empty list} {
+        r del k
+        r lpush k aa
+        # Module client blocks to pop 5 elements from list
+        set rd [redis_deferring_client]
+        $rd blockonkeys.blpopn k 5
+        # Wait until client is actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Client is not blocked"
+        }
+        # Check that RM_SignalKeyAsReady() can wake up BLPOPN
+        r blockonkeys.lpush_unblock k bb cc ;# Not enough elements for BLPOPN
+        r lpush k dd ee ff                  ;# Doesn't unblock module
+        r blockonkeys.lpush_unblock k gg    ;# Unblocks module
+        assert_equal {gg ff ee dd cc} [$rd read]
+        $rd close
+    }
 }

From f225891526ff621504c8985b6d80b374d341686f Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sat, 23 Jan 2021 21:53:58 +0200
Subject: [PATCH 0997/1098] Fix recent test failures (#8386)

1. Valgrind leak in a recent change in a module api test
2. Increase treshold of a RESTORE TTL test
3. Change assertions to use assert_range which prints the values
---
 tests/modules/blockonkeys.c | 3 ++-
 tests/unit/dump.tcl         | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/modules/blockonkeys.c b/tests/modules/blockonkeys.c
index 2b7b614a9..b7ab977e9 100644
--- a/tests/modules/blockonkeys.c
+++ b/tests/modules/blockonkeys.c
@@ -66,7 +66,8 @@ int get_fsl(RedisModuleCtx *ctx, RedisModuleString *keyname, int mode, int creat
         RedisModule_CloseKey(key);
         if (reply_on_failure)
             RedisModule_ReplyWithError(ctx, REDISMODULE_ERRORMSG_WRONGTYPE);
-        RedisModule_Call(ctx, "INCR", "c", "fsl_wrong_type");
+        RedisModuleCallReply *reply = RedisModule_Call(ctx, "INCR", "c", "fsl_wrong_type");
+        RedisModule_FreeCallReply(reply);
         return 0;
     }
 
diff --git a/tests/unit/dump.tcl b/tests/unit/dump.tcl
index affce65e5..d43820ae3 100644
--- a/tests/unit/dump.tcl
+++ b/tests/unit/dump.tcl
@@ -12,7 +12,7 @@ start_server {tags {"dump"}} {
         r del foo
         r restore foo 5000 $encoded
         set ttl [r pttl foo]
-        assert {$ttl >= 3000 && $ttl <= 5000}
+        assert_range $ttl 3000 5000
         r get foo
     } {bar}
 
@@ -22,7 +22,7 @@ start_server {tags {"dump"}} {
         r del foo
         r restore foo 2569591501 $encoded
         set ttl [r pttl foo]
-        assert {$ttl >= (2569591501-3000) && $ttl <= 2569591501}
+        assert_range $ttl (2569591501-3000) 2569591501
         r get foo
     } {bar}
     
@@ -33,7 +33,7 @@ start_server {tags {"dump"}} {
         set now [clock milliseconds]
         r restore foo [expr $now+3000] $encoded absttl
         set ttl [r pttl foo]
-        assert {$ttl >= 2900 && $ttl <= 3100}
+        assert_range $ttl 2000 3100
         r get foo
     } {bar}
 

From 437e2583848ff3fb85af166a381842aa19fe076a Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 25 Jan 2021 11:05:25 +0200
Subject: [PATCH 0998/1098] Fix rare test failures due to
 repl-ping-replica-period (#8393)

some tests use attach_to_replication_stream to watch what's propagated
to replicas, but in some cases the periodic ping may slip in and fail
the test.
we disable that ping by setting the period to once an hour (tests should
not run for that long).

other change is so that the next time this oom-score-adj test fails,
we'll see the value (assert_equals prints it)
---
 tests/test_helper.tcl        | 2 ++
 tests/unit/oom-score-adj.tcl | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 4bef921ff..f947c024a 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -717,6 +717,7 @@ if {[llength $filtered_tests] < [llength $::all_tests]} {
 }
 
 proc attach_to_replication_stream {} {
+    r config set repl-ping-replica-period 3600
     if {$::tls} {
         set s [::tls::socket [srv 0 "host"] [srv 0 "port"]]
     } else {
@@ -774,6 +775,7 @@ proc assert_replication_stream {s patterns} {
 
 proc close_replication_stream {s} {
     close $s
+    r config set repl-ping-replica-period 10
 }
 
 # With the parallel test running multiple Redis instances at the same time
diff --git a/tests/unit/oom-score-adj.tcl b/tests/unit/oom-score-adj.tcl
index cf671fe6a..efa61b759 100644
--- a/tests/unit/oom-score-adj.tcl
+++ b/tests/unit/oom-score-adj.tcl
@@ -39,7 +39,7 @@ if {$system_name eq {linux}} {
             r bgsave
 
             set child_pid [get_child_pid 0]
-            assert {[get_oom_score_adj $child_pid] == [expr $base + 30]}
+            assert_equal [get_oom_score_adj $child_pid] [expr $base + 30]
         }
 
         # Failed oom-score-adj tests can only run unprivileged

From 951d25ff6625e0555038687699b9d9a15dd4e35d Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <276441700@qq.com>
Date: Tue, 26 Jan 2021 09:45:53 +0800
Subject: [PATCH 0999/1098] latency: add expire-del event (#8394)

---
 src/expire.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/expire.c b/src/expire.c
index d2945524c..f79510817 100644
--- a/src/expire.c
+++ b/src/expire.c
@@ -53,15 +53,19 @@
  * to the function to avoid too many gettimeofday() syscalls. */
 int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) {
     long long t = dictGetSignedIntegerVal(de);
+    mstime_t expire_latency;
     if (now > t) {
         sds key = dictGetKey(de);
         robj *keyobj = createStringObject(key,sdslen(key));
 
         propagateExpire(db,keyobj,server.lazyfree_lazy_expire);
+        latencyStartMonitor(expire_latency);
         if (server.lazyfree_lazy_expire)
             dbAsyncDelete(db,keyobj);
         else
             dbSyncDelete(db,keyobj);
+        latencyEndMonitor(expire_latency);
+        latencyAddSampleIfNeeded("expire-del",expire_latency);
         notifyKeyspaceEvent(NOTIFY_EXPIRED,
             "expired",keyobj,db->id);
         signalModifiedKey(NULL, db, keyobj);

From 70789cf4bb15c7d1552c2eff32edfe7b608589ee Mon Sep 17 00:00:00 2001
From: Vladimir Maksimovski <ReaLNeroM@users.noreply.github.com>
Date: Tue, 26 Jan 2021 05:58:03 +0100
Subject: [PATCH 1000/1098] Remove check leading to duplicate branches (#8398)

Remove check leading to duplicate branches and unused withscores parameter
---
 src/t_zset.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/t_zset.c b/src/t_zset.c
index eb4f274a3..bacac1c8d 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -3073,8 +3073,8 @@ void zrevrangeCommand(client *c) {
 
 /* This command implements ZRANGEBYSCORE, ZREVRANGEBYSCORE. */
 void genericZrangebyscoreCommand(zrange_result_handler *handler,
-    zrangespec *range, robj *zobj, int withscores, long offset,
-    long limit, int reverse) {
+    zrangespec *range, robj *zobj, long offset, long limit, 
+    int reverse) {
 
     client *c = handler->client;
     unsigned long rangelen = 0;
@@ -3174,8 +3174,7 @@ void genericZrangebyscoreCommand(zrange_result_handler *handler,
             }
 
             rangelen++;
-            handler->emitResultFromCBuffer(handler, ln->ele, sdslen(ln->ele),
-              ((withscores) ? ln->score : ln->score));
+            handler->emitResultFromCBuffer(handler, ln->ele, sdslen(ln->ele), ln->score);
 
             /* Move to next node */
             if (reverse) {
@@ -3626,8 +3625,8 @@ void zrangeGenericCommand(zrange_result_handler *handler, int argc_start, int st
         break;
 
     case ZRANGE_SCORE:
-        genericZrangebyscoreCommand(handler, &range, zobj, opt_withscores || store,
-            opt_offset, opt_limit, direction == ZRANGE_DIRECTION_REVERSE);
+        genericZrangebyscoreCommand(handler, &range, zobj, opt_offset,
+            opt_limit, direction == ZRANGE_DIRECTION_REVERSE);
         break;
 
     case ZRANGE_LEX:

From 1aad55b66fac0f8b7a855cafc1ee80ef37a80414 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Tue, 26 Jan 2021 02:31:54 -0500
Subject: [PATCH 1001/1098] Sentinel: Fix Config Dependency and Rewrite
 Sequence (#8271)

This commit fixes a well known and an annoying issue in Sentinel mode.

Cause of this issue:
Currently, Redis rewrite process works well in server mode, however in sentinel mode,
the sentinel config has variant semantics for different configurations, in example configuration
https://github.com/redis/redis/blob/unstable/sentinel.conf, we put comments on these.
However the rewrite process only treat the sentinel config as a single option. During rewrite
process, it will mess up with the lines and comments.

Approaches:
In order to solve this issue, we need to differentiate different subconfig options in sentinel separately,
for example, sentinel monitor <master-name> <ip> <redis-port> <quorum>
we can treat it as sentinel monitor option, instead of the sentinel option.

This commit also fixes the dependency issue when putting configurations in sentinel.conf.
For example before this commit,we must put
`sentinel monitor <master-name> <ip> <redis-port> <quorum>` before
`sentinel auth-pass <master-name> <password>` for a single master,
otherwise the server cannot start and will return error. This commit fixes this issue, as long as
the monitoring master was configured, no matter the sequence is, the sentinel can start and run properly.
---
 src/config.c                                  |  14 +-
 src/sentinel.c                                | 230 ++++++++++++++++--
 src/server.c                                  |   1 +
 src/server.h                                  |  18 ++
 tests/instances.tcl                           |  11 +-
 tests/sentinel/run.tcl                        |   2 +-
 tests/sentinel/tests/00-base.tcl              |   2 +-
 tests/sentinel/tests/includes/sentinel.conf   |  11 +
 .../tests/includes/start-init-tests.tcl       |  18 ++
 9 files changed, 278 insertions(+), 29 deletions(-)
 create mode 100644 tests/sentinel/tests/includes/sentinel.conf
 create mode 100644 tests/sentinel/tests/includes/start-init-tests.tcl

diff --git a/src/config.c b/src/config.c
index ec8867342..5db08f8bc 100644
--- a/src/config.c
+++ b/src/config.c
@@ -624,8 +624,7 @@ void loadServerConfigFromString(char *config) {
                     err = "sentinel directive while not in sentinel mode";
                     goto loaderr;
                 }
-                err = sentinelHandleConfiguration(argv+1,argc-1);
-                if (err) goto loaderr;
+                queueSentinelConfig(argv+1,argc-1,linenum,lines[i]);
             }
         } else {
             err = "Bad directive or wrong number of arguments"; goto loaderr;
@@ -1221,7 +1220,16 @@ struct rewriteConfigState *rewriteConfigReadOldFile(char *path) {
             sdsfree(argv[0]);
             argv[0] = alt;
         }
-        rewriteConfigAddLineNumberToOption(state,argv[0],linenum);
+        /* If this is sentinel config, we use sentinel "sentinel <config>" as option 
+            to avoid messing up the sequence. */
+        if (server.sentinel_mode && argc > 1 && !strcasecmp(argv[0],"sentinel")) {
+            sds sentinelOption = sdsempty();
+            sentinelOption = sdscatfmt(sentinelOption,"%S %S",argv[0],argv[1]);
+            rewriteConfigAddLineNumberToOption(state,sentinelOption,linenum);
+            sdsfree(sentinelOption);
+        } else {
+            rewriteConfigAddLineNumberToOption(state,argv[0],linenum);
+        }
         sdsfreesplitres(argv,argc);
     }
     fclose(fp);
diff --git a/src/sentinel.c b/src/sentinel.c
index 75030df78..7f79e702a 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -474,6 +474,20 @@ struct redisCommand sentinelcmds[] = {
     {"command",commandCommand,-1, "random @connection", 0,NULL,0,0,0,0,0,0}
 };
 
+/* this array is used for sentinel config lookup, which need to be loaded
+ * before monitoring masters config to avoid dependency issues */
+const char *preMonitorCfgName[] = { 
+    "announce-ip",
+    "announce-port",
+    "deny-scripts-reconfig",
+    "sentinel-user",
+    "sentinel-pass",
+    "current-epoch",
+    "myid",
+    "resolve-hostnames",
+    "announce-hostnames"
+};
+
 /* This function overwrites a few normal Redis config default with Sentinel
  * specific defaults. */
 void initSentinelConfig(void) {
@@ -481,6 +495,8 @@ void initSentinelConfig(void) {
     server.protected_mode = 0; /* Sentinel must be exposed. */
 }
 
+void freeSentinelLoadQueueEntry(void *item);
+
 /* Perform the Sentinel mode initialization. */
 void initSentinel(void) {
     unsigned int j;
@@ -520,6 +536,7 @@ void initSentinel(void) {
     sentinel.sentinel_auth_pass = NULL;
     sentinel.sentinel_auth_user = NULL;
     memset(sentinel.myid,0,sizeof(sentinel.myid));
+    server.sentinel_config = NULL;
 }
 
 /* This function gets called when the server is in Sentinel mode, started,
@@ -1669,7 +1686,137 @@ const char *sentinelCheckCreateInstanceErrors(int role) {
     }
 }
 
+/* init function for server.sentinel_config */
+void initializeSentinelConfig() {
+    server.sentinel_config = zmalloc(sizeof(struct sentinelConfig));
+    server.sentinel_config->monitor_cfg = listCreate();
+    server.sentinel_config->pre_monitor_cfg = listCreate();
+    server.sentinel_config->post_monitor_cfg = listCreate();
+    listSetFreeMethod(server.sentinel_config->monitor_cfg,freeSentinelLoadQueueEntry);
+    listSetFreeMethod(server.sentinel_config->pre_monitor_cfg,freeSentinelLoadQueueEntry);
+    listSetFreeMethod(server.sentinel_config->post_monitor_cfg,freeSentinelLoadQueueEntry);
+}
+
+/* destroy function for server.sentinel_config */
+void freeSentinelConfig() {
+    /* release these three config queues since we will not use it anymore */
+    listRelease(server.sentinel_config->pre_monitor_cfg);
+    listRelease(server.sentinel_config->monitor_cfg);
+    listRelease(server.sentinel_config->post_monitor_cfg);
+    zfree(server.sentinel_config);
+    server.sentinel_config = NULL;
+}
+
+/* Search config name in pre monitor config name array, return 1 if found,
+ * 0 if not found. */
+int searchPreMonitorCfgName(const char *name) {
+    for (unsigned int i = 0; i < sizeof(preMonitorCfgName)/sizeof(preMonitorCfgName[0]); i++) {
+        if (!strcasecmp(preMonitorCfgName[i],name)) return 1;
+    }
+    return 0;
+}
+
+/* free method for sentinelLoadQueueEntry when release the list */
+void freeSentinelLoadQueueEntry(void *item) {
+    struct sentinelLoadQueueEntry *entry = item;
+    sdsfreesplitres(entry->argv,entry->argc);
+    sdsfree(entry->line);
+    zfree(entry);
+}
+
+/* This function is used for queuing sentinel configuration, the main
+ * purpose of this function is to delay parsing the sentinel config option
+ * in order to avoid the order dependent issue from the config. */
+void queueSentinelConfig(sds *argv, int argc, int linenum, sds line) {
+    int i;
+    struct sentinelLoadQueueEntry *entry;
+
+    /* initialize sentinel_config for the first call */
+    if (server.sentinel_config == NULL) initializeSentinelConfig();
+
+    entry = zmalloc(sizeof(struct sentinelLoadQueueEntry));
+    entry->argv = zmalloc(sizeof(char*)*argc);
+    entry->argc = argc;
+    entry->linenum = linenum;
+    entry->line = sdsdup(line);
+    for (i = 0; i < argc; i++) {
+        entry->argv[i] = sdsdup(argv[i]);
+    }
+    /*  Separate config lines with pre monitor config, monitor config and
+     *  post monitor config, in order to parsing config dependencies
+     *  correctly. */
+    if (!strcasecmp(argv[0],"monitor")) {
+        listAddNodeTail(server.sentinel_config->monitor_cfg,entry);
+    } else if (searchPreMonitorCfgName(argv[0])) {
+        listAddNodeTail(server.sentinel_config->pre_monitor_cfg,entry);
+    } else{
+        listAddNodeTail(server.sentinel_config->post_monitor_cfg,entry);
+    }
+}
+
+/* This function is used for loading the sentinel configuration from
+ * pre_monitor_cfg, monitor_cfg and post_monitor_cfg list */
+void loadSentinelConfigFromQueue(void) {
+    const char *err = NULL;
+    listIter li;
+    listNode *ln;
+    int linenum = 0;
+    sds line = NULL;
+
+    /* if there is no sentinel_config entry, we can return immediately */
+    if (server.sentinel_config == NULL) return;
+
+    /* loading from pre monitor config queue first to avoid dependency issues */
+    listRewind(server.sentinel_config->pre_monitor_cfg,&li);
+    while((ln = listNext(&li))) {
+        struct sentinelLoadQueueEntry *entry = ln->value;
+        err = sentinelHandleConfiguration(entry->argv,entry->argc);
+        if (err) {
+            linenum = entry->linenum;
+            line = entry->line;
+            goto loaderr;
+        }
+    }
+
+    /* loading from monitor config queue */
+    listRewind(server.sentinel_config->monitor_cfg,&li);
+    while((ln = listNext(&li))) {
+        struct sentinelLoadQueueEntry *entry = ln->value;
+        err = sentinelHandleConfiguration(entry->argv,entry->argc);
+        if (err) {
+            linenum = entry->linenum;
+            line = entry->line;
+            goto loaderr;
+        }
+    }
+
+    /* loading from the post monitor config queue */
+    listRewind(server.sentinel_config->post_monitor_cfg,&li);
+    while((ln = listNext(&li))) {
+        struct sentinelLoadQueueEntry *entry = ln->value;
+        err = sentinelHandleConfiguration(entry->argv,entry->argc);
+        if (err) {
+            linenum = entry->linenum;
+            line = entry->line;
+            goto loaderr;
+        }
+    }
+
+    /* free sentinel_config when config loading is finished */
+    freeSentinelConfig();
+    return;
+
+loaderr:
+    fprintf(stderr, "\n*** FATAL CONFIG FILE ERROR (Redis %s) ***\n",
+        REDIS_VERSION);
+    fprintf(stderr, "Reading the configuration file, at line %d\n", linenum);
+    fprintf(stderr, ">>> '%s'\n", line);
+    fprintf(stderr, "%s\n", err);
+    exit(1);
+}
+
 const char *sentinelHandleConfiguration(char **argv, int argc) {
+
     sentinelRedisInstance *ri;
 
     if (!strcasecmp(argv[0],"monitor") && argc == 5) {
@@ -1830,12 +1977,12 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
 
     /* sentinel unique ID. */
     line = sdscatprintf(sdsempty(), "sentinel myid %s", sentinel.myid);
-    rewriteConfigRewriteLine(state,"sentinel",line,1);
+    rewriteConfigRewriteLine(state,"sentinel myid",line,1);
 
     /* sentinel deny-scripts-reconfig. */
     line = sdscatprintf(sdsempty(), "sentinel deny-scripts-reconfig %s",
         sentinel.deny_scripts_reconfig ? "yes" : "no");
-    rewriteConfigRewriteLine(state,"sentinel",line,
+    rewriteConfigRewriteLine(state,"sentinel deny-scripts-reconfig",line,
         sentinel.deny_scripts_reconfig != SENTINEL_DEFAULT_DENY_SCRIPTS_RECONFIG);
 
     /* For every master emit a "sentinel monitor" config entry. */
@@ -1850,14 +1997,16 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
         line = sdscatprintf(sdsempty(),"sentinel monitor %s %s %d %d",
             master->name, master_addr->ip, master_addr->port,
             master->quorum);
-        rewriteConfigRewriteLine(state,"sentinel",line,1);
+        rewriteConfigRewriteLine(state,"sentinel monitor",line,1);
+        /* rewriteConfigMarkAsProcessed is handled after the loop */
 
         /* sentinel down-after-milliseconds */
         if (master->down_after_period != SENTINEL_DEFAULT_DOWN_AFTER) {
             line = sdscatprintf(sdsempty(),
                 "sentinel down-after-milliseconds %s %ld",
                 master->name, (long) master->down_after_period);
-            rewriteConfigRewriteLine(state,"sentinel",line,1);
+            rewriteConfigRewriteLine(state,"sentinel down-after-milliseconds",line,1);
+            /* rewriteConfigMarkAsProcessed is handled after the loop */
         }
 
         /* sentinel failover-timeout */
@@ -1865,7 +2014,9 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
             line = sdscatprintf(sdsempty(),
                 "sentinel failover-timeout %s %ld",
                 master->name, (long) master->failover_timeout);
-            rewriteConfigRewriteLine(state,"sentinel",line,1);
+            rewriteConfigRewriteLine(state,"sentinel failover-timeout",line,1);
+            /* rewriteConfigMarkAsProcessed is handled after the loop */
+
         }
 
         /* sentinel parallel-syncs */
@@ -1873,7 +2024,8 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
             line = sdscatprintf(sdsempty(),
                 "sentinel parallel-syncs %s %d",
                 master->name, master->parallel_syncs);
-            rewriteConfigRewriteLine(state,"sentinel",line,1);
+            rewriteConfigRewriteLine(state,"sentinel parallel-syncs",line,1);
+            /* rewriteConfigMarkAsProcessed is handled after the loop */
         }
 
         /* sentinel notification-script */
@@ -1881,7 +2033,8 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
             line = sdscatprintf(sdsempty(),
                 "sentinel notification-script %s %s",
                 master->name, master->notification_script);
-            rewriteConfigRewriteLine(state,"sentinel",line,1);
+            rewriteConfigRewriteLine(state,"sentinel notification-script",line,1);
+            /* rewriteConfigMarkAsProcessed is handled after the loop */
         }
 
         /* sentinel client-reconfig-script */
@@ -1889,7 +2042,8 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
             line = sdscatprintf(sdsempty(),
                 "sentinel client-reconfig-script %s %s",
                 master->name, master->client_reconfig_script);
-            rewriteConfigRewriteLine(state,"sentinel",line,1);
+            rewriteConfigRewriteLine(state,"sentinel client-reconfig-script",line,1);
+            /* rewriteConfigMarkAsProcessed is handled after the loop */
         }
 
         /* sentinel auth-pass & auth-user */
@@ -1897,27 +2051,32 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
             line = sdscatprintf(sdsempty(),
                 "sentinel auth-pass %s %s",
                 master->name, master->auth_pass);
-            rewriteConfigRewriteLine(state,"sentinel",line,1);
+            rewriteConfigRewriteLine(state,"sentinel auth-pass",line,1);
+            /* rewriteConfigMarkAsProcessed is handled after the loop */
         }
 
         if (master->auth_user) {
             line = sdscatprintf(sdsempty(),
                 "sentinel auth-user %s %s",
                 master->name, master->auth_user);
-            rewriteConfigRewriteLine(state,"sentinel",line,1);
+            rewriteConfigRewriteLine(state,"sentinel auth-user",line,1);
+            /* rewriteConfigMarkAsProcessed is handled after the loop */
         }
 
         /* sentinel config-epoch */
         line = sdscatprintf(sdsempty(),
             "sentinel config-epoch %s %llu",
             master->name, (unsigned long long) master->config_epoch);
-        rewriteConfigRewriteLine(state,"sentinel",line,1);
+        rewriteConfigRewriteLine(state,"sentinel config-epoch",line,1);
+        /* rewriteConfigMarkAsProcessed is handled after the loop */
+
 
         /* sentinel leader-epoch */
         line = sdscatprintf(sdsempty(),
             "sentinel leader-epoch %s %llu",
             master->name, (unsigned long long) master->leader_epoch);
-        rewriteConfigRewriteLine(state,"sentinel",line,1);
+        rewriteConfigRewriteLine(state,"sentinel leader-epoch",line,1);
+        /* rewriteConfigMarkAsProcessed is handled after the loop */
 
         /* sentinel known-slave */
         di2 = dictGetIterator(master->slaves);
@@ -1937,7 +2096,8 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
             line = sdscatprintf(sdsempty(),
                 "sentinel known-replica %s %s %d",
                 master->name, slave_addr->ip, slave_addr->port);
-            rewriteConfigRewriteLine(state,"sentinel",line,1);
+            rewriteConfigRewriteLine(state,"sentinel known-replica",line,1);
+            /* rewriteConfigMarkAsProcessed is handled after the loop */
         }
         dictReleaseIterator(di2);
 
@@ -1949,7 +2109,8 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
             line = sdscatprintf(sdsempty(),
                 "sentinel known-sentinel %s %s %d %s",
                 master->name, ri->addr->ip, ri->addr->port, ri->runid);
-            rewriteConfigRewriteLine(state,"sentinel",line,1);
+            rewriteConfigRewriteLine(state,"sentinel known-sentinel",line,1);
+            /* rewriteConfigMarkAsProcessed is handled after the loop */
         }
         dictReleaseIterator(di2);
 
@@ -1961,7 +2122,8 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
             line = sdscatprintf(sdsempty(),
                 "sentinel rename-command %s %s %s",
                 master->name, oldname, newname);
-            rewriteConfigRewriteLine(state,"sentinel",line,1);
+            rewriteConfigRewriteLine(state,"sentinel rename-command",line,1);
+            /* rewriteConfigMarkAsProcessed is handled after the loop */
         }
         dictReleaseIterator(di2);
     }
@@ -1969,36 +2131,62 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
     /* sentinel current-epoch is a global state valid for all the masters. */
     line = sdscatprintf(sdsempty(),
         "sentinel current-epoch %llu", (unsigned long long) sentinel.current_epoch);
-    rewriteConfigRewriteLine(state,"sentinel",line,1);
+    rewriteConfigRewriteLine(state,"sentinel current-epoch",line,1);
 
     /* sentinel announce-ip. */
     if (sentinel.announce_ip) {
         line = sdsnew("sentinel announce-ip ");
         line = sdscatrepr(line, sentinel.announce_ip, sdslen(sentinel.announce_ip));
-        rewriteConfigRewriteLine(state,"sentinel",line,1);
+        rewriteConfigRewriteLine(state,"sentinel announce-ip",line,1);
+    } else {
+        rewriteConfigMarkAsProcessed(state,"sentinel announce-ip");
     }
 
     /* sentinel announce-port. */
     if (sentinel.announce_port) {
         line = sdscatprintf(sdsempty(),"sentinel announce-port %d",
                             sentinel.announce_port);
-        rewriteConfigRewriteLine(state,"sentinel",line,1);
+        rewriteConfigRewriteLine(state,"sentinel announce-port",line,1);
+    } else {
+        rewriteConfigMarkAsProcessed(state,"sentinel announce-port");
     }
 
     /* sentinel sentinel-user. */
     if (sentinel.sentinel_auth_user) {
         line = sdscatprintf(sdsempty(), "sentinel sentinel-user %s", sentinel.sentinel_auth_user);
-        rewriteConfigRewriteLine(state,"sentinel",line,1);
+        rewriteConfigRewriteLine(state,"sentinel sentinel-user",line,1);
+    } else {
+        rewriteConfigMarkAsProcessed(state,"sentinel sentinel-user");
     }
 
     /* sentinel sentinel-pass. */
     if (sentinel.sentinel_auth_pass) {
         line = sdscatprintf(sdsempty(), "sentinel sentinel-pass %s", sentinel.sentinel_auth_pass);
-        rewriteConfigRewriteLine(state,"sentinel",line,1);
+        rewriteConfigRewriteLine(state,"sentinel sentinel-pass",line,1);
+    } else {
+        rewriteConfigMarkAsProcessed(state,"sentinel sentinel-pass");  
     }
 
-
     dictReleaseIterator(di);
+
+    /* NOTE: the purpose here is in case due to the state change, the config rewrite 
+     does not handle the configs, however, previously the config was set in the config file, 
+     rewriteConfigMarkAsProcessed should be put here to mark it as processed in order to 
+     delete the old config entry.
+    */
+    rewriteConfigMarkAsProcessed(state,"sentinel monitor");
+    rewriteConfigMarkAsProcessed(state,"sentinel down-after-milliseconds");
+    rewriteConfigMarkAsProcessed(state,"sentinel failover-timeout");
+    rewriteConfigMarkAsProcessed(state,"sentinel parallel-syncs");
+    rewriteConfigMarkAsProcessed(state,"sentinel notification-script");
+    rewriteConfigMarkAsProcessed(state,"sentinel client-reconfig-script");
+    rewriteConfigMarkAsProcessed(state,"sentinel auth-pass");
+    rewriteConfigMarkAsProcessed(state,"sentinel auth-user");
+    rewriteConfigMarkAsProcessed(state,"sentinel config-epoch");
+    rewriteConfigMarkAsProcessed(state,"sentinel leader-epoch");
+    rewriteConfigMarkAsProcessed(state,"sentinel known-replica");
+    rewriteConfigMarkAsProcessed(state,"sentinel known-sentinel");
+    rewriteConfigMarkAsProcessed(state,"sentinel rename-command");
 }
 
 /* This function uses the config rewriting Redis engine in order to persist
diff --git a/src/server.c b/src/server.c
index dc661b6e2..bdedf72d0 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5851,6 +5851,7 @@ int main(int argc, char **argv) {
             exit(1);
         }
         loadServerConfig(server.configfile, config_from_stdin, options);
+        if (server.sentinel_mode) loadSentinelConfigFromQueue();
         sdsfree(options);
     }
 
diff --git a/src/server.h b/src/server.h
index cb103aca8..8f1ca50b1 100644
--- a/src/server.h
+++ b/src/server.h
@@ -942,6 +942,19 @@ struct moduleLoadQueueEntry {
     robj **argv;
 };
 
+struct sentinelLoadQueueEntry {
+    int argc;
+    sds *argv;
+    int linenum;
+    sds line;
+};
+
+struct sentinelConfig {
+    list *pre_monitor_cfg;
+    list *monitor_cfg;
+    list *post_monitor_cfg;
+};
+
 struct sharedObjectsStruct {
     robj *crlf, *ok, *err, *emptybulk, *czero, *cone, *pong, *space,
     *colon, *queued, *null[4], *nullarray[4], *emptymap[4], *emptyset[4],
@@ -1556,6 +1569,8 @@ struct redisServer {
     char *bio_cpulist; /* cpu affinity list of bio thread. */
     char *aof_rewrite_cpulist; /* cpu affinity list of aof rewrite process. */
     char *bgsave_cpulist; /* cpu affinity list of bgsave process. */
+    /* Sentinel config */
+    struct sentinelConfig *sentinel_config; /* sentinel config to load at startup time. */
 };
 
 typedef struct pubsubPattern {
@@ -2238,6 +2253,7 @@ void appendServerSaveParams(time_t seconds, int changes);
 void resetServerSaveParams(void);
 struct rewriteConfigState; /* Forward declaration to export API. */
 void rewriteConfigRewriteLine(struct rewriteConfigState *state, const char *option, sds line, int force);
+void rewriteConfigMarkAsProcessed(struct rewriteConfigState *state, const char *option);
 int rewriteConfig(char *path, int force_all);
 void initConfigValues();
 
@@ -2333,6 +2349,8 @@ void initSentinelConfig(void);
 void initSentinel(void);
 void sentinelTimer(void);
 const char *sentinelHandleConfiguration(char **argv, int argc);
+void queueSentinelConfig(sds *argv, int argc, int linenum, sds line);
+void loadSentinelConfigFromQueue(void);
 void sentinelIsRunning(void);
 
 /* redis-check-rdb & aof */
diff --git a/tests/instances.tcl b/tests/instances.tcl
index e35aaefa0..03af2408b 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -59,10 +59,9 @@ proc exec_instance {type dirname cfgfile} {
 }
 
 # Spawn a redis or sentinel instance, depending on 'type'.
-proc spawn_instance {type base_port count {conf {}}} {
+proc spawn_instance {type base_port count {conf {}} {base_conf_file ""}} {
     for {set j 0} {$j < $count} {incr j} {
         set port [find_available_port $base_port $::redis_port_count]
-
         # Create a directory for this instance.
         set dirname "${type}_${j}"
         lappend ::dirs $dirname
@@ -71,7 +70,13 @@ proc spawn_instance {type base_port count {conf {}}} {
 
         # Write the instance config file.
         set cfgfile [file join $dirname $type.conf]
-        set cfg [open $cfgfile w]
+        if {$base_conf_file ne ""} {
+            file copy -- $base_conf_file $cfgfile
+            set cfg [open $cfgfile a+]
+        } else {
+            set cfg [open $cfgfile w]
+        }
+
         if {$::tls} {
             puts $cfg "tls-port $port"
             puts $cfg "tls-replication yes"
diff --git a/tests/sentinel/run.tcl b/tests/sentinel/run.tcl
index 99362c193..c275aa762 100644
--- a/tests/sentinel/run.tcl
+++ b/tests/sentinel/run.tcl
@@ -10,7 +10,7 @@ set ::tlsdir "../../tls"
 
 proc main {} {
     parse_options
-    spawn_instance sentinel $::sentinel_base_port $::instances_count [list "sentinel deny-scripts-reconfig no"]
+    spawn_instance sentinel $::sentinel_base_port $::instances_count [list "sentinel deny-scripts-reconfig no"] "../tests/includes/sentinel.conf"
     spawn_instance redis $::redis_base_port $::instances_count
     run_tests
     cleanup
diff --git a/tests/sentinel/tests/00-base.tcl b/tests/sentinel/tests/00-base.tcl
index 7fb1a8bef..75baf9817 100644
--- a/tests/sentinel/tests/00-base.tcl
+++ b/tests/sentinel/tests/00-base.tcl
@@ -1,5 +1,5 @@
 # Check the basic monitoring and failover capabilities.
-
+source "../tests/includes/start-init-tests.tcl"
 source "../tests/includes/init-tests.tcl"
 
 if {$::simulate_error} {
diff --git a/tests/sentinel/tests/includes/sentinel.conf b/tests/sentinel/tests/includes/sentinel.conf
new file mode 100644
index 000000000..94f2804a4
--- /dev/null
+++ b/tests/sentinel/tests/includes/sentinel.conf
@@ -0,0 +1,11 @@
+# assume master is down after being unresponsive for 20s
+sentinel down-after-milliseconds setmaster 20000
+# reconfigure one slave at a time
+sentinel parallel-syncs setmaster 2
+# wait for 4m before assuming failover went wrong
+sentinel failover-timeout setmaster 240000
+# monitoring set
+sentinel monitor setmaster 10.0.0.1 30000 2
+
+
+
diff --git a/tests/sentinel/tests/includes/start-init-tests.tcl b/tests/sentinel/tests/includes/start-init-tests.tcl
new file mode 100644
index 000000000..b0523506a
--- /dev/null
+++ b/tests/sentinel/tests/includes/start-init-tests.tcl
@@ -0,0 +1,18 @@
+test "(start-init) Flush config and compare rewrite config file lines" {
+    foreach_sentinel_id id {
+        assert_match "OK" [S $id SENTINEL FLUSHCONFIG]
+        set file1 ../tests/includes/sentinel.conf
+        set file2 [file join "sentinel_${id}" "sentinel.conf"] 
+        set fh1 [open $file1 r]
+        set fh2 [open $file2 r]
+        while {[gets $fh1 line1]} {
+            if {[gets $fh2 line2]} {
+                assert [string equal $line1 $line2]
+            } else {
+                fail "sentinel config file rewrite sequence changed"
+            }
+        }
+        close $fh1
+        close $fh2  
+    }
+}
\ No newline at end of file

From 9e56d3969a3d19ef73d7042f91ba34bef39ed189 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Tue, 26 Jan 2021 17:55:32 +0200
Subject: [PATCH 1002/1098] Add tests for RESP3 responce of ZINTER and ZRANGE
 (#8391)

It was confusing as to why these don't return a map type.
the reason is that order matters, so we need to make sure the client
library knows to respect it.
Added comments in the implementation and tests to cover it.
---
 src/t_zset.c             |  6 ++++++
 tests/support/redis.tcl  |  1 +
 tests/unit/type/zset.tcl | 12 ++++++++++++
 3 files changed, 19 insertions(+)

diff --git a/src/t_zset.c b/src/t_zset.c
index bacac1c8d..6851ac86c 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -2751,6 +2751,9 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in
         unsigned long length = dstzset->zsl->length;
         zskiplist *zsl = dstzset->zsl;
         zskiplistNode *zn = zsl->header->level[0].forward;
+        /* In case of WITHSCORES, respond with a single array in RESP2, and
+         * nested arrays in RESP3. We can't use a map response type since the
+         * client library needs to know to respect the order. */
         if (withscores && c->resp == 2)
             addReplyArrayLen(c, length*2);
         else
@@ -2868,6 +2871,9 @@ static void zrangeResultEmitLongLongToClient(zrange_result_handler *handler,
 static void zrangeResultFinalizeClient(zrange_result_handler *handler,
     size_t result_count)
 {
+    /* In case of WITHSCORES, respond with a single array in RESP2, and
+     * nested arrays in RESP3. We can't use a map response type since the
+     * client library needs to know to respect the order. */
     if (handler->withscores && (handler->client->resp == 2)) {
         result_count *= 2;
     }
diff --git a/tests/support/redis.tcl b/tests/support/redis.tcl
index 8eca2ac32..54b49920d 100644
--- a/tests/support/redis.tcl
+++ b/tests/support/redis.tcl
@@ -244,6 +244,7 @@ proc ::redis::redis_read_reply {id fd} {
         _ {redis_read_null $fd}
         : -
         + {redis_read_line $fd}
+        , {expr {double([redis_read_line $fd])}}
         - {return -code error [redis_read_line $fd]}
         $ {redis_bulk_read $fd}
         > -
diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl
index 8318ebb63..f3c952f0e 100644
--- a/tests/unit/type/zset.tcl
+++ b/tests/unit/type/zset.tcl
@@ -713,6 +713,12 @@ start_server {tags {"zset"}} {
             assert_equal {b 3 c 5} [r zinter 2 zseta zsetb withscores]
         }
 
+        test "ZINTER RESP3 - $encoding" {
+            r hello 3
+            assert_equal {{b 3.0} {c 5.0}} [r zinter 2 zseta zsetb withscores]
+        }
+        r hello 2
+
         test "ZINTERSTORE with weights - $encoding" {
             assert_equal 2 [r zinterstore zsetc 2 zseta zsetb weights 2 3]
             assert_equal {b 7 c 12} [r zrange zsetc 0 -1 withscores]
@@ -1481,6 +1487,12 @@ start_server {tags {"zset"}} {
         r zrange z2 0 -1 withscores
     } {a 1 b 2 c 3 d 4}
 
+    test {ZRANGESTORE RESP3} {
+        r hello 3
+        r zrange z2 0 -1 withscores
+    } {{a 1.0} {b 2.0} {c 3.0} {d 4.0}}
+    r hello 2
+
     test {ZRANGESTORE range} {
         set res [r zrangestore z2 z1 1 2]
         assert_equal $res 2

From ad7d4c6b709ce94ed9a2fa3a1fcf9b8cf7843eff Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuan21@baidu.com>
Date: Wed, 27 Jan 2021 23:13:10 +0800
Subject: [PATCH 1003/1098] Implement rdb-only replication (#8303)

In some scenarios, such as remote backup, we only want to get remote
redis server db snapshot. Currently, redis-cli acts as a replica and
sends SYNC to redis, but redis still accumulates replication buffer
in the replica client output buffer, that may result in using vast
memory, or failing to transfer RDB because of client-output-buffer-limit.
In this commit, we add 'replconf rdb-only 0|1', redis doesn't send
incremental replication buffer to them if they send 'replconf rdb-only 1',
so we can reduce used memory and improve success of getting RDB.
---
 src/redis-cli.c   |  6 ++++-
 src/replication.c | 57 +++++++++++++++++++++++++++++++++++++----------
 src/server.h      |  2 ++
 3 files changed, 52 insertions(+), 13 deletions(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 2e8984c71..ed3075317 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -6940,6 +6940,10 @@ void sendCapa() {
     sendReplconf("capa", "eof");
 }
 
+void sendRdbOnly(void) {
+    sendReplconf("rdb-only", "1");
+}
+
 /* Read raw bytes through a redisContext. The read operation is not greedy
  * and may not fill the buffer entirely.
  */
@@ -7137,7 +7141,6 @@ static void getRDB(clusterManagerNode *node) {
         node->context = NULL;
     fsync(fd);
     close(fd);
-    fprintf(stderr,"Transfer finished with success.\n");
     if (node) {
         sdsfree(filename);
         return;
@@ -8258,6 +8261,7 @@ int main(int argc, char **argv) {
     if (config.getrdb_mode) {
         if (cliConnect(0) == REDIS_ERR) exit(1);
         sendCapa();
+        sendRdbOnly();
         getRDB(NULL);
     }
 
diff --git a/src/replication.c b/src/replication.c
index 9fb19eaca..a9ec30e74 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -200,6 +200,16 @@ void feedReplicationBacklogWithObject(robj *o) {
     feedReplicationBacklog(p,len);
 }
 
+int canFeedReplicaReplBuffer(client *replica) {
+    /* Don't feed replicas that only want the RDB. */
+    if (replica->flags & CLIENT_REPL_RDBONLY) return 0;
+
+    /* Don't feed replicas that are still waiting for BGSAVE to start. */
+    if (replica->replstate == SLAVE_STATE_WAIT_BGSAVE_START) return 0;
+
+    return 1;
+}
+
 /* Propagate write commands to slaves, and populate the replication backlog
  * as well. This function is used if the instance is a master: we use
  * the commands received by our clients in order to create the replication
@@ -249,7 +259,8 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) {
         listRewind(slaves,&li);
         while((ln = listNext(&li))) {
             client *slave = ln->value;
-            if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START) continue;
+
+            if (!canFeedReplicaReplBuffer(slave)) continue;
             addReply(slave,selectcmd);
         }
 
@@ -290,8 +301,7 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) {
     while((ln = listNext(&li))) {
         client *slave = ln->value;
 
-        /* Don't feed slaves that are still waiting for BGSAVE to start. */
-        if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START) continue;
+        if (!canFeedReplicaReplBuffer(slave)) continue;
 
         /* Feed slaves that are waiting for the initial SYNC (so these commands
          * are queued in the output buffer until the initial SYNC completes),
@@ -363,8 +373,7 @@ void replicationFeedSlavesFromMasterStream(list *slaves, char *buf, size_t bufle
     while((ln = listNext(&li))) {
         client *slave = ln->value;
 
-        /* Don't feed slaves that are still waiting for BGSAVE to start. */
-        if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START) continue;
+        if (!canFeedReplicaReplBuffer(slave)) continue;
         addReplyProto(slave,buf,buflen);
     }
 }
@@ -799,14 +808,20 @@ void syncCommand(client *c) {
         listRewind(server.slaves,&li);
         while((ln = listNext(&li))) {
             slave = ln->value;
-            if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END) break;
+            /* If the client needs a buffer of commands, we can't use
+             * a replica without replication buffer. */
+            if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END &&
+                (!(slave->flags & CLIENT_REPL_RDBONLY) ||
+                 (c->flags & CLIENT_REPL_RDBONLY)))
+                break;
         }
         /* To attach this slave, we check that it has at least all the
          * capabilities of the slave that triggered the current BGSAVE. */
         if (ln && ((c->slave_capa & slave->slave_capa) == slave->slave_capa)) {
             /* Perfect, the server is already registering differences for
-             * another slave. Set the right state, and copy the buffer. */
-            copyClientOutputBuffer(c,slave);
+             * another slave. Set the right state, and copy the buffer.
+             * We don't copy buffer if clients don't want. */
+            if (!(c->flags & CLIENT_REPL_RDBONLY)) copyClientOutputBuffer(c,slave);
             replicationSetupSlaveForFullResync(c,slave->psync_initial_offset);
             serverLog(LL_NOTICE,"Waiting for end of BGSAVE for SYNC");
         } else {
@@ -925,6 +940,15 @@ void replconfCommand(client *c) {
              * to the slave. */
             if (server.masterhost && server.master) replicationSendAck();
             return;
+        } else if (!strcasecmp(c->argv[j]->ptr,"rdb-only")) {
+           /* REPLCONF RDB-ONLY is used to identify the client only wants
+            * RDB snapshot without replication buffer. */
+            long rdb_only = 0;
+            if (getRangeLongFromObjectOrReply(c,c->argv[j+1],
+                    0,1,&rdb_only,NULL) != C_OK)
+                return;
+            if (rdb_only == 1) c->flags |= CLIENT_REPL_RDBONLY;
+            else c->flags &= ~CLIENT_REPL_RDBONLY;
         } else {
             addReplyErrorFormat(c,"Unrecognized REPLCONF option: %s",
                 (char*)c->argv[j]->ptr);
@@ -939,19 +963,28 @@ void replconfCommand(client *c) {
  * we are finally ready to send the incremental stream of commands.
  *
  * It does a few things:
- *
- * 1) Put the slave in ONLINE state. Note that the function may also be called
+ * 1) Close the replica's connection async if it doesn't need replication
+ *    commands buffer stream, since it actually isn't a valid replica.
+ * 2) Put the slave in ONLINE state. Note that the function may also be called
  *    for a replicas that are already in ONLINE state, but having the flag
  *    repl_put_online_on_ack set to true: we still have to install the write
  *    handler in that case. This function will take care of that.
- * 2) Make sure the writable event is re-installed, since calling the SYNC
+ * 3) Make sure the writable event is re-installed, since calling the SYNC
  *    command disables it, so that we can accumulate output buffer without
  *    sending it to the replica.
- * 3) Update the count of "good replicas". */
+ * 4) Update the count of "good replicas". */
 void putSlaveOnline(client *slave) {
     slave->replstate = SLAVE_STATE_ONLINE;
     slave->repl_put_online_on_ack = 0;
     slave->repl_ack_time = server.unixtime; /* Prevent false timeout. */
+
+    if (slave->flags & CLIENT_REPL_RDBONLY) {
+        serverLog(LL_NOTICE,
+            "Close the connection with replica %s as RDB transfer is complete",
+            replicationGetSlaveName(slave));
+        freeClientAsync(slave);
+        return;
+    }
     if (connSetWriteHandler(slave->conn, sendReplyToClient) == C_ERR) {
         serverLog(LL_WARNING,"Unable to register writable event for replica bulk transfer: %s", strerror(errno));
         freeClient(slave);
diff --git a/src/server.h b/src/server.h
index 8f1ca50b1..5568155f9 100644
--- a/src/server.h
+++ b/src/server.h
@@ -270,6 +270,8 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
 #define CLIENT_DENY_BLOCKING (1ULL<<41) /* Indicate that the client should not be blocked.
                                            currently, turned on inside MULTI, Lua, RM_Call,
                                            and AOF client */
+#define CLIENT_REPL_RDBONLY (1ULL<<42) /* This client is a replica that only wants
+                                          RDB without replication buffer. */
 
 /* Client block type (btype field in client structure)
  * if CLIENT_BLOCKED flag is set. */

From f395119eded8294e80918331ebc4948eb5e5e006 Mon Sep 17 00:00:00 2001
From: Huang Zw <huang_zhw@126.com>
Date: Thu, 28 Jan 2021 00:56:43 +0800
Subject: [PATCH 1004/1098] Fix bug in activeDefragSdsListAndDict dead code
 (#8403)

In activeDefragSdsListAndDict when dict_val_type is DEFRAG_SDS_DICT_VAL_VOID_PTR, it should update de->v.val not ln->value.
Because this code path will never be executed, so this bug never happened.
---
 src/defrag.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/defrag.c b/src/defrag.c
index e189deddd..db797711e 100644
--- a/src/defrag.c
+++ b/src/defrag.c
@@ -367,7 +367,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) {
         } else if (dict_val_type == DEFRAG_SDS_DICT_VAL_VOID_PTR) {
             void *newptr, *ptr = dictGetVal(de);
             if ((newptr = activeDefragAlloc(ptr)))
-                ln->value = newptr, defragged++;
+                de->v.val = newptr, defragged++;
         }
         defragged += dictIterDefragEntry(di);
     }

From 0367a80819dec9aeb5e3d0972535155fc1c48867 Mon Sep 17 00:00:00 2001
From: Raghav Muddur <r@nmvk.com>
Date: Wed, 27 Jan 2021 09:47:26 -0800
Subject: [PATCH 1005/1098] GETEX, GETDEL and SET PXAT/EXAT (#8327)

This commit introduces two new command and two options for an existing command

GETEX <key> [PERSIST][EX seconds][PX milliseconds] [EXAT seconds-timestamp]
[PXAT milliseconds-timestamp]

The getexCommand() function implements extended options and variants of the GET
command. Unlike GET command this command is not read-only. Only one of the options
can be used at a given time.

1. PERSIST removes any TTL associated with the key.
2. EX Set expiry TTL in seconds.
3. PX Set expiry TTL in milliseconds.
4. EXAT Same like EX instead of specifying the number of seconds representing the
    TTL (time to live), it takes an absolute Unix timestamp
5. PXAT Same like PX instead of specifying the number of milliseconds representing the
    TTL (time to live), it takes an absolute Unix timestamp

Command would return either the bulk string, error or nil.

GETDEL <key>
Would delete the key after getting.

SET key value [NX] [XX] [KEEPTTL] [GET] [EX <seconds>] [PX <milliseconds>]
[EXAT <seconds-timestamp>][PXAT <milliseconds-timestamp>]

Two new options added here are EXAT and PXAT

Key implementation notes
- `SET` with `PX/EX/EXAT/PXAT` is always translated to `PXAT` in `AOF`. When relative time is
  specified (`PX/EX`), replication will always use `PX`.
- `setexCommand` and `psetexCommand` would no longer need translation in `feedAppendOnlyFile`
  as they are modified to invoke `setGenericCommand ` with appropriate flags which will take care of
  correct AOF translation.
- `GETEX` without any optional argument behaves like `GET`.
- `GETEX` command is never propagated, It is either propagated as `PEXPIRE[AT], or PERSIST`.
- `GETDEL` command is propagated as `DEL`
- Combined the validation for `SET` and `GETEX` arguments.
- Test cases to validate AOF/Replication propagation
---
 src/aof.c                  |  47 +++--
 src/server.c               |  14 ++
 src/server.h               |   5 +-
 src/t_string.c             | 346 ++++++++++++++++++++++++++++---------
 tests/integration/aof.tcl  |  11 ++
 tests/unit/expire.tcl      |  71 +++++++-
 tests/unit/type/string.tcl |  96 ++++++++++
 7 files changed, 478 insertions(+), 112 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index d3191277f..5518f0206 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -581,8 +581,6 @@ sds catAppendOnlyExpireAtCommand(sds buf, struct redisCommand *cmd, robj *key, r
 
 void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc) {
     sds buf = sdsempty();
-    robj *tmpargv[3];
-
     /* The DB this command was targeting is not the same as the last command
      * we appended. To issue a SELECT command is needed. */
     if (dictid != server.aof_selected_db) {
@@ -598,32 +596,31 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a
         cmd->proc == expireatCommand) {
         /* Translate EXPIRE/PEXPIRE/EXPIREAT into PEXPIREAT */
         buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]);
-    } else if (cmd->proc == setexCommand || cmd->proc == psetexCommand) {
-        /* Translate SETEX/PSETEX to SET and PEXPIREAT */
-        tmpargv[0] = createStringObject("SET",3);
-        tmpargv[1] = argv[1];
-        tmpargv[2] = argv[3];
-        buf = catAppendOnlyGenericCommand(buf,3,tmpargv);
-        decrRefCount(tmpargv[0]);
-        buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]);
     } else if (cmd->proc == setCommand && argc > 3) {
-        int i;
-        robj *exarg = NULL, *pxarg = NULL;
-        for (i = 3; i < argc; i ++) {
-            if (!strcasecmp(argv[i]->ptr, "ex")) exarg = argv[i+1];
-            if (!strcasecmp(argv[i]->ptr, "px")) pxarg = argv[i+1];
+        robj *pxarg = NULL;
+        /* When SET is used with EX/PX argument setGenericCommand propagates them with PX millisecond argument.
+         * So since the command arguments are re-written there, we can rely here on the index of PX being 3. */
+        if (!strcasecmp(argv[3]->ptr, "px")) {
+            pxarg = argv[4];
         }
-        serverAssert(!(exarg && pxarg));
+        /* For AOF we convert SET key value relative time in milliseconds to SET key value absolute time in
+         * millisecond. Whenever the condition is true it implies that original SET has been transformed
+         * to SET PX with millisecond time argument so we do not need to worry about unit here.*/
+        if (pxarg) {
+            robj *millisecond = getDecodedObject(pxarg);
+            long long when = strtoll(millisecond->ptr,NULL,10);
+            when += mstime();
 
-        if (exarg || pxarg) {
-            /* Translate SET [EX seconds][PX milliseconds] to SET and PEXPIREAT */
-            buf = catAppendOnlyGenericCommand(buf,3,argv);
-            if (exarg)
-                buf = catAppendOnlyExpireAtCommand(buf,server.expireCommand,argv[1],
-                                                   exarg);
-            if (pxarg)
-                buf = catAppendOnlyExpireAtCommand(buf,server.pexpireCommand,argv[1],
-                                                   pxarg);
+            decrRefCount(millisecond);
+
+            robj *newargs[5];
+            newargs[0] = argv[0];
+            newargs[1] = argv[1];
+            newargs[2] = argv[2];
+            newargs[3] = shared.pxat;
+            newargs[4] = createStringObjectFromLongLong(when);
+            buf = catAppendOnlyGenericCommand(buf,5,newargs);
+            decrRefCount(newargs[4]);
         } else {
             buf = catAppendOnlyGenericCommand(buf,argc,argv);
         }
diff --git a/src/server.c b/src/server.c
index bdedf72d0..763e3f5df 100644
--- a/src/server.c
+++ b/src/server.c
@@ -201,6 +201,14 @@ struct redisCommand redisCommandTable[] = {
      "read-only fast @string",
      0,NULL,1,1,1,0,0,0},
 
+    {"getex",getexCommand,-2,
+     "write fast @string",
+     0,NULL,1,1,1,0,0,0},
+
+    {"getdel",getdelCommand,2,
+     "write fast @string",
+     0,NULL,1,1,1,0,0,0},
+
     /* Note that we can't flag set as fast, since it may perform an
      * implicit DEL of a large key. */
     {"set",setCommand,-3,
@@ -2532,6 +2540,12 @@ void createSharedObjects(void) {
     /* Used in the LMOVE/BLMOVE commands */
     shared.left = createStringObject("left",4);
     shared.right = createStringObject("right",5);
+    shared.pexpireat = createStringObject("PEXPIREAT",9);
+    shared.pexpire = createStringObject("PEXPIRE",7);
+    shared.persist = createStringObject("PERSIST",7);
+    shared.set = createStringObject("SET",3);
+    shared.pxat = createStringObject("PXAT", 4);
+    shared.px = createStringObject("PX",2);
     for (j = 0; j < OBJ_SHARED_INTEGERS; j++) {
         shared.integers[j] =
             makeObjectShared(createObject(OBJ_STRING,(void*)(long)j));
diff --git a/src/server.h b/src/server.h
index 5568155f9..83e007ebe 100644
--- a/src/server.h
+++ b/src/server.h
@@ -966,7 +966,8 @@ struct sharedObjectsStruct {
     *busykeyerr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk,
     *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *unlink,
     *rpop, *lpop, *lpush, *rpoplpush, *lmove, *blmove, *zpopmin, *zpopmax,
-    *emptyscan, *multi, *exec, *left, *right,
+    *emptyscan, *multi, *exec, *left, *right, *persist, *set, *pexpireat,
+    *pexpire, *pxat, *px,
     *select[PROTO_SHARED_SELECT_CMDS],
     *integers[OBJ_SHARED_INTEGERS],
     *mbulkhdr[OBJ_SHARED_BULKHDR_LEN], /* "*<value>\r\n" */
@@ -2426,6 +2427,8 @@ void setnxCommand(client *c);
 void setexCommand(client *c);
 void psetexCommand(client *c);
 void getCommand(client *c);
+void getexCommand(client *c);
+void getdelCommand(client *c);
 void delCommand(client *c);
 void unlinkCommand(client *c);
 void existsCommand(client *c);
diff --git a/src/t_string.c b/src/t_string.c
index 2792f5557..de67484fc 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -61,13 +61,16 @@ static int checkStringLength(client *c, long long size) {
  * If ok_reply is NULL "+OK" is used.
  * If abort_reply is NULL, "$-1" is used. */
 
-#define OBJ_SET_NO_FLAGS 0
+#define OBJ_NO_FLAGS 0
 #define OBJ_SET_NX (1<<0)          /* Set if key not exists. */
 #define OBJ_SET_XX (1<<1)          /* Set if key exists. */
-#define OBJ_SET_EX (1<<2)          /* Set if time in seconds is given */
-#define OBJ_SET_PX (1<<3)          /* Set if time in ms in given */
-#define OBJ_SET_KEEPTTL (1<<4)     /* Set and keep the ttl */
+#define OBJ_EX (1<<2)              /* Set if time in seconds is given */
+#define OBJ_PX (1<<3)              /* Set if time in ms in given */
+#define OBJ_KEEPTTL (1<<4)         /* Set and keep the ttl */
 #define OBJ_SET_GET (1<<5)         /* Set if want to get key before set */
+#define OBJ_EXAT (1<<6)            /* Set if timestamp in second is given */
+#define OBJ_PXAT (1<<7)            /* Set if timestamp in ms is given */
+#define OBJ_PERSIST (1<<8)         /* Set if we need to remove the ttl */
 
 void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire, int unit, robj *ok_reply, robj *abort_reply) {
     long long milliseconds = 0; /* initialized to avoid any harmness warning */
@@ -93,77 +96,41 @@ void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire,
         if (getGenericCommand(c) == C_ERR) return;
     }
 
-    genericSetKey(c,c->db,key,val,flags & OBJ_SET_KEEPTTL,1);
+    genericSetKey(c,c->db,key, val,flags & OBJ_KEEPTTL,1);
     server.dirty++;
-    if (expire) setExpire(c,c->db,key,mstime()+milliseconds);
     notifyKeyspaceEvent(NOTIFY_STRING,"set",key,c->db->id);
-    if (expire) notifyKeyspaceEvent(NOTIFY_GENERIC,
-        "expire",key,c->db->id);
+    if (expire) {
+        robj *exp = shared.pxat;
+
+        if ((flags & OBJ_PX) || (flags & OBJ_EX)) {
+            setExpire(c,c->db,key,milliseconds + mstime());
+            exp = shared.px;
+        } else {
+            setExpire(c,c->db,key,milliseconds);
+        }
+        notifyKeyspaceEvent(NOTIFY_GENERIC,"expire",key,c->db->id);
+
+        /* Propagate as SET Key Value PXAT millisecond-timestamp if there is EXAT/PXAT or
+         * propagate as SET Key Value PX millisecond if there is EX/PX flag.
+         *
+         * Additionally when we propagate the SET with PX (relative millisecond) we translate
+         * it again to SET with PXAT for the AOF.
+         *
+         * Additional care is required while modifying the argument order. AOF relies on the
+         * exp argument being at index 3. (see feedAppendOnlyFile)
+         * */
+        robj *millisecondObj = createStringObjectFromLongLong(milliseconds);
+        rewriteClientCommandVector(c,5,shared.set,key,val,exp,millisecondObj);
+        decrRefCount(millisecondObj);
+    }
     if (!(flags & OBJ_SET_GET)) {
         addReply(c, ok_reply ? ok_reply : shared.ok);
     }
-}
 
-/* SET key value [NX] [XX] [KEEPTTL] [GET] [EX <seconds>] [PX <milliseconds>] */
-void setCommand(client *c) {
-    int j;
-    robj *expire = NULL;
-    int unit = UNIT_SECONDS;
-    int flags = OBJ_SET_NO_FLAGS;
-
-    for (j = 3; j < c->argc; j++) {
-        char *a = c->argv[j]->ptr;
-        robj *next = (j == c->argc-1) ? NULL : c->argv[j+1];
-
-        if ((a[0] == 'n' || a[0] == 'N') &&
-            (a[1] == 'x' || a[1] == 'X') && a[2] == '\0' &&
-            !(flags & OBJ_SET_XX) && !(flags & OBJ_SET_GET))
-        {
-            flags |= OBJ_SET_NX;
-        } else if ((a[0] == 'x' || a[0] == 'X') &&
-                   (a[1] == 'x' || a[1] == 'X') && a[2] == '\0' &&
-                   !(flags & OBJ_SET_NX))
-        {
-            flags |= OBJ_SET_XX;
-        } else if ((a[0] == 'g' || a[0] == 'G') &&
-                   (a[1] == 'e' || a[1] == 'E') &&
-                   (a[2] == 't' || a[2] == 'T') && a[3] == '\0' &&
-                   !(flags & OBJ_SET_NX)) {
-            flags |= OBJ_SET_GET;
-        } else if (!strcasecmp(c->argv[j]->ptr,"KEEPTTL") &&
-                   !(flags & OBJ_SET_EX) && !(flags & OBJ_SET_PX))
-        {
-            flags |= OBJ_SET_KEEPTTL;
-        } else if ((a[0] == 'e' || a[0] == 'E') &&
-                   (a[1] == 'x' || a[1] == 'X') && a[2] == '\0' &&
-                   !(flags & OBJ_SET_KEEPTTL) &&
-                   !(flags & OBJ_SET_PX) && next)
-        {
-            flags |= OBJ_SET_EX;
-            unit = UNIT_SECONDS;
-            expire = next;
-            j++;
-        } else if ((a[0] == 'p' || a[0] == 'P') &&
-                   (a[1] == 'x' || a[1] == 'X') && a[2] == '\0' &&
-                   !(flags & OBJ_SET_KEEPTTL) &&
-                   !(flags & OBJ_SET_EX) && next)
-        {
-            flags |= OBJ_SET_PX;
-            unit = UNIT_MILLISECONDS;
-            expire = next;
-            j++;
-        } else {
-            addReplyErrorObject(c,shared.syntaxerr);
-            return;
-        }
-    }
-
-    c->argv[2] = tryObjectEncoding(c->argv[2]);
-    setGenericCommand(c,flags,c->argv[1],c->argv[2],expire,unit,NULL,NULL);
-
-    /* Propagate without the GET argument */
-    if (flags & OBJ_SET_GET) {
+    /* Propagate without the GET argument (Isn't needed if we had expire since in that case we completely re-written the command argv) */
+    if ((flags & OBJ_SET_GET) && !expire) {
         int argc = 0;
+        int j;
         robj **argv = zmalloc((c->argc-1)*sizeof(robj*));
         for (j=0; j < c->argc; j++) {
             char *a = c->argv[j]->ptr;
@@ -180,6 +147,123 @@ void setCommand(client *c) {
     }
 }
 
+#define COMMAND_GET 0
+#define COMMAND_SET 1
+/*
+ * The parseExtendedStringArgumentsOrReply() function performs the common validation for extended
+ * string arguments used in SET and GET command.
+ *
+ * Get specific commands - PERSIST/DEL
+ * Set specific commands - XX/NX/GET
+ * Common commands - EX/EXAT/PX/PXAT/KEEPTTL
+ *
+ * Function takes pointers to client, flags, unit, pointer to pointer of expire obj if needed
+ * to be determined and command_type which can be COMMAND_GET or COMMAND_SET.
+ *
+ * If there are any syntax violations C_ERR is returned else C_OK is returned.
+ *
+ * Input flags are updated upon parsing the arguments. Unit and expire are updated if there are any
+ * EX/EXAT/PX/PXAT arguments. Unit is updated to millisecond if PX/PXAT is set.
+ */
+int parseExtendedStringArgumentsOrReply(client *c, int *flags, int *unit, robj **expire, int command_type) {
+
+    int j = command_type == COMMAND_GET ? 2 : 3;
+    for (; j < c->argc; j++) {
+        char *opt = c->argv[j]->ptr;
+        robj *next = (j == c->argc-1) ? NULL : c->argv[j+1];
+
+        if ((opt[0] == 'n' || opt[0] == 'N') &&
+            (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' &&
+            !(*flags & OBJ_SET_XX) && !(*flags & OBJ_SET_GET) && (command_type == COMMAND_SET))
+        {
+            *flags |= OBJ_SET_NX;
+        } else if ((opt[0] == 'x' || opt[0] == 'X') &&
+                   (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' &&
+                   !(*flags & OBJ_SET_NX) && (command_type == COMMAND_SET))
+        {
+            *flags |= OBJ_SET_XX;
+        } else if ((opt[0] == 'g' || opt[0] == 'G') &&
+                   (opt[1] == 'e' || opt[1] == 'E') &&
+                   (opt[2] == 't' || opt[2] == 'T') && opt[3] == '\0' &&
+                   !(*flags & OBJ_SET_NX) && (command_type == COMMAND_SET))
+        {
+            *flags |= OBJ_SET_GET;
+        } else if (!strcasecmp(opt, "KEEPTTL") && !(*flags & OBJ_PERSIST) &&
+            !(*flags & OBJ_EX) && !(*flags & OBJ_EXAT) &&
+            !(*flags & OBJ_PX) && !(*flags & OBJ_PXAT) && (command_type == COMMAND_SET))
+        {
+            *flags |= OBJ_KEEPTTL;
+        } else if (!strcasecmp(opt,"PERSIST") && (command_type == COMMAND_GET) &&
+               !(*flags & OBJ_EX) && !(*flags & OBJ_EXAT) &&
+               !(*flags & OBJ_PX) && !(*flags & OBJ_PXAT) &&
+               !(*flags & OBJ_KEEPTTL))
+        {
+            *flags |= OBJ_PERSIST;
+        } else if ((opt[0] == 'e' || opt[0] == 'E') &&
+                   (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' &&
+                   !(*flags & OBJ_KEEPTTL) && !(*flags & OBJ_PERSIST) &&
+                   !(*flags & OBJ_EXAT) && !(*flags & OBJ_PX) &&
+                   !(*flags & OBJ_PXAT) && next)
+        {
+            *flags |= OBJ_EX;
+            *expire = next;
+            j++;
+        } else if ((opt[0] == 'p' || opt[0] == 'P') &&
+                   (opt[1] == 'x' || opt[1] == 'X') && opt[2] == '\0' &&
+                   !(*flags & OBJ_KEEPTTL) && !(*flags & OBJ_PERSIST) &&
+                   !(*flags & OBJ_EX) && !(*flags & OBJ_EXAT) &&
+                   !(*flags & OBJ_PXAT) && next)
+        {
+            *flags |= OBJ_PX;
+            *unit = UNIT_MILLISECONDS;
+            *expire = next;
+            j++;
+        } else if ((opt[0] == 'e' || opt[0] == 'E') &&
+                   (opt[1] == 'x' || opt[1] == 'X') &&
+                   (opt[2] == 'a' || opt[2] == 'A') &&
+                   (opt[3] == 't' || opt[3] == 'T') && opt[4] == '\0' &&
+                   !(*flags & OBJ_KEEPTTL) && !(*flags & OBJ_PERSIST) &&
+                   !(*flags & OBJ_EX) && !(*flags & OBJ_PX) &&
+                   !(*flags & OBJ_PXAT) && next)
+        {
+            *flags |= OBJ_EXAT;
+            *expire = next;
+            j++;
+        } else if ((opt[0] == 'p' || opt[0] == 'P') &&
+                   (opt[1] == 'x' || opt[1] == 'X') &&
+                   (opt[2] == 'a' || opt[2] == 'A') &&
+                   (opt[3] == 't' || opt[3] == 'T') && opt[4] == '\0' &&
+                   !(*flags & OBJ_KEEPTTL) && !(*flags & OBJ_PERSIST) &&
+                   !(*flags & OBJ_EX) && !(*flags & OBJ_EXAT) &&
+                   !(*flags & OBJ_PX) && next)
+        {
+            *flags |= OBJ_PXAT;
+            *unit = UNIT_MILLISECONDS;
+            *expire = next;
+            j++;
+        } else {
+            addReplyErrorObject(c,shared.syntaxerr);
+            return C_ERR;
+        }
+    }
+    return C_OK;
+}
+
+/* SET key value [NX] [XX] [KEEPTTL] [GET] [EX <seconds>] [PX <milliseconds>]
+ *     [EXAT <seconds-timestamp>][PXAT <milliseconds-timestamp>] */
+void setCommand(client *c) {
+    robj *expire = NULL;
+    int unit = UNIT_SECONDS;
+    int flags = OBJ_NO_FLAGS;
+
+    if (parseExtendedStringArgumentsOrReply(c,&flags,&unit,&expire,COMMAND_SET) != C_OK) {
+        return;
+    }
+
+    c->argv[2] = tryObjectEncoding(c->argv[2]);
+    setGenericCommand(c,flags,c->argv[1],c->argv[2],expire,unit,NULL,NULL);
+}
+
 void setnxCommand(client *c) {
     c->argv[2] = tryObjectEncoding(c->argv[2]);
     setGenericCommand(c,OBJ_SET_NX,c->argv[1],c->argv[2],NULL,0,shared.cone,shared.czero);
@@ -187,12 +271,12 @@ void setnxCommand(client *c) {
 
 void setexCommand(client *c) {
     c->argv[3] = tryObjectEncoding(c->argv[3]);
-    setGenericCommand(c,OBJ_SET_NO_FLAGS,c->argv[1],c->argv[3],c->argv[2],UNIT_SECONDS,NULL,NULL);
+    setGenericCommand(c,OBJ_EX,c->argv[1],c->argv[3],c->argv[2],UNIT_SECONDS,NULL,NULL);
 }
 
 void psetexCommand(client *c) {
     c->argv[3] = tryObjectEncoding(c->argv[3]);
-    setGenericCommand(c,OBJ_SET_NO_FLAGS,c->argv[1],c->argv[3],c->argv[2],UNIT_MILLISECONDS,NULL,NULL);
+    setGenericCommand(c,OBJ_PX,c->argv[1],c->argv[3],c->argv[2],UNIT_MILLISECONDS,NULL,NULL);
 }
 
 int getGenericCommand(client *c) {
@@ -213,6 +297,112 @@ void getCommand(client *c) {
     getGenericCommand(c);
 }
 
+/*
+ * GETEX <key> [PERSIST][EX seconds][PX milliseconds][EXAT seconds-timestamp][PXAT milliseconds-timestamp]
+ *
+ * The getexCommand() function implements extended options and variants of the GET command. Unlike GET
+ * command this command is not read-only.
+ *
+ * The default behavior when no options are specified is same as GET and does not alter any TTL.
+ *
+ * Only one of the below options can be used at a given time.
+ *
+ * 1. PERSIST removes any TTL associated with the key.
+ * 2. EX Set expiry TTL in seconds.
+ * 3. PX Set expiry TTL in milliseconds.
+ * 4. EXAT Same like EX instead of specifying the number of seconds representing the TTL
+ *      (time to live), it takes an absolute Unix timestamp
+ * 5. PXAT Same like PX instead of specifying the number of milliseconds representing the TTL
+ *      (time to live), it takes an absolute Unix timestamp
+ *
+ * Command would either return the bulk string, error or nil.
+ */
+void getexCommand(client *c) {
+    robj *expire = NULL;
+    int unit = UNIT_SECONDS;
+    int flags = OBJ_NO_FLAGS;
+
+    if (parseExtendedStringArgumentsOrReply(c,&flags,&unit,&expire,COMMAND_GET) != C_OK) {
+        return;
+    }
+
+    robj *o;
+
+    if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) == NULL)
+        return;
+
+    if (checkType(c,o,OBJ_STRING)) {
+        return;
+    }
+
+    long long milliseconds = 0;
+
+    /* Validate the expiration time value first */
+    if (expire) {
+        if (getLongLongFromObjectOrReply(c, expire, &milliseconds, NULL) != C_OK)
+            return;
+        if (milliseconds <= 0) {
+            addReplyErrorFormat(c,"invalid expire time in %s",c->cmd->name);
+            return;
+        }
+        if (unit == UNIT_SECONDS) milliseconds *= 1000;
+    }
+
+    /* We need to do this before we expire the key or delete it */
+    addReplyBulk(c,o);
+
+    /* This command is never propagated as is. It is either propagated as PEXPIRE[AT],DEL,UNLINK or PERSIST.
+     * This why it doesn't need special handling in feedAppendOnlyFile to convert relative expire time to absolute one. */
+    if (((flags & OBJ_PXAT) || (flags & OBJ_EXAT)) && checkAlreadyExpired(milliseconds)) {
+        /* When PXAT/EXAT absolute timestamp is specified, there can be a chance that timestamp
+         * has already elapsed so delete the key in that case. */
+        int deleted = server.lazyfree_lazy_expire ? dbAsyncDelete(c->db, c->argv[1]) :
+                      dbSyncDelete(c->db, c->argv[1]);
+        serverAssert(deleted);
+        robj *aux = server.lazyfree_lazy_expire ? shared.unlink : shared.del;
+        rewriteClientCommandVector(c,2,aux,c->argv[1]);
+        signalModifiedKey(c, c->db, c->argv[1]);
+        notifyKeyspaceEvent(NOTIFY_GENERIC, "del", c->argv[1], c->db->id);
+        server.dirty++;
+    } else if (expire) {
+        robj *exp = shared.pexpireat;
+        if ((flags & OBJ_PX) || (flags & OBJ_EX)) {
+            setExpire(c,c->db,c->argv[1],milliseconds + mstime());
+            exp = shared.pexpire;
+        } else {
+            setExpire(c,c->db,c->argv[1],milliseconds);
+        }
+
+        robj* millisecondObj = createStringObjectFromLongLong(milliseconds);
+        rewriteClientCommandVector(c,3,exp,c->argv[1],millisecondObj);
+        decrRefCount(millisecondObj);
+        signalModifiedKey(c, c->db, c->argv[1]);
+        notifyKeyspaceEvent(NOTIFY_GENERIC,"expire",c->argv[1],c->db->id);
+        server.dirty++;
+    } else if (flags & OBJ_PERSIST) {
+        if (removeExpire(c->db, c->argv[1])) {
+            signalModifiedKey(c, c->db, c->argv[1]);
+            rewriteClientCommandVector(c, 2, shared.persist, c->argv[1]);
+            notifyKeyspaceEvent(NOTIFY_GENERIC,"persist",c->argv[1],c->db->id);
+            server.dirty++;
+        }
+    }
+}
+
+void getdelCommand(client *c) {
+    if (getGenericCommand(c) == C_ERR) return;
+    int deleted = server.lazyfree_lazy_user_del ? dbAsyncDelete(c->db, c->argv[1]) :
+                  dbSyncDelete(c->db, c->argv[1]);
+    if (deleted) {
+        /* Propagate as DEL/UNLINK command */
+        robj *aux = server.lazyfree_lazy_user_del ? shared.unlink : shared.del;
+        rewriteClientCommandVector(c,2,aux,c->argv[1]);
+        signalModifiedKey(c, c->db, c->argv[1]);
+        notifyKeyspaceEvent(NOTIFY_GENERIC, "del", c->argv[1], c->db->id);
+        server.dirty++;
+    }
+}
+
 void getsetCommand(client *c) {
     if (getGenericCommand(c) == C_ERR) return;
     c->argv[2] = tryObjectEncoding(c->argv[2]);
@@ -221,9 +411,7 @@ void getsetCommand(client *c) {
     server.dirty++;
 
     /* Propagate as SET command */
-    robj *setcmd = createStringObject("SET",3);
-    rewriteClientCommandArgument(c,0,setcmd);
-    decrRefCount(setcmd);
+    rewriteClientCommandArgument(c,0,shared.set);
 }
 
 void setrangeCommand(client *c) {
@@ -443,7 +631,7 @@ void decrbyCommand(client *c) {
 
 void incrbyfloatCommand(client *c) {
     long double incr, value;
-    robj *o, *new, *aux1, *aux2;
+    robj *o, *new, *aux;
 
     o = lookupKeyWrite(c->db,c->argv[1]);
     if (checkType(c,o,OBJ_STRING)) return;
@@ -469,13 +657,11 @@ void incrbyfloatCommand(client *c) {
     /* Always replicate INCRBYFLOAT as a SET command with the final value
      * in order to make sure that differences in float precision or formatting
      * will not create differences in replicas or after an AOF restart. */
-    aux1 = createStringObject("SET",3);
-    rewriteClientCommandArgument(c,0,aux1);
-    decrRefCount(aux1);
+    rewriteClientCommandArgument(c,0,shared.set);
     rewriteClientCommandArgument(c,2,new);
-    aux2 = createStringObject("KEEPTTL",7);
-    rewriteClientCommandArgument(c,3,aux2);
-    decrRefCount(aux2);
+    aux = createStringObject("KEEPTTL",7);
+    rewriteClientCommandArgument(c,3,aux);
+    decrRefCount(aux);
 }
 
 void appendCommand(client *c) {
diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl
index d81521374..e64e2022a 100644
--- a/tests/integration/aof.tcl
+++ b/tests/integration/aof.tcl
@@ -272,4 +272,15 @@ tags {"aof"} {
             }
         }
     }
+
+    start_server {overrides {appendonly {yes} appendfilename {appendonly.aof}}} {
+        test {GETEX should not append to AOF} {
+            set aof [file join [lindex [r config get dir] 1] appendonly.aof]
+            r set foo bar
+            set before [file size $aof]
+            r getex foo
+            set after [file size $aof]
+            assert_equal $before $after
+        }
+    }
 }
diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl
index e71018073..9bde4809f 100644
--- a/tests/unit/expire.tcl
+++ b/tests/unit/expire.tcl
@@ -209,7 +209,7 @@ start_server {tags {"expire"}} {
         set e
     } {*not an integer*}
 
-    test {EXPIRE and SET EX/PX option, TTL should not be reset after loadaof} {
+    test {EXPIRE and SET/GETEX EX/PX/EXAT/PXAT option, TTL should not be reset after loadaof} {
         # This test makes sure that expire times are propagated as absolute
         # times to the AOF file and not as relative time, so that when the AOF
         # is reloaded the TTLs are not being shifted forward to the future.
@@ -224,6 +224,17 @@ start_server {tags {"expire"}} {
         r pexpire foo4 100000
         r setex foo5 100 bar
         r psetex foo6 100000 bar
+        r set foo7 bar EXAT [expr [clock seconds] + 100]
+        r set foo8 bar PXAT [expr [clock milliseconds] + 100000]
+        r set foo9 bar
+        r getex foo9 EX 100
+        r set foo10 bar
+        r getex foo10 PX 100000
+        r set foo11 bar
+        r getex foo11 EXAT [expr [clock seconds] + 100]
+        r set foo12 bar
+        r getex foo12 PXAT [expr [clock milliseconds] + 100000]
+
         after 2000
         r debug loadaof
         assert_range [r ttl foo1] 90 98
@@ -232,6 +243,12 @@ start_server {tags {"expire"}} {
         assert_range [r ttl foo4] 90 98
         assert_range [r ttl foo5] 90 98
         assert_range [r ttl foo6] 90 98
+        assert_range [r ttl foo7] 90 98
+        assert_range [r ttl foo8] 90 98
+        assert_range [r ttl foo9] 90 98
+        assert_range [r ttl foo10] 90 98
+        assert_range [r ttl foo11] 90 98
+        assert_range [r ttl foo12] 90 98
     }
 
     test {EXPIRE relative and absolute propagation to replicas} {
@@ -248,8 +265,10 @@ start_server {tags {"expire"}} {
         # https://github.com/redis/redis/pull/5171#issuecomment-409553266
 
         set repl [attach_to_replication_stream]
-        r set foo1 bar ex 100
+        r set foo1 bar ex 200
         r set foo1 bar px 100000
+        r set foo1 bar exat [expr [clock seconds]+100]
+        r set foo1 bar pxat [expr [clock milliseconds]+10000]
         r setex foo1 100 bar
         r psetex foo1 100000 bar
         r set foo2 bar
@@ -259,12 +278,19 @@ start_server {tags {"expire"}} {
         r expireat foo3 [expr [clock seconds]+100]
         r pexpireat foo3 [expr [clock seconds]*1000+100000]
         r expireat foo3 [expr [clock seconds]-100]
+        r set foo4 bar
+        r getex foo4 ex 200
+        r getex foo4 px 200000
+        r getex foo4 exat [expr [clock seconds]+100]
+        r getex foo4 pxat [expr [clock milliseconds]+10000]
         assert_replication_stream $repl {
             {select *}
-            {set foo1 bar ex 100}
-            {set foo1 bar px 100000}
-            {setex foo1 100 bar}
-            {psetex foo1 100000 bar}
+            {set foo1 bar PX 200000}
+            {set foo1 bar PX 100000}
+            {set foo1 bar PXAT *}
+            {set foo1 bar PXAT *}
+            {set foo1 bar PX 100000}
+            {set foo1 bar PX 100000}
             {set foo2 bar}
             {expire foo2 100}
             {pexpire foo2 100000}
@@ -272,6 +298,11 @@ start_server {tags {"expire"}} {
             {expireat foo3 *}
             {pexpireat foo3 *}
             {del foo3}
+            {set foo4 bar}
+            {pexpire foo4 200000}
+            {pexpire foo4 200000}
+            {pexpireat foo4 *}
+            {pexpireat foo4 *}
         }
     }
 
@@ -297,4 +328,32 @@ start_server {tags {"expire"}} {
         set ttl [r ttl foo]
         assert {$ttl <= 98 && $ttl > 90}
     }
+
+    test {GETEX use of PERSIST option should remove TTL} {
+       r set foo bar EX 100
+       r getex foo PERSIST
+       r ttl foo
+    } {-1}
+
+    test {GETEX use of PERSIST option should remove TTL after loadaof} {
+       r set foo bar EX 100
+       r getex foo PERSIST
+       after 2000
+       r debug loadaof
+       r ttl foo
+    } {-1}
+
+    test {GETEX propagate as to replica as PERSIST, DEL, or nothing} {
+       set repl [attach_to_replication_stream]
+       r set foo bar EX 100
+       r getex foo PERSIST
+       r getex foo
+       r getex foo exat [expr [clock seconds]-100]
+       assert_replication_stream $repl {
+           {select *}
+           {set foo bar PX 100000}
+           {persist foo}
+           {del foo}
+        }
+    }
 }
diff --git a/tests/unit/type/string.tcl b/tests/unit/type/string.tcl
index 16e961623..43968b26b 100644
--- a/tests/unit/type/string.tcl
+++ b/tests/unit/type/string.tcl
@@ -102,6 +102,91 @@ start_server {tags {"string"}} {
         assert_equal 20 [r get x]
     }
 
+    test "GETEX EX option" {
+        r del foo
+        r set foo bar
+        r getex foo ex 10
+        assert_range [r ttl foo] 5 10
+    }
+
+    test "GETEX PX option" {
+        r del foo
+        r set foo bar
+        r getex foo px 10000
+        assert_range [r pttl foo] 5000 10000
+    }
+
+    test "GETEX EXAT option" {
+        r del foo
+        r set foo bar
+        r getex foo exat [expr [clock seconds] + 10]
+        assert_range [r ttl foo] 5 10
+    }
+
+    test "GETEX PXAT option" {
+        r del foo
+        r set foo bar
+        r getex foo pxat [expr [clock milliseconds] + 10000]
+        assert_range [r pttl foo] 5000 10000
+    }
+
+    test "GETEX PERSIST option" {
+        r del foo
+        r set foo bar ex 10
+        assert_range [r ttl foo] 5 10
+        r getex foo persist
+        assert_equal -1 [r ttl foo]
+    }
+
+    test "GETEX no option" {
+        r del foo
+        r set foo bar
+        r getex foo
+        assert_equal bar [r getex foo]
+    }
+
+    test "GETEX syntax errors" {
+        set ex {}
+        catch {r getex foo non-existent-option} ex
+        set ex
+    } {*syntax*}
+
+    test "GETEX no arguments" {
+         set ex {}
+         catch {r getex} ex
+         set ex
+     } {*wrong number of arguments*}
+
+    test "GETDEL command" {
+        r del foo
+        r set foo bar
+        assert_equal bar [r getdel foo ]
+        assert_equal {} [r getdel foo ]
+    }
+
+    test {GETDEL propagate as DEL command to replica} {
+        set repl [attach_to_replication_stream]
+        r set foo bar
+        r getdel foo
+        assert_replication_stream $repl {
+            {select *}
+            {set foo bar}
+            {del foo}
+        }
+    }
+
+    test {GETEX without argument does not propagate to replica} {
+        set repl [attach_to_replication_stream]
+        r set foo bar
+        r getex foo
+        r del foo
+        assert_replication_stream $repl {
+            {select *}
+            {set foo bar}
+            {del foo}
+        }
+    }
+
     test {MGET} {
         r flushdb
         r set foo BAR
@@ -437,6 +522,17 @@ start_server {tags {"string"}} {
         assert {$ttl <= 10 && $ttl > 5}
     }
 
+    test "Extended SET EXAT option" {
+        r del foo
+        r set foo bar exat [expr [clock seconds] + 10]
+        assert_range [r ttl foo] 5 10
+    }
+
+    test "Extended SET PXAT option" {
+        r del foo
+        r set foo bar pxat [expr [clock milliseconds] + 10000]
+        assert_range [r ttl foo] 5 10
+    }
     test {Extended SET using multiple options at once} {
         r set foo val
         assert {[r set foo bar xx px 10000] eq {OK}}

From 3a5049042ac06b6ed5e526f331d5378bf7c7b7ed Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 28 Jan 2021 10:33:45 +0200
Subject: [PATCH 1006/1098] Avoid assertions when testing arm64 cow bug.
 (#8405)

At least in one case the arm64 cow kernel bug test triggers an assert, which is a problem because it cannot be ignored like cases where the bug is found.

On older systems (Linux <4.5) madvise fails because MADV_FREE is not supported. We treat these failures as an indication the system is not affected.

Fixes #8351, #8406
---
 src/server.c | 101 ++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 68 insertions(+), 33 deletions(-)

diff --git a/src/server.c b/src/server.c
index 763e3f5df..a0badea87 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5200,7 +5200,7 @@ static int smapsGetSharedDirty(unsigned long addr) {
     FILE *f;
 
     f = fopen("/proc/self/smaps", "r");
-    serverAssert(f);
+    if (!f) return -1;
 
     while (1) {
         if (!fgets(buf, sizeof(buf), f))
@@ -5211,8 +5211,8 @@ static int smapsGetSharedDirty(unsigned long addr) {
             in_mapping = from <= addr && addr < to;
 
         if (in_mapping && !memcmp(buf, "Shared_Dirty:", 13)) {
-            ret = sscanf(buf, "%*s %d", &val);
-            serverAssert(ret == 1);
+            sscanf(buf, "%*s %d", &val);
+            /* If parsing fails, we remain with val == -1 */
             break;
         }
     }
@@ -5226,23 +5226,33 @@ static int smapsGetSharedDirty(unsigned long addr) {
  * kernel is affected.
  * The bug was fixed in commit ff1712f953e27f0b0718762ec17d0adb15c9fd0b
  * titled: "arm64: pgtable: Ensure dirty bit is preserved across pte_wrprotect()"
- * Return 1 if the kernel seems to be affected, and 0 otherwise. */
+ * Return -1 on unexpected test failure, 1 if the kernel seems to be affected,
+ * and 0 otherwise. */
 int linuxMadvFreeForkBugCheck(void) {
-    int ret, pipefd[2];
+    int ret, pipefd[2] = { -1, -1 };
     pid_t pid;
-    char *p, *q, bug_found = 0;
-    const long map_size = 3 * 4096;
+    char *p = NULL, *q;
+    int bug_found = 0;
+    long page_size = sysconf(_SC_PAGESIZE);
+    long map_size = 3 * page_size;
 
     /* Create a memory map that's in our full control (not one used by the allocator). */
     p = mmap(NULL, map_size, PROT_READ, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
-    serverAssert(p != MAP_FAILED);
+    if (p == MAP_FAILED) {
+        serverLog(LL_WARNING, "Failed to mmap(): %s", strerror(errno));
+        return -1;
+    }
 
-    q = p + 4096;
+    q = p + page_size;
 
     /* Split the memory map in 3 pages by setting their protection as RO|RW|RO to prevent
      * Linux from merging this memory map with adjacent VMAs. */
-    ret = mprotect(q, 4096, PROT_READ | PROT_WRITE);
-    serverAssert(!ret);
+    ret = mprotect(q, page_size, PROT_READ | PROT_WRITE);
+    if (ret < 0) {
+        serverLog(LL_WARNING, "Failed to mprotect(): %s", strerror(errno));
+        bug_found = -1;
+        goto exit;
+    }
 
     /* Write to the page once to make it resident */
     *(volatile char*)q = 0;
@@ -5251,8 +5261,16 @@ int linuxMadvFreeForkBugCheck(void) {
 #ifndef MADV_FREE
 #define MADV_FREE 8
 #endif
-    ret = madvise(q, 4096, MADV_FREE);
-    serverAssert(!ret);
+    ret = madvise(q, page_size, MADV_FREE);
+    if (ret < 0) {
+        /* MADV_FREE is not available on older kernels that are presumably
+         * not affected. */
+        if (errno == EINVAL) goto exit;
+
+        serverLog(LL_WARNING, "Failed to madvise(): %s", strerror(errno));
+        bug_found = -1;
+        goto exit;
+    }
 
     /* Write to the page after being marked for freeing, this is supposed to take
      * ownership of that page again. */
@@ -5260,37 +5278,47 @@ int linuxMadvFreeForkBugCheck(void) {
 
     /* Create a pipe for the child to return the info to the parent. */
     ret = pipe(pipefd);
-    serverAssert(!ret);
+    if (ret < 0) {
+        serverLog(LL_WARNING, "Failed to create pipe: %s", strerror(errno));
+        bug_found = -1;
+        goto exit;
+    }
 
     /* Fork the process. */
     pid = fork();
-    serverAssert(pid >= 0);
-    if (!pid) {
-        /* Child: check if the page is marked as dirty, expecing 4 (kB).
+    if (pid < 0) {
+        serverLog(LL_WARNING, "Failed to fork: %s", strerror(errno));
+        bug_found = -1;
+        goto exit;
+    } else if (!pid) {
+        /* Child: check if the page is marked as dirty, page_size in kb.
          * A value of 0 means the kernel is affected by the bug. */
-        if (!smapsGetSharedDirty((unsigned long)q))
+        ret = smapsGetSharedDirty((unsigned long) q);
+        if (!ret)
             bug_found = 1;
+        else if (ret == -1)     /* Failed to read */
+            bug_found = -1;
 
-        ret = write(pipefd[1], &bug_found, 1);
-        serverAssert(ret == 1);
-
+        if (write(pipefd[1], &bug_found, sizeof(bug_found)) < 0)
+            serverLog(LL_WARNING, "Failed to write to parent: %s", strerror(errno));
         exit(0);
     } else {
         /* Read the result from the child. */
-        ret = read(pipefd[0], &bug_found, 1);
-        serverAssert(ret == 1);
+        ret = read(pipefd[0], &bug_found, sizeof(bug_found));
+        if (ret < 0) {
+            serverLog(LL_WARNING, "Failed to read from child: %s", strerror(errno));
+            bug_found = -1;
+        }
 
         /* Reap the child pid. */
-        serverAssert(waitpid(pid, NULL, 0) == pid);
+        waitpid(pid, NULL, 0);
     }
 
+exit:
     /* Cleanup */
-    ret = close(pipefd[0]);
-    serverAssert(!ret);
-    ret = close(pipefd[1]);
-    serverAssert(!ret);
-    ret = munmap(p, map_size);
-    serverAssert(!ret);
+    if (pipefd[0] != -1) close(pipefd[0]);
+    if (pipefd[1] != -1) close(pipefd[1]);
+    if (p != NULL) munmap(p, map_size);
 
     return bug_found;
 }
@@ -5901,10 +5929,17 @@ int main(int argc, char **argv) {
     #ifdef __linux__
         linuxMemoryWarnings();
     #if defined (__arm64__)
-        if (linuxMadvFreeForkBugCheck()) {
-            serverLog(LL_WARNING,"WARNING Your kernel has a bug that could lead to data corruption during background save. Please upgrade to the latest stable kernel.");
+        int ret;
+        if ((ret = linuxMadvFreeForkBugCheck())) {
+            if (ret == 1)
+                serverLog(LL_WARNING,"WARNING Your kernel has a bug that could lead to data corruption during background save. "
+                                     "Please upgrade to the latest stable kernel.");
+            else
+                serverLog(LL_WARNING, "Failed to test the kernel for a bug that could lead to data corruption during background save. "
+                                      "Your system could be affected, please report this error.");
             if (!checkIgnoreWarning("ARM64-COW-BUG")) {
-                serverLog(LL_WARNING,"Redis will now exit to prevent data corruption. Note that it is possible to suppress this warning by setting the following config: ignore-warnings ARM64-COW-BUG");
+                serverLog(LL_WARNING,"Redis will now exit to prevent data corruption. "
+                                     "Note that it is possible to suppress this warning by setting the following config: ignore-warnings ARM64-COW-BUG");
                 exit(1);
             }
         }

From 17b34c73091968a5a37a267f69563c155efe8ce4 Mon Sep 17 00:00:00 2001
From: "Z. Liu" <clan@users.noreply.github.com>
Date: Thu, 28 Jan 2021 17:12:39 +0800
Subject: [PATCH 1007/1098] Add 'set-proc-title' config so that this mechanism
 can be disabled (#3623)

if option `set-proc-title' is no, then do nothing for proc title.

The reason has been explained long ago, see following:

We update redis to 2.8.8, then found there are some side effect when
redis always change the process title.

We run several slave instance on one computer, and all these salves
listen on unix socket only, then ps will show:

  1 S redis 18036 1 0 80 0 - 56130 ep_pol 14:02 ? 00:00:31 /usr/sbin/redis-server *:0
  1 S redis 23949 1 0 80 0 - 11074 ep_pol 15:41 ? 00:00:00 /usr/sbin/redis-server *:0

for redis 2.6 the output of ps is like following:

  1 S redis 18036 1 0 80 0 - 56130 ep_pol 14:02 ? 00:00:31 /usr/sbin/redis-server /etc/redis/a.conf
  1 S redis 23949 1 0 80 0 - 11074 ep_pol 15:41 ? 00:00:00 /usr/sbin/redis-server /etc/redis/b.conf

Later is more informational in our case. The situation
is worse when we manage the config and process running
state by salt. Salt check the process by running "ps |
grep SIG" (for Gentoo System) to check the running
state, where SIG is the string to search for when
looking for the service process with ps. Previously, we
define sig as "/usr/sbin/redis-server
/etc/redis/a.conf". Since the ps output is identical for
our case, so we have no way to check the state of
specified redis instance.

So, for our case, we prefer the old behavior, i.e, do
not change the process title for the main redis process.
Or add an option such as "set-proc-title [yes|no]" to
control this behavior.

Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
---
 redis.conf                   | 5 +++++
 src/config.c                 | 1 +
 src/server.c                 | 2 +-
 src/server.h                 | 1 +
 tests/unit/introspection.tcl | 1 +
 5 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/redis.conf b/redis.conf
index a131b406f..370542344 100644
--- a/redis.conf
+++ b/redis.conf
@@ -325,6 +325,11 @@ databases 16
 # ASCII art logo in startup logs by setting the following option to yes.
 always-show-logo no
 
+# By default, Redis modifies the process title (as seen in 'top' and 'ps') to
+# provide some runtime information. It is possible to disable this and leave
+# the process name as executed by setting the following to no.
+set-proc-title yes
+
 ################################ SNAPSHOTTING  ################################
 
 # Save the DB to disk.
diff --git a/src/config.c b/src/config.c
index 5db08f8bc..20d432573 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2388,6 +2388,7 @@ standardConfig configs[] = {
     createBoolConfig("rdb-del-sync-files", NULL, MODIFIABLE_CONFIG, server.rdb_del_sync_files, 0, NULL, NULL),
     createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, 1, NULL, NULL),
     createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, 1, NULL, NULL),
+    createBoolConfig("set-proc-title", NULL, IMMUTABLE_CONFIG, server.set_proc_title, 1, NULL, NULL), /* Should setproctitle be used? */
     createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, 1, NULL, NULL), /* Adapt hz to # of clients.*/
     createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, 0, NULL, NULL),
     createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, 0, NULL, NULL),
diff --git a/src/server.c b/src/server.c
index a0badea87..ce34af3df 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5919,7 +5919,7 @@ int main(int argc, char **argv) {
     readOOMScoreAdj();
     initServer();
     if (background || server.pidfile) createPidFile();
-    redisSetProcTitle(argv[0]);
+    if (server.set_proc_title) redisSetProcTitle(argv[0]);
     redisAsciiArt();
     checkTcpBacklogSettings();
 
diff --git a/src/server.h b/src/server.h
index 83e007ebe..b72d7bd0d 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1297,6 +1297,7 @@ struct redisServer {
     int supervised;                 /* 1 if supervised, 0 otherwise. */
     int supervised_mode;            /* See SUPERVISED_* */
     int daemonize;                  /* True if running as a daemon */
+    int set_proc_title;             /* True if change proc title */
     clientBufferLimitsConfig client_obuf_limits[CLIENT_TYPE_OBUF_COUNT];
     /* AOF persistence */
     int aof_enabled;                /* AOF configuration */
diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl
index 0a7f7a9c9..ba28341ff 100644
--- a/tests/unit/introspection.tcl
+++ b/tests/unit/introspection.tcl
@@ -112,6 +112,7 @@ start_server {tags {"introspection"}} {
             bio_cpulist
             aof_rewrite_cpulist
             bgsave_cpulist
+            set-proc-title
         }
 
         if {!$::tls} {

From bb7cd97439aa91698bee192cddd7ceb18d628a00 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 28 Jan 2021 12:09:11 +0200
Subject: [PATCH 1008/1098] Add hostname support in Sentinel. (#8282)

This is both a bugfix and an enhancement.

Internally, Sentinel relies entirely on IP addresses to identify
instances. When configured with a new master, it also requires users to
specify and IP and not hostname.

However, replicas may use the replica-announce-ip configuration to
announce a hostname. When that happens, Sentinel fails to match the
announced hostname with the expected IP and considers that a different
instance, triggering reconfiguration, etc.

Another use case is where TLS is used and clients are expected to match
the hostname to connect to with the certificate's SAN attribute. To
properly implement this configuration, it is necessary for Sentinel to
redirect clients to a hostname rather than an IP address.

The new 'resolve-hostnames' configuration parameter determines if
Sentinel is willing to accept hostnames. It is set by default to no,
which maintains backwards compatibility and avoids unexpected DNS
resolution delays on systems with DNS configuration issues.

Internally, Sentinel continues to identify instances by their resolved
IP address and will also report the IP by default. The new
'announce-hostnames' parameter determines if Sentinel should prefer to
announce a hostname, when available, rather than an IP address. This
applies to addresses returned to clients, as well as their
representation in the configuration file, REPLICAOF configuration
commands, etc.

This commit also introduces SENTINEL CONFIG GET and SENTINEL CONFIG SET
which can be used to introspect or configure global Sentinel
configuration that was previously was only possible by directly
accessing the configuration file and possibly restarting the instance.

Co-authored-by: myl1024 <myl92916@qq.com>
Co-authored-by: sundb <sundbcn@gmail.com>
---
 sentinel.conf                                |  18 +
 src/anet.c                                   |  15 +-
 src/anet.h                                   |   3 +-
 src/sentinel.c                               | 326 +++++++++++++++----
 tests/instances.tcl                          |  11 +-
 tests/sentinel/tests/08-hostname-conf.tcl    |  62 ++++
 tests/sentinel/tests/09-acl-support.tcl      |  50 +++
 tests/sentinel/tests/includes/init-tests.tcl |   6 +-
 8 files changed, 411 insertions(+), 80 deletions(-)
 create mode 100644 tests/sentinel/tests/08-hostname-conf.tcl
 create mode 100644 tests/sentinel/tests/09-acl-support.tcl

diff --git a/sentinel.conf b/sentinel.conf
index 39d6929e7..8647379d8 100644
--- a/sentinel.conf
+++ b/sentinel.conf
@@ -321,3 +321,21 @@ sentinel deny-scripts-reconfig yes
 # is possible to just rename a command to itself:
 #
 # SENTINEL rename-command mymaster CONFIG CONFIG
+
+# HOSTNAMES SUPPORT
+#
+# Normally Sentinel uses only IP addresses and requires SENTINEL MONITOR
+# to specify an IP address. Also, it requires the Redis replica-announce-ip
+# keyword to specify only IP addresses.
+#
+# You may enable hostnames support by enabling resolve-hostnames. Note
+# that you must make sure your DNS is configured properly and that DNS
+# resolution does not introduce very long delays.
+#
+SENTINEL resolve-hostnames no
+
+# When resolve-hostnames is enabled, Sentinel still uses IP addresses
+# when exposing instances to users, configuration files, etc. If you want
+# to retain the hostnames when announced, enable announce-hostnames below.
+#
+SENTINEL announce-hostnames no
diff --git a/src/anet.c b/src/anet.c
index f2c39b200..0bfa575f5 100644
--- a/src/anet.c
+++ b/src/anet.c
@@ -235,14 +235,13 @@ int anetRecvTimeout(char *err, int fd, long long ms) {
     return ANET_OK;
 }
 
-/* anetGenericResolve() is called by anetResolve() and anetResolveIP() to
- * do the actual work. It resolves the hostname "host" and set the string
- * representation of the IP address into the buffer pointed by "ipbuf".
+/* Resolve the hostname "host" and set the string representation of the
+ * IP address into the buffer pointed by "ipbuf".
  *
  * If flags is set to ANET_IP_ONLY the function only resolves hostnames
  * that are actually already IPv4 or IPv6 addresses. This turns the function
  * into a validating / normalizing function. */
-int anetGenericResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len,
+int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len,
                        int flags)
 {
     struct addrinfo hints, *info;
@@ -269,14 +268,6 @@ int anetGenericResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len,
     return ANET_OK;
 }
 
-int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len) {
-    return anetGenericResolve(err,host,ipbuf,ipbuf_len,ANET_NONE);
-}
-
-int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len) {
-    return anetGenericResolve(err,host,ipbuf,ipbuf_len,ANET_IP_ONLY);
-}
-
 static int anetSetReuseAddr(char *err, int fd) {
     int yes = 1;
     /* Make sure connection-intensive things like the redis benchmark
diff --git a/src/anet.h b/src/anet.h
index dc3cbeb32..5da2f3b46 100644
--- a/src/anet.h
+++ b/src/anet.h
@@ -60,8 +60,7 @@ int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port,
 int anetUnixConnect(char *err, const char *path);
 int anetUnixNonBlockConnect(char *err, const char *path);
 int anetRead(int fd, char *buf, int count);
-int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len);
-int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len);
+int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, int flags);
 int anetTcpServer(char *err, int port, char *bindaddr, int backlog);
 int anetTcp6Server(char *err, int port, char *bindaddr, int backlog);
 int anetUnixServer(char *err, char *path, mode_t perm, int backlog);
diff --git a/src/sentinel.c b/src/sentinel.c
index 7f79e702a..a87766ebe 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -55,7 +55,8 @@ extern SSL_CTX *redis_tls_client_ctx;
 
 /* Address object, used to describe an ip:port pair. */
 typedef struct sentinelAddr {
-    char *ip;
+    char *hostname;         /* Hostname OR address, as specified */
+    char *ip;               /* Always a resolved address */
     int port;
 } sentinelAddr;
 
@@ -94,6 +95,8 @@ typedef struct sentinelAddr {
 #define SENTINEL_ELECTION_TIMEOUT 10000
 #define SENTINEL_MAX_DESYNC 1000
 #define SENTINEL_DEFAULT_DENY_SCRIPTS_RECONFIG 1
+#define SENTINEL_DEFAULT_RESOLVE_HOSTNAMES 0
+#define SENTINEL_DEFAULT_ANNOUNCE_HOSTNAMES 0
 
 /* Failover machine different states. */
 #define SENTINEL_FAILOVER_STATE_NONE 0  /* No failover in progress. */
@@ -260,6 +263,8 @@ struct sentinelState {
                                   paths at runtime? */
     char *sentinel_auth_pass;    /* Password to use for AUTH against other sentinel */
     char *sentinel_auth_user;    /* Username for ACLs AUTH against other sentinel. */
+    int resolve_hostnames;       /* Support use of hostnames, assuming DNS is well configured. */
+    int announce_hostnames;      /* Announce hostnames instead of IPs when we have them. */
 } sentinel;
 
 /* A script execution job. */
@@ -387,7 +392,7 @@ sentinelRedisInstance *sentinelSelectSlave(sentinelRedisInstance *master);
 void sentinelScheduleScriptExecution(char *path, ...);
 void sentinelStartFailover(sentinelRedisInstance *master);
 void sentinelDiscardReplyCallback(redisAsyncContext *c, void *reply, void *privdata);
-int sentinelSendSlaveOf(sentinelRedisInstance *ri, char *host, int port);
+int sentinelSendSlaveOf(sentinelRedisInstance *ri, const sentinelAddr *addr);
 char *sentinelVoteLeader(sentinelRedisInstance *master, uint64_t req_epoch, char *req_runid, uint64_t *leader_epoch);
 void sentinelFlushConfig(void);
 void sentinelGenerateInitialMonitorEvents(void);
@@ -455,6 +460,8 @@ void sentinelInfoCommand(client *c);
 void sentinelSetCommand(client *c);
 void sentinelPublishCommand(client *c);
 void sentinelRoleCommand(client *c);
+void sentinelConfigGetCommand(client *c);
+void sentinelConfigSetCommand(client *c);
 
 struct redisCommand sentinelcmds[] = {
     {"ping",pingCommand,1,"fast @connection",0,NULL,0,0,0,0,0},
@@ -535,6 +542,8 @@ void initSentinel(void) {
     sentinel.deny_scripts_reconfig = SENTINEL_DEFAULT_DENY_SCRIPTS_RECONFIG;
     sentinel.sentinel_auth_pass = NULL;
     sentinel.sentinel_auth_user = NULL;
+    sentinel.resolve_hostnames = SENTINEL_DEFAULT_RESOLVE_HOSTNAMES;
+    sentinel.announce_hostnames = SENTINEL_DEFAULT_ANNOUNCE_HOSTNAMES;
     memset(sentinel.myid,0,sizeof(sentinel.myid));
     server.sentinel_config = NULL;
 }
@@ -590,11 +599,13 @@ sentinelAddr *createSentinelAddr(char *hostname, int port) {
         errno = EINVAL;
         return NULL;
     }
-    if (anetResolve(NULL,hostname,ip,sizeof(ip)) == ANET_ERR) {
+    if (anetResolve(NULL,hostname,ip,sizeof(ip),
+                    sentinel.resolve_hostnames ? ANET_NONE : ANET_IP_ONLY) == ANET_ERR) {
         errno = ENOENT;
         return NULL;
     }
     sa = zmalloc(sizeof(*sa));
+    sa->hostname = sdsnew(hostname);
     sa->ip = sdsnew(ip);
     sa->port = port;
     return sa;
@@ -605,6 +616,7 @@ sentinelAddr *dupSentinelAddr(sentinelAddr *src) {
     sentinelAddr *sa;
 
     sa = zmalloc(sizeof(*sa));
+    sa->hostname = sdsnew(src->hostname);
     sa->ip = sdsnew(src->ip);
     sa->port = src->port;
     return sa;
@@ -612,6 +624,7 @@ sentinelAddr *dupSentinelAddr(sentinelAddr *src) {
 
 /* Free a Sentinel address. Can't fail. */
 void releaseSentinelAddr(sentinelAddr *sa) {
+    sdsfree(sa->hostname);
     sdsfree(sa->ip);
     zfree(sa);
 }
@@ -621,6 +634,21 @@ int sentinelAddrIsEqual(sentinelAddr *a, sentinelAddr *b) {
     return a->port == b->port && !strcasecmp(a->ip,b->ip);
 }
 
+/* Return non-zero if a hostname matches an address. */
+int sentinelAddrEqualsHostname(sentinelAddr *a, char *hostname) {
+    char ip[NET_IP_STR_LEN];
+
+    /* We always resolve the hostname and compare it to the address */
+    if (anetResolve(NULL, hostname, ip, sizeof(ip),
+                    sentinel.resolve_hostnames ? ANET_NONE : ANET_IP_ONLY) == ANET_ERR)
+        return 0;
+    return !strcasecmp(a->ip, ip);
+}
+
+const char *announceSentinelAddr(const sentinelAddr *a) {
+    return sentinel.announce_hostnames ? a->hostname : a->ip;
+}
+
 /* =========================== Events notification ========================== */
 
 /* Send an event to log, pub/sub, user notification script.
@@ -661,12 +689,12 @@ void sentinelEvent(int level, char *type, sentinelRedisInstance *ri,
         if (master) {
             snprintf(msg, sizeof(msg), "%s %s %s %d @ %s %s %d",
                 sentinelRedisInstanceTypeStr(ri),
-                ri->name, ri->addr->ip, ri->addr->port,
-                master->name, master->addr->ip, master->addr->port);
+                ri->name, announceSentinelAddr(ri->addr), ri->addr->port,
+                master->name, announceSentinelAddr(master->addr), master->addr->port);
         } else {
             snprintf(msg, sizeof(msg), "%s %s %s %d",
                 sentinelRedisInstanceTypeStr(ri),
-                ri->name, ri->addr->ip, ri->addr->port);
+                ri->name, announceSentinelAddr(ri->addr), ri->addr->port);
         }
         fmt += 2;
     } else {
@@ -988,7 +1016,8 @@ void sentinelCallClientReconfScript(sentinelRedisInstance *master, int role, cha
     sentinelScheduleScriptExecution(master->client_reconfig_script,
         master->name,
         (role == SENTINEL_LEADER) ? "leader" : "observer",
-        state, from->ip, fromport, to->ip, toport, NULL);
+        state, announceSentinelAddr(from), fromport,
+        announceSentinelAddr(to), toport, NULL);
 }
 
 /* =============================== instanceLink ============================= */
@@ -1114,6 +1143,35 @@ int sentinelTryConnectionSharing(sentinelRedisInstance *ri) {
     return C_ERR;
 }
 
+/* Drop all connections to other sentinels. Returns the number of connections
+ * dropped.*/
+int sentinelDropConnections(void) {
+    dictIterator *di;
+    dictEntry *de;
+    int dropped = 0;
+
+    di = dictGetIterator(sentinel.masters);
+    while ((de = dictNext(di)) != NULL) {
+        dictIterator *sdi;
+        dictEntry *sde;
+
+        sentinelRedisInstance *ri = dictGetVal(de);
+        sdi = dictGetIterator(ri->sentinels);
+        while ((sde = dictNext(sdi)) != NULL) {
+            sentinelRedisInstance *si = dictGetVal(sde);
+            if (!si->link->disconnected) {
+                instanceLinkCloseConnection(si->link, si->link->pc);
+                instanceLinkCloseConnection(si->link, si->link->cc);
+                dropped++;
+            }
+        }
+        dictReleaseIterator(sdi);
+    }
+    dictReleaseIterator(di);
+
+    return dropped;
+}
+
 /* When we detect a Sentinel to switch address (reporting a different IP/port
  * pair in Hello messages), let's update all the matching Sentinels in the
  * context of other masters as well and disconnect the links, so that everybody
@@ -1226,7 +1284,7 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char *
 
     /* For slaves use ip:port as name. */
     if (flags & SRI_SLAVE) {
-        anetFormatAddr(slavename, sizeof(slavename), hostname, port);
+        anetFormatAddr(slavename, sizeof(slavename), addr->ip, port);
         name = slavename;
     }
 
@@ -1337,14 +1395,25 @@ void releaseSentinelRedisInstance(sentinelRedisInstance *ri) {
 
 /* Lookup a slave in a master Redis instance, by ip and port. */
 sentinelRedisInstance *sentinelRedisInstanceLookupSlave(
-                sentinelRedisInstance *ri, char *ip, int port)
+                sentinelRedisInstance *ri, char *slave_addr, int port)
 {
     sds key;
     sentinelRedisInstance *slave;
     char buf[NET_ADDR_STR_LEN];
+    sentinelAddr *addr;
 
     serverAssert(ri->flags & SRI_MASTER);
-    anetFormatAddr(buf,sizeof(buf),ip,port);
+
+    /* We need to handle a slave_addr that is potentially a hostname.
+     * If that is the case, depending on configuration we either resolve
+     * it and use the IP addres or fail.
+     */
+    addr = createSentinelAddr(slave_addr, port);
+    if (!addr) return NULL;
+
+    anetFormatAddr(buf,sizeof(buf),addr->ip,addr->port);
+    releaseSentinelAddr(addr);
+
     key = sdsnew(buf);
     slave = dictFetchValue(ri->slaves,key);
     sdsfree(key);
@@ -1394,21 +1463,27 @@ int removeMatchingSentinelFromMaster(sentinelRedisInstance *master, char *runid)
  * of instances. Return NULL if not found, otherwise return the instance
  * pointer.
  *
- * runid or ip can be NULL. In such a case the search is performed only
+ * runid or addr can be NULL. In such a case the search is performed only
  * by the non-NULL field. */
-sentinelRedisInstance *getSentinelRedisInstanceByAddrAndRunID(dict *instances, char *ip, int port, char *runid) {
+sentinelRedisInstance *getSentinelRedisInstanceByAddrAndRunID(dict *instances, char *addr, int port, char *runid) {
     dictIterator *di;
     dictEntry *de;
     sentinelRedisInstance *instance = NULL;
+    sentinelAddr *ri_addr = NULL;
 
-    serverAssert(ip || runid);   /* User must pass at least one search param. */
+    serverAssert(addr || runid);   /* User must pass at least one search param. */
+    if (addr != NULL) {
+        /* Resolve addr, we use the IP as a key even if a hostname is used */
+        ri_addr = createSentinelAddr(addr, port);
+        if (!ri_addr) return NULL;
+    }
     di = dictGetIterator(instances);
     while((de = dictNext(di)) != NULL) {
         sentinelRedisInstance *ri = dictGetVal(de);
 
         if (runid && !ri->runid) continue;
         if ((runid == NULL || strcmp(ri->runid, runid) == 0) &&
-            (ip == NULL || (strcmp(ri->addr->ip, ip) == 0 &&
+            (addr == NULL || (strcmp(ri->addr->ip, ri_addr->ip) == 0 &&
                             ri->addr->port == port)))
         {
             instance = ri;
@@ -1416,6 +1491,9 @@ sentinelRedisInstance *getSentinelRedisInstanceByAddrAndRunID(dict *instances, c
         }
     }
     dictReleaseIterator(di);
+    if (ri_addr != NULL)
+        releaseSentinelAddr(ri_addr);
+
     return instance;
 }
 
@@ -1530,14 +1608,14 @@ int sentinelResetMastersByPattern(char *pattern, int flags) {
  *
  * The function returns C_ERR if the address can't be resolved for some
  * reason. Otherwise C_OK is returned.  */
-int sentinelResetMasterAndChangeAddress(sentinelRedisInstance *master, char *ip, int port) {
+int sentinelResetMasterAndChangeAddress(sentinelRedisInstance *master, char *hostname, int port) {
     sentinelAddr *oldaddr, *newaddr;
     sentinelAddr **slaves = NULL;
     int numslaves = 0, j;
     dictIterator *di;
     dictEntry *de;
 
-    newaddr = createSentinelAddr(ip,port);
+    newaddr = createSentinelAddr(hostname,port);
     if (newaddr == NULL) return C_ERR;
 
     /* There can be only 0 or 1 slave that has the newaddr.
@@ -1551,8 +1629,7 @@ int sentinelResetMasterAndChangeAddress(sentinelRedisInstance *master, char *ip,
         sentinelRedisInstance *slave = dictGetVal(de);
 
         if (sentinelAddrIsEqual(slave->addr,newaddr)) continue;
-        slaves[numslaves++] = createSentinelAddr(slave->addr->ip,
-                                                 slave->addr->port);
+        slaves[numslaves++] = dupSentinelAddr(slave->addr);
     }
     dictReleaseIterator(di);
 
@@ -1560,8 +1637,7 @@ int sentinelResetMasterAndChangeAddress(sentinelRedisInstance *master, char *ip,
      * as a slave as well, so that we'll be able to sense / reconfigure
      * the old master. */
     if (!sentinelAddrIsEqual(newaddr,master->addr)) {
-        slaves[numslaves++] = createSentinelAddr(master->addr->ip,
-                                                 master->addr->port);
+        slaves[numslaves++] = dupSentinelAddr(master->addr);
     }
 
     /* Reset and switch address. */
@@ -1575,7 +1651,7 @@ int sentinelResetMasterAndChangeAddress(sentinelRedisInstance *master, char *ip,
     for (j = 0; j < numslaves; j++) {
         sentinelRedisInstance *slave;
 
-        slave = createSentinelRedisInstance(NULL,SRI_SLAVE,slaves[j]->ip,
+        slave = createSentinelRedisInstance(NULL,SRI_SLAVE,slaves[j]->hostname,
                     slaves[j]->port, master->quorum, master);
         releaseSentinelAddr(slaves[j]);
         if (slave) sentinelEvent(LL_NOTICE,"+slave",slave,"%@");
@@ -1959,6 +2035,16 @@ const char *sentinelHandleConfiguration(char **argv, int argc) {
         /* sentinel-pass <password> */
         if (strlen(argv[1]))
             sentinel.sentinel_auth_pass = sdsnew(argv[1]);
+    } else if (!strcasecmp(argv[0],"resolve-hostnames") && argc == 2) {
+        /* resolve-hostnames <yes|no> */
+        if ((sentinel.resolve_hostnames = yesnotoi(argv[1])) == -1) {
+            return "Please specify yes or not for the resolve-hostnames option.";
+        }
+    } else if (!strcasecmp(argv[0],"announce-hostnames") && argc == 2) {
+        /* announce-hostnames <yes|no> */
+        if ((sentinel.announce_hostnames = yesnotoi(argv[1])) == -1) {
+            return "Please specify yes or not for the announce-hostnames option.";
+        }
     } else {
         return "Unrecognized sentinel configuration statement.";
     }
@@ -1985,6 +2071,21 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
     rewriteConfigRewriteLine(state,"sentinel deny-scripts-reconfig",line,
         sentinel.deny_scripts_reconfig != SENTINEL_DEFAULT_DENY_SCRIPTS_RECONFIG);
 
+    /* sentinel resolve-hostnames.
+     * This must be included early in the file so it is already in effect
+     * when reading the file.
+     */
+    line = sdscatprintf(sdsempty(), "sentinel resolve-hostnames %s",
+                        sentinel.resolve_hostnames ? "yes" : "no");
+    rewriteConfigRewriteLine(state,"sentinel",line,
+                             sentinel.resolve_hostnames != SENTINEL_DEFAULT_RESOLVE_HOSTNAMES);
+
+    /* sentinel announce-hostnames. */
+    line = sdscatprintf(sdsempty(), "sentinel announce-hostnames %s",
+                        sentinel.announce_hostnames ? "yes" : "no");
+    rewriteConfigRewriteLine(state,"sentinel",line,
+                             sentinel.announce_hostnames != SENTINEL_DEFAULT_ANNOUNCE_HOSTNAMES);
+
     /* For every master emit a "sentinel monitor" config entry. */
     di = dictGetIterator(sentinel.masters);
     while((de = dictNext(di)) != NULL) {
@@ -1995,7 +2096,7 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
         master = dictGetVal(de);
         master_addr = sentinelGetCurrentMasterAddress(master);
         line = sdscatprintf(sdsempty(),"sentinel monitor %s %s %d %d",
-            master->name, master_addr->ip, master_addr->port,
+            master->name, announceSentinelAddr(master_addr), master_addr->port,
             master->quorum);
         rewriteConfigRewriteLine(state,"sentinel monitor",line,1);
         /* rewriteConfigMarkAsProcessed is handled after the loop */
@@ -2095,7 +2196,7 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
                 slave_addr = master->addr;
             line = sdscatprintf(sdsempty(),
                 "sentinel known-replica %s %s %d",
-                master->name, slave_addr->ip, slave_addr->port);
+                master->name, announceSentinelAddr(slave_addr), slave_addr->port);
             rewriteConfigRewriteLine(state,"sentinel known-replica",line,1);
             /* rewriteConfigMarkAsProcessed is handled after the loop */
         }
@@ -2108,7 +2209,7 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
             if (ri->runid == NULL) continue;
             line = sdscatprintf(sdsempty(),
                 "sentinel known-sentinel %s %s %d %s",
-                master->name, ri->addr->ip, ri->addr->port, ri->runid);
+                master->name, announceSentinelAddr(ri->addr), ri->addr->port, ri->runid);
             rewriteConfigRewriteLine(state,"sentinel known-sentinel",line,1);
             /* rewriteConfigMarkAsProcessed is handled after the loop */
         }
@@ -2242,7 +2343,7 @@ void sentinelSendAuthIfNeeded(sentinelRedisInstance *ri, redisAsyncContext *c) {
         auth_user = ri->master->auth_user;
     } else if (ri->flags & SRI_SENTINEL) {
         /* If sentinel_auth_user is NULL, AUTH will use default user
-           with sentinel_auth_pass to autenticate */
+           with sentinel_auth_pass to authenticate */
         if (sentinel.sentinel_auth_pass) {
             auth_pass = sentinel.sentinel_auth_pass;
             auth_user = sentinel.sentinel_auth_user;
@@ -2589,9 +2690,7 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) {
                  sentinelRedisInstanceNoDownFor(ri,wait_time) &&
                  mstime() - ri->role_reported_time > wait_time)
             {
-                int retval = sentinelSendSlaveOf(ri,
-                        ri->master->addr->ip,
-                        ri->master->addr->port);
+                int retval = sentinelSendSlaveOf(ri,ri->master->addr);
                 if (retval == C_OK)
                     sentinelEvent(LL_NOTICE,"+convert-to-slave",ri,"%@");
             }
@@ -2602,7 +2701,7 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) {
     if ((ri->flags & SRI_SLAVE) &&
         role == SRI_SLAVE &&
         (ri->slave_master_port != ri->master->addr->port ||
-         strcasecmp(ri->slave_master_host,ri->master->addr->ip)))
+         !sentinelAddrEqualsHostname(ri->master->addr, ri->slave_master_host)))
     {
         mstime_t wait_time = ri->master->failover_timeout;
 
@@ -2612,9 +2711,7 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) {
             sentinelRedisInstanceNoDownFor(ri,wait_time) &&
             mstime() - ri->slave_conf_change_time > wait_time)
         {
-            int retval = sentinelSendSlaveOf(ri,
-                    ri->master->addr->ip,
-                    ri->master->addr->port);
+            int retval = sentinelSendSlaveOf(ri,ri->master->addr);
             if (retval == C_OK)
                 sentinelEvent(LL_NOTICE,"+fix-slave-config",ri,"%@");
         }
@@ -2628,8 +2725,8 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) {
         /* SRI_RECONF_SENT -> SRI_RECONF_INPROG. */
         if ((ri->flags & SRI_RECONF_SENT) &&
             ri->slave_master_host &&
-            strcmp(ri->slave_master_host,
-                    ri->master->promoted_slave->addr->ip) == 0 &&
+            sentinelAddrEqualsHostname(ri->master->promoted_slave->addr,
+                ri->slave_master_host) &&
             ri->slave_master_port == ri->master->promoted_slave->addr->port)
         {
             ri->flags &= ~SRI_RECONF_SENT;
@@ -2806,7 +2903,7 @@ void sentinelProcessHelloMessage(char *hello, int hello_len) {
         if (si && master->config_epoch < master_config_epoch) {
             master->config_epoch = master_config_epoch;
             if (master_port != master->addr->port ||
-                strcmp(master->addr->ip, token[5]))
+                !sentinelAddrEqualsHostname(master->addr, token[5]))
             {
                 sentinelAddr *old_addr;
 
@@ -2814,7 +2911,7 @@ void sentinelProcessHelloMessage(char *hello, int hello_len) {
                 sentinelEvent(LL_WARNING,"+switch-master",
                     master,"%s %s %d %s %d",
                     master->name,
-                    master->addr->ip, master->addr->port,
+                    announceSentinelAddr(master->addr), master->addr->port,
                     token[5], master_port);
 
                 old_addr = dupSentinelAddr(master->addr);
@@ -2907,7 +3004,7 @@ int sentinelSendHello(sentinelRedisInstance *ri) {
         announce_ip, announce_port, sentinel.myid,
         (unsigned long long) sentinel.current_epoch,
         /* --- */
-        master->name,master_addr->ip,master_addr->port,
+        master->name,announceSentinelAddr(master_addr),master_addr->port,
         (unsigned long long) master->config_epoch);
     retval = redisAsyncCommand(ri->link->cc,
         sentinelPublishReplyCallback, ri, "%s %s %s",
@@ -3041,6 +3138,101 @@ void sentinelSendPeriodicCommands(sentinelRedisInstance *ri) {
 
 /* =========================== SENTINEL command ============================= */
 
+/* SENTINEL CONFIG SET <option> */
+void sentinelConfigSetCommand(client *c) {
+    robj *o = c->argv[3];
+    robj *val = c->argv[4];
+    long long numval;
+    int drop_conns = 0;
+
+    if (!strcasecmp(o->ptr, "resolve-hostnames")) {
+        if ((numval = yesnotoi(val->ptr)) == -1) goto badfmt;
+        sentinel.resolve_hostnames = numval;
+    } else if (!strcasecmp(o->ptr, "announce-hostnames")) {
+        if ((numval = yesnotoi(val->ptr)) == -1) goto badfmt;
+        sentinel.announce_hostnames = numval;
+    } else if (!strcasecmp(o->ptr, "announce-ip")) {
+        if (sentinel.announce_ip) sdsfree(sentinel.announce_ip);
+        sentinel.announce_ip = sdsnew(val->ptr);
+    } else if (!strcasecmp(o->ptr, "announce-port")) {
+        if (getLongLongFromObject(val, &numval) == C_ERR ||
+            numval < 0 || numval > 65535)
+            goto badfmt;
+        sentinel.announce_port = numval;
+    } else if (!strcasecmp(o->ptr, "sentinel-user")) {
+        sdsfree(sentinel.sentinel_auth_user);
+        sentinel.sentinel_auth_user = sdsnew(val->ptr);
+        drop_conns = 1;
+    } else if (!strcasecmp(o->ptr, "sentinel-pass")) {
+        sdsfree(sentinel.sentinel_auth_pass);
+        sentinel.sentinel_auth_pass = sdsnew(val->ptr);
+        drop_conns = 1;
+    } else {
+        addReplyErrorFormat(c, "Invalid argument '%s' to SENTINEL CONFIG SET",
+                            (char *) o->ptr);
+        return;
+    }
+
+    sentinelFlushConfig();
+    addReply(c, shared.ok);
+
+    /* Drop Sentinel connections to initiate a reconnect if needed. */
+    if (drop_conns)
+        sentinelDropConnections();
+
+    return;
+
+badfmt:
+    addReplyErrorFormat(c, "Invalid value '%s' to SENTINEL CONFIG SET '%s'",
+                        (char *) val->ptr, (char *) o->ptr);
+}
+
+/* SENTINEL CONFIG GET <option> */
+void sentinelConfigGetCommand(client *c) {
+    robj *o = c->argv[3];
+    const char *pattern = o->ptr;
+    void *replylen = addReplyDeferredLen(c);
+    int matches = 0;
+
+    if (stringmatch(pattern,"resolve-hostnames",1)) {
+        addReplyBulkCString(c,"resolve-hostnames");
+        addReplyBulkCString(c,sentinel.resolve_hostnames ? "yes" : "no");
+        matches++;
+    }
+
+    if (stringmatch(pattern, "announce-hostnames", 1)) {
+        addReplyBulkCString(c,"announce-hostnames");
+        addReplyBulkCString(c,sentinel.announce_hostnames ? "yes" : "no");
+        matches++;
+    }
+
+    if (stringmatch(pattern, "announce-ip", 1)) {
+        addReplyBulkCString(c,"announce-ip");
+        addReplyBulkCString(c,sentinel.announce_ip ? sentinel.announce_ip : "");
+        matches++;
+    }
+
+    if (stringmatch(pattern, "announce-port", 1)) {
+        addReplyBulkCString(c, "announce-port");
+        addReplyBulkLongLong(c, sentinel.announce_port);
+        matches++;
+    }
+
+    if (stringmatch(pattern, "sentinel-user", 1)) {
+        addReplyBulkCString(c, "sentinel-user");
+        addReplyBulkCString(c, sentinel.sentinel_auth_user ? sentinel.sentinel_auth_user : "");
+        matches++;
+    }
+
+    if (stringmatch(pattern, "sentinel-pass", 1)) {
+        addReplyBulkCString(c, "sentinel-pass");
+        addReplyBulkCString(c, sentinel.sentinel_auth_pass ? sentinel.sentinel_auth_pass : "");
+        matches++;
+    }
+
+    setDeferredMapLen(c, replylen, matches);
+}
+
 const char *sentinelFailoverStateStr(int state) {
     switch(state) {
     case SENTINEL_FAILOVER_STATE_NONE: return "none";
@@ -3067,7 +3259,7 @@ void addReplySentinelRedisInstance(client *c, sentinelRedisInstance *ri) {
     fields++;
 
     addReplyBulkCString(c,"ip");
-    addReplyBulkCString(c,ri->addr->ip);
+    addReplyBulkCString(c,announceSentinelAddr(ri->addr));
     fields++;
 
     addReplyBulkCString(c,"port");
@@ -3308,6 +3500,10 @@ void sentinelCommand(client *c) {
 "    Check if the current Sentinel configuration is able to reach the quorum",
 "    needed to failover a master and the majority needed to authorize the",
 "    failover.",
+"CONFIG SET <param> <value>",
+"    Set a global Sentinel configuration parameter.",
+"CONFIG GET <param>",
+"    Get global Sentinel configuration parameter.",
 "GET-MASTER-ADDR-BY-NAME <master-name>",
 "    Return the ip and port number of the master with that name.",
 "FAILOVER <master-name>",
@@ -3449,7 +3645,7 @@ NULL
             sentinelAddr *addr = sentinelGetCurrentMasterAddress(ri);
 
             addReplyArrayLen(c,2);
-            addReplyBulkCString(c,addr->ip);
+            addReplyBulkCString(c,announceSentinelAddr(addr));
             addReplyBulkLongLong(c,addr->port);
         }
     } else if (!strcasecmp(c->argv[1]->ptr,"failover")) {
@@ -3494,11 +3690,12 @@ NULL
             return;
         }
 
-        /* Make sure the IP field is actually a valid IP before passing it
-         * to createSentinelRedisInstance(), otherwise we may trigger a
-         * DNS lookup at runtime. */
-        if (anetResolveIP(NULL,c->argv[3]->ptr,ip,sizeof(ip)) == ANET_ERR) {
-            addReplyError(c,"Invalid IP address specified");
+        /* If resolve-hostnames is used, actual DNS resolution may take place.
+         * Otherwise just validate address.
+         */
+        if (anetResolve(NULL,c->argv[3]->ptr,ip,sizeof(ip),
+                        sentinel.resolve_hostnames ? ANET_NONE : ANET_IP_ONLY) == ANET_ERR) {
+            addReplyError(c, "Invalid IP address or hostname specified");
             return;
         }
 
@@ -3568,6 +3765,14 @@ NULL
     } else if (!strcasecmp(c->argv[1]->ptr,"set")) {
         if (c->argc < 3) goto numargserr;
         sentinelSetCommand(c);
+    } else if (!strcasecmp(c->argv[1]->ptr,"config")) {
+        if (c->argc < 3) goto numargserr;
+        if (!strcasecmp(c->argv[2]->ptr,"set") && c->argc == 5)
+            sentinelConfigSetCommand(c);
+        else if (!strcasecmp(c->argv[2]->ptr,"get") && c->argc == 4)
+            sentinelConfigGetCommand(c);
+        else
+            addReplyError(c, "Only SENTINEL CONFIG GET <option> / SET <option> <value> are supported.");
     } else if (!strcasecmp(c->argv[1]->ptr,"info-cache")) {
         /* SENTINEL INFO-CACHE <name> */
         if (c->argc < 2) goto numargserr;
@@ -3731,7 +3936,7 @@ void sentinelInfoCommand(client *c) {
                 "master%d:name=%s,status=%s,address=%s:%d,"
                 "slaves=%lu,sentinels=%lu\r\n",
                 master_id++, ri->name, status,
-                ri->addr->ip, ri->addr->port,
+                announceSentinelAddr(ri->addr), ri->addr->port,
                 dictSize(ri->slaves),
                 dictSize(ri->sentinels)+1);
         }
@@ -4127,7 +4332,7 @@ void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master, int f
                     sentinelReceiveIsMasterDownReply, ri,
                     "%s is-master-down-by-addr %s %s %llu %s",
                     sentinelInstanceMapCommand(ri,"SENTINEL"),
-                    master->addr->ip, port,
+                    announceSentinelAddr(master->addr), port,
                     sentinel.current_epoch,
                     (master->failover_state > SENTINEL_FAILOVER_STATE_NONE) ?
                     sentinel.myid : "*");
@@ -4281,17 +4486,19 @@ char *sentinelGetLeader(sentinelRedisInstance *master, uint64_t epoch) {
  * The command returns C_OK if the SLAVEOF command was accepted for
  * (later) delivery otherwise C_ERR. The command replies are just
  * discarded. */
-int sentinelSendSlaveOf(sentinelRedisInstance *ri, char *host, int port) {
+int sentinelSendSlaveOf(sentinelRedisInstance *ri, const sentinelAddr *addr) {
     char portstr[32];
+    const char *host;
     int retval;
 
-    ll2string(portstr,sizeof(portstr),port);
-
     /* If host is NULL we send SLAVEOF NO ONE that will turn the instance
-     * into a master. */
-    if (host == NULL) {
+    * into a master. */
+    if (!addr) {
         host = "NO";
         memcpy(portstr,"ONE",4);
+    } else {
+        host = announceSentinelAddr(addr);
+        ll2string(portstr,sizeof(portstr),addr->port);
     }
 
     /* In order to send SLAVEOF in a safe way, we send a transaction performing
@@ -4576,7 +4783,7 @@ void sentinelFailoverSendSlaveOfNoOne(sentinelRedisInstance *ri) {
      * We actually register a generic callback for this command as we don't
      * really care about the reply. We check if it worked indirectly observing
      * if INFO returns a different role (master instead of slave). */
-    retval = sentinelSendSlaveOf(ri->promoted_slave,NULL,0);
+    retval = sentinelSendSlaveOf(ri->promoted_slave,NULL);
     if (retval != C_OK) return;
     sentinelEvent(LL_NOTICE, "+failover-state-wait-promotion",
         ri->promoted_slave,"%@");
@@ -4646,9 +4853,7 @@ void sentinelFailoverDetectEnd(sentinelRedisInstance *master) {
             if (slave->flags & (SRI_PROMOTED|SRI_RECONF_DONE|SRI_RECONF_SENT)) continue;
             if (slave->link->disconnected) continue;
 
-            retval = sentinelSendSlaveOf(slave,
-                    master->promoted_slave->addr->ip,
-                    master->promoted_slave->addr->port);
+            retval = sentinelSendSlaveOf(slave,master->promoted_slave->addr);
             if (retval == C_OK) {
                 sentinelEvent(LL_NOTICE,"+slave-reconf-sent-be",slave,"%@");
                 slave->flags |= SRI_RECONF_SENT;
@@ -4703,9 +4908,7 @@ void sentinelFailoverReconfNextSlave(sentinelRedisInstance *master) {
         if (slave->link->disconnected) continue;
 
         /* Send SLAVEOF <new master>. */
-        retval = sentinelSendSlaveOf(slave,
-                master->promoted_slave->addr->ip,
-                master->promoted_slave->addr->port);
+        retval = sentinelSendSlaveOf(slave,master->promoted_slave->addr);
         if (retval == C_OK) {
             slave->flags |= SRI_RECONF_SENT;
             slave->slave_reconf_sent_time = mstime();
@@ -4727,10 +4930,10 @@ void sentinelFailoverSwitchToPromotedSlave(sentinelRedisInstance *master) {
                                  master->promoted_slave : master;
 
     sentinelEvent(LL_WARNING,"+switch-master",master,"%s %s %d %s %d",
-        master->name, master->addr->ip, master->addr->port,
-        ref->addr->ip, ref->addr->port);
+        master->name, announceSentinelAddr(master->addr), master->addr->port,
+        announceSentinelAddr(ref->addr), ref->addr->port);
 
-    sentinelResetMasterAndChangeAddress(master,ref->addr->ip,ref->addr->port);
+    sentinelResetMasterAndChangeAddress(master,ref->addr->hostname,ref->addr->port);
 }
 
 void sentinelFailoverStateMachine(sentinelRedisInstance *ri) {
@@ -4887,4 +5090,3 @@ void sentinelTimer(void) {
      * election because of split brain voting). */
     server.hz = CONFIG_DEFAULT_HZ + rand() % CONFIG_DEFAULT_HZ;
 }
-
diff --git a/tests/instances.tcl b/tests/instances.tcl
index 03af2408b..b9d1b3fe5 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -28,6 +28,7 @@ set ::global_config {}
 set ::sentinel_base_port 20000
 set ::redis_base_port 30000
 set ::redis_port_count 1024
+set ::host "127.0.0.1"
 set ::pids {} ; # We kill everything at exit
 set ::dirs {} ; # We remove all the temp dirs at exit
 set ::run_matching {} ; # If non empty, only tests matching pattern are run.
@@ -128,18 +129,18 @@ proc spawn_instance {type base_port count {conf {}} {base_conf_file ""}} {
         }
 
         # Check availability finally
-        if {[server_is_up 127.0.0.1 $port 100] == 0} {
+        if {[server_is_up $::host $port 100] == 0} {
             set logfile [file join $dirname log.txt]
             puts [exec tail $logfile]
             abort_sentinel_test "Problems starting $type #$j: ping timeout, maybe server start failed, check $logfile"
         }
 
         # Push the instance into the right list
-        set link [redis 127.0.0.1 $port 0 $::tls]
+        set link [redis $::host $port 0 $::tls]
         $link reconnect 1
         lappend ::${type}_instances [list \
             pid $pid \
-            host 127.0.0.1 \
+            host $::host \
             port $port \
             link $link \
         ]
@@ -241,6 +242,9 @@ proc parse_options {} {
             set ::simulate_error 1
         } elseif {$opt eq {--valgrind}} {
             set ::valgrind 1
+        } elseif {$opt eq {--host}} {
+            incr j
+            set ::host ${val}
         } elseif {$opt eq {--tls}} {
             package require tls 1.6
             ::tls::init \
@@ -259,6 +263,7 @@ proc parse_options {} {
             puts "--fail                  Simulate a test failure."
             puts "--valgrind              Run with valgrind."
             puts "--tls                   Run tests in TLS mode."
+            puts "--host <host>           Use hostname instead of 127.0.0.1."
             puts "--config <k> <v>        Extra config argument(s)."
             puts "--help                  Shows this help."
             exit 0
diff --git a/tests/sentinel/tests/08-hostname-conf.tcl b/tests/sentinel/tests/08-hostname-conf.tcl
new file mode 100644
index 000000000..45bb56687
--- /dev/null
+++ b/tests/sentinel/tests/08-hostname-conf.tcl
@@ -0,0 +1,62 @@
+proc set_redis_announce_ip {addr} {
+    foreach_redis_id id {
+        R $id config set replica-announce-ip $addr
+    }
+}
+
+proc set_sentinel_config {keyword value} {
+    foreach_sentinel_id id {
+        S $id sentinel config set $keyword $value
+    }
+}
+
+proc set_all_instances_hostname {hostname} {
+    foreach_sentinel_id id {
+        set_instance_attrib sentinel $id host $hostname
+    }
+    foreach_redis_id id {
+        set_instance_attrib redis $id host $hostname
+    }
+}
+
+test "(pre-init) Configure instances and sentinel for hostname use" {
+    set ::host "localhost"
+    restart_killed_instances
+    set_all_instances_hostname $::host
+    set_redis_announce_ip $::host
+    set_sentinel_config resolve-hostnames yes
+    set_sentinel_config announce-hostnames yes
+}
+
+source "../tests/includes/init-tests.tcl"
+
+proc verify_hostname_announced {hostname} {
+    foreach_sentinel_id id {
+        # Master is reported with its hostname
+        assert_equal [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 0] $hostname
+
+        # Replicas are reported with their hostnames
+        foreach replica [S $id SENTINEL REPLICAS mymaster] {
+            assert_equal [dict get $replica ip] $hostname
+        }
+    }
+}
+
+test "Sentinel announces hostnames" {
+    # Check initial state
+    verify_hostname_announced $::host
+
+    # Disable announce-hostnames and confirm IPs are used
+    set_sentinel_config announce-hostnames no
+    verify_hostname_announced "127.0.0.1"
+}
+
+# We need to revert any special configuration because all tests currently
+# share the same instances.
+test "(post-cleanup) Configure instances and sentinel for IPs" {
+    set ::host "127.0.0.1"
+    set_all_instances_hostname $::host
+    set_redis_announce_ip $::host
+    set_sentinel_config resolve-hostnames no
+    set_sentinel_config announce-hostnames no
+}
\ No newline at end of file
diff --git a/tests/sentinel/tests/09-acl-support.tcl b/tests/sentinel/tests/09-acl-support.tcl
new file mode 100644
index 000000000..1366fc4d5
--- /dev/null
+++ b/tests/sentinel/tests/09-acl-support.tcl
@@ -0,0 +1,50 @@
+
+source "../tests/includes/init-tests.tcl"
+
+set ::user "testuser"
+set ::password "secret"
+
+proc setup_acl {} {
+    foreach_sentinel_id id {
+        assert_equal {OK} [S $id ACL SETUSER $::user >$::password +@all on]
+        assert_equal {OK} [S $id ACL SETUSER default off]
+
+        S $id CLIENT KILL USER default SKIPME no
+        assert_equal {OK} [S $id AUTH $::user $::password]
+    }
+}
+
+proc teardown_acl {} {
+    foreach_sentinel_id id {
+        assert_equal {OK} [S $id ACL SETUSER default on]
+        assert_equal {1} [S $id ACL DELUSER $::user]
+
+        S $id SENTINEL CONFIG SET sentinel-user ""
+        S $id SENTINEL CONFIG SET sentinel-pass ""
+    }
+}
+
+test "(post-init) Set up ACL configuration" {
+    setup_acl
+    assert_equal $::user [S 1 ACL WHOAMI]
+}
+
+test "SENTINEL CONFIG SET handles on-the-fly credentials reconfiguration" {
+    # Make sure we're starting with a broken state...
+    after 5000
+    catch {S 1 SENTINEL CKQUORUM mymaster} err
+    assert_match {*NOQUORUM*} $err
+
+    foreach_sentinel_id id {
+        assert_equal {OK} [S $id SENTINEL CONFIG SET sentinel-user $::user]
+        assert_equal {OK} [S $id SENTINEL CONFIG SET sentinel-pass $::password]
+    }
+
+    after 5000
+    assert_match {*OK*} [S 1 SENTINEL CKQUORUM mymaster]
+}
+
+test "(post-cleanup) Tear down ACL configuration" {
+    teardown_acl
+}
+
diff --git a/tests/sentinel/tests/includes/init-tests.tcl b/tests/sentinel/tests/includes/init-tests.tcl
index d6796fda6..b4626caed 100644
--- a/tests/sentinel/tests/includes/init-tests.tcl
+++ b/tests/sentinel/tests/includes/init-tests.tcl
@@ -1,6 +1,6 @@
 # Initialization tests -- most units will start including this.
 
-test "(init) Restart killed instances" {
+proc restart_killed_instances {} {
     foreach type {redis sentinel} {
         foreach_${type}_id id {
             if {[get_instance_attrib $type $id pid] == -1} {
@@ -12,6 +12,10 @@ test "(init) Restart killed instances" {
     }
 }
 
+test "(init) Restart killed instances" {
+    restart_killed_instances
+}
+
 test "(init) Remove old master entry from sentinels" {
     foreach_sentinel_id id {
         catch {S $id SENTINEL REMOVE mymaster}

From 4355145a6268a6ae811a2bf39911966879512877 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= <viktor.soderqvist@est.tech>
Date: Thu, 28 Jan 2021 15:19:43 +0100
Subject: [PATCH 1009/1098] Add modules API for streams (#8288)

APIs added for these stream operations: add, delete, iterate and
trim (by ID or maxlength). The functions are prefixed by RM_Stream.

* RM_StreamAdd
* RM_StreamDelete
* RM_StreamIteratorStart
* RM_StreamIteratorStop
* RM_StreamIteratorNextID
* RM_StreamIteratorNextField
* RM_StreamIteratorDelete
* RM_StreamTrimByLength
* RM_StreamTrimByID

The type RedisModuleStreamID is added and functions for converting
from and to RedisModuleString.

* RM_CreateStringFromStreamID
* RM_StringToStreamID

Whenever the stream functions return REDISMODULE_ERR, errno is set to
provide additional error information.

Refactoring: The zset iterator fields in the RedisModuleKey struct
are wrapped in a union, to allow the same space to be used for type-
specific info for streams and allow future use for other key types.
---
 runtest-moduleapi               |   1 +
 src/module.c                    | 676 ++++++++++++++++++++++++++++----
 src/redismodule.h               |  36 ++
 src/stream.h                    |   7 +
 src/t_stream.c                  |  29 +-
 tests/modules/Makefile          |   3 +-
 tests/modules/stream.c          | 258 ++++++++++++
 tests/unit/moduleapi/stream.tcl | 155 ++++++++
 8 files changed, 1089 insertions(+), 76 deletions(-)
 create mode 100644 tests/modules/stream.c
 create mode 100644 tests/unit/moduleapi/stream.tcl

diff --git a/runtest-moduleapi b/runtest-moduleapi
index 9a48867d2..878b5c6ad 100755
--- a/runtest-moduleapi
+++ b/runtest-moduleapi
@@ -31,4 +31,5 @@ $TCLSH tests/test_helper.tcl \
 --single unit/moduleapi/getkeys \
 --single unit/moduleapi/test_lazyfree \
 --single unit/moduleapi/defrag \
+--single unit/moduleapi/stream \
 "${@}"
diff --git a/src/module.c b/src/module.c
index 56a3ef98a..cabccfb7a 100644
--- a/src/module.c
+++ b/src/module.c
@@ -177,15 +177,25 @@ struct RedisModuleKey {
     void *iter;     /* Iterator. */
     int mode;       /* Opening mode. */
 
-    /* Zset iterator. */
-    uint32_t ztype;         /* REDISMODULE_ZSET_RANGE_* */
-    zrangespec zrs;         /* Score range. */
-    zlexrangespec zlrs;     /* Lex range. */
-    uint32_t zstart;        /* Start pos for positional ranges. */
-    uint32_t zend;          /* End pos for positional ranges. */
-    void *zcurrent;         /* Zset iterator current node. */
-    int zer;                /* Zset iterator end reached flag
-                               (true if end was reached). */
+    union {
+        struct {
+            /* Zset iterator, use only if value->type == OBJ_ZSET */
+            uint32_t type;         /* REDISMODULE_ZSET_RANGE_* */
+            zrangespec rs;         /* Score range. */
+            zlexrangespec lrs;     /* Lex range. */
+            uint32_t start;        /* Start pos for positional ranges. */
+            uint32_t end;          /* End pos for positional ranges. */
+            void *current;         /* Zset iterator current node. */
+            int er;                /* Zset iterator end reached flag
+                                       (true if end was reached). */
+        } zset;
+        struct {
+            /* Stream, use only if value->type == OBJ_STREAM */
+            streamID currentid;    /* Current entry while iterating. */
+            int64_t numfieldsleft; /* Fields left to fetch for current entry. */
+            int signalready;       /* Flag that signalKeyAsReady() is needed. */
+        } stream;
+    } u;
 };
 typedef struct RedisModuleKey RedisModuleKey;
 
@@ -376,6 +386,7 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int
 void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx);
 void RM_ZsetRangeStop(RedisModuleKey *kp);
 static void zsetKeyReset(RedisModuleKey *key);
+static void moduleInitKeyTypeSpecific(RedisModuleKey *key);
 void RM_FreeDict(RedisModuleCtx *ctx, RedisModuleDict *d);
 void RM_FreeServerInfo(RedisModuleCtx *ctx, RedisModuleServerInfoData *data);
 
@@ -509,10 +520,14 @@ int moduleCreateEmptyKey(RedisModuleKey *key, int type) {
     case REDISMODULE_KEYTYPE_HASH:
         obj = createHashObject();
         break;
+    case REDISMODULE_KEYTYPE_STREAM:
+        obj = createStreamObject();
+        break;
     default: return REDISMODULE_ERR;
     }
     dbAdd(key->db,key->key,obj);
     key->value = obj;
+    moduleInitKeyTypeSpecific(key);
     return REDISMODULE_OK;
 }
 
@@ -1113,6 +1128,18 @@ RedisModuleString *RM_CreateStringFromString(RedisModuleCtx *ctx, const RedisMod
     return o;
 }
 
+/* Creates a string from a stream ID. The returned string must be released with
+ * RedisModule_FreeString(), unless automatic memory is enabled.
+ *
+ * The passed context `ctx` may be NULL if necessary. See the
+ * RedisModule_CreateString() documentation for more info. */
+RedisModuleString *RM_CreateStringFromStreamID(RedisModuleCtx *ctx, const RedisModuleStreamID *id) {
+    streamID streamid = {id->ms, id->seq};
+    RedisModuleString *o = createObjectFromStreamID(&streamid);
+    if (ctx != NULL) autoMemoryAdd(ctx, REDISMODULE_AM_STRING, o);
+    return o;
+}
+
 /* Free a module string object obtained with one of the Redis modules API calls
  * that return new string objects.
  *
@@ -1270,6 +1297,30 @@ int RM_StringToLongDouble(const RedisModuleString *str, long double *ld) {
     return retval ? REDISMODULE_OK : REDISMODULE_ERR;
 }
 
+/* Convert the string into a stream ID, storing it at `*id`.
+ * Returns REDISMODULE_OK on success and returns REDISMODULE_ERR if the string
+ * is not a valid string representation of a stream ID. The special IDs "+" and
+ * "-" are allowed.
+ *
+ * RedisModuleStreamID is a struct with two 64-bit fields, which is used in
+ * stream functions and defined as
+ *
+ *     typedef struct RedisModuleStreamID {
+ *         uint64_t ms;
+ *         uint64_t seq;
+ *     } RedisModuleStreamID;
+ */
+int RM_StringToStreamID(const RedisModuleString *str, RedisModuleStreamID *id) {
+    streamID streamid;
+    if (streamParseID(str, &streamid) == C_OK) {
+        id->ms = streamid.ms;
+        id->seq = streamid.seq;
+        return REDISMODULE_OK;
+    } else {
+        return REDISMODULE_ERR;
+    }
+}
+
 /* Compare two string objects, returning -1, 0 or 1 respectively if
  * a < b, a == b, a > b. Strings are compared byte by byte as two
  * binary blobs without any encoding care / collation attempt. */
@@ -2072,7 +2123,15 @@ static void moduleInitKey(RedisModuleKey *kp, RedisModuleCtx *ctx, robj *keyname
     kp->value = value;
     kp->iter = NULL;
     kp->mode = mode;
-    zsetKeyReset(kp);
+    if (kp->value) moduleInitKeyTypeSpecific(kp);
+}
+
+/* Initialize the type-specific part of the key. Only when key has a value. */
+static void moduleInitKeyTypeSpecific(RedisModuleKey *key) {
+    switch (key->value->type) {
+    case OBJ_ZSET: zsetKeyReset(key); break;
+    case OBJ_STREAM: key->u.stream.signalready = 0; break;
+    }
 }
 
 /* Return an handle representing a Redis key, so that it is possible
@@ -2115,8 +2174,13 @@ static void moduleCloseKey(RedisModuleKey *key) {
     int signal = SHOULD_SIGNAL_MODIFIED_KEYS(key->ctx);
     if ((key->mode & REDISMODULE_WRITE) && signal)
         signalModifiedKey(key->ctx->client,key->db,key->key);
-    /* TODO: if (key->iter) RM_KeyIteratorStop(kp); */
+    if (key->iter) zfree(key->iter);
     RM_ZsetRangeStop(key);
+    if (key && key->value && key->value->type == OBJ_STREAM &&
+        key->u.stream.signalready) {
+        /* One of more RM_StreamAdd() have been done. */
+        signalKeyAsReady(key->db, key->key, OBJ_STREAM);
+    }
     decrRefCount(key->key);
 }
 
@@ -2545,16 +2609,17 @@ int RM_ZsetScore(RedisModuleKey *key, RedisModuleString *ele, double *score) {
  * -------------------------------------------------------------------------- */
 
 void zsetKeyReset(RedisModuleKey *key) {
-    key->ztype = REDISMODULE_ZSET_RANGE_NONE;
-    key->zcurrent = NULL;
-    key->zer = 1;
+    key->u.zset.type = REDISMODULE_ZSET_RANGE_NONE;
+    key->u.zset.current = NULL;
+    key->u.zset.er = 1;
 }
 
 /* Stop a sorted set iteration. */
 void RM_ZsetRangeStop(RedisModuleKey *key) {
+    if (!key->value || key->value->type != OBJ_ZSET) return;
     /* Free resources if needed. */
-    if (key->ztype == REDISMODULE_ZSET_RANGE_LEX)
-        zslFreeLexRange(&key->zlrs);
+    if (key->u.zset.type == REDISMODULE_ZSET_RANGE_LEX)
+        zslFreeLexRange(&key->u.zset.lrs);
     /* Setup sensible values so that misused iteration API calls when an
      * iterator is not active will result into something more sensible
      * than crashing. */
@@ -2563,7 +2628,7 @@ void RM_ZsetRangeStop(RedisModuleKey *key) {
 
 /* Return the "End of range" flag value to signal the end of the iteration. */
 int RM_ZsetRangeEndReached(RedisModuleKey *key) {
-    return key->zer;
+    return key->u.zset.er;
 }
 
 /* Helper function for RM_ZsetFirstInScoreRange() and RM_ZsetLastInScoreRange().
@@ -2576,29 +2641,29 @@ int zsetInitScoreRange(RedisModuleKey *key, double min, double max, int minex, i
     if (!key->value || key->value->type != OBJ_ZSET) return REDISMODULE_ERR;
 
     RM_ZsetRangeStop(key);
-    key->ztype = REDISMODULE_ZSET_RANGE_SCORE;
-    key->zer = 0;
+    key->u.zset.type = REDISMODULE_ZSET_RANGE_SCORE;
+    key->u.zset.er = 0;
 
     /* Setup the range structure used by the sorted set core implementation
      * in order to seek at the specified element. */
-    zrangespec *zrs = &key->zrs;
+    zrangespec *zrs = &key->u.zset.rs;
     zrs->min = min;
     zrs->max = max;
     zrs->minex = minex;
     zrs->maxex = maxex;
 
     if (key->value->encoding == OBJ_ENCODING_ZIPLIST) {
-        key->zcurrent = first ? zzlFirstInRange(key->value->ptr,zrs) :
-                                zzlLastInRange(key->value->ptr,zrs);
+        key->u.zset.current = first ? zzlFirstInRange(key->value->ptr,zrs) :
+                                      zzlLastInRange(key->value->ptr,zrs);
     } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) {
         zset *zs = key->value->ptr;
         zskiplist *zsl = zs->zsl;
-        key->zcurrent = first ? zslFirstInRange(zsl,zrs) :
-                                zslLastInRange(zsl,zrs);
+        key->u.zset.current = first ? zslFirstInRange(zsl,zrs) :
+                                      zslLastInRange(zsl,zrs);
     } else {
         serverPanic("Unsupported zset encoding");
     }
-    if (key->zcurrent == NULL) key->zer = 1;
+    if (key->u.zset.current == NULL) key->u.zset.er = 1;
     return REDISMODULE_OK;
 }
 
@@ -2640,29 +2705,29 @@ int zsetInitLexRange(RedisModuleKey *key, RedisModuleString *min, RedisModuleStr
     if (!key->value || key->value->type != OBJ_ZSET) return REDISMODULE_ERR;
 
     RM_ZsetRangeStop(key);
-    key->zer = 0;
+    key->u.zset.er = 0;
 
     /* Setup the range structure used by the sorted set core implementation
      * in order to seek at the specified element. */
-    zlexrangespec *zlrs = &key->zlrs;
+    zlexrangespec *zlrs = &key->u.zset.lrs;
     if (zslParseLexRange(min, max, zlrs) == C_ERR) return REDISMODULE_ERR;
 
     /* Set the range type to lex only after successfully parsing the range,
      * otherwise we don't want the zlexrangespec to be freed. */
-    key->ztype = REDISMODULE_ZSET_RANGE_LEX;
+    key->u.zset.type = REDISMODULE_ZSET_RANGE_LEX;
 
     if (key->value->encoding == OBJ_ENCODING_ZIPLIST) {
-        key->zcurrent = first ? zzlFirstInLexRange(key->value->ptr,zlrs) :
-                                zzlLastInLexRange(key->value->ptr,zlrs);
+        key->u.zset.current = first ? zzlFirstInLexRange(key->value->ptr,zlrs) :
+                                      zzlLastInLexRange(key->value->ptr,zlrs);
     } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) {
         zset *zs = key->value->ptr;
         zskiplist *zsl = zs->zsl;
-        key->zcurrent = first ? zslFirstInLexRange(zsl,zlrs) :
-                                zslLastInLexRange(zsl,zlrs);
+        key->u.zset.current = first ? zslFirstInLexRange(zsl,zlrs) :
+                                      zslLastInLexRange(zsl,zlrs);
     } else {
         serverPanic("Unsupported zset encoding");
     }
-    if (key->zcurrent == NULL) key->zer = 1;
+    if (key->u.zset.current == NULL) key->u.zset.er = 1;
 
     return REDISMODULE_OK;
 }
@@ -2695,10 +2760,11 @@ int RM_ZsetLastInLexRange(RedisModuleKey *key, RedisModuleString *min, RedisModu
 RedisModuleString *RM_ZsetRangeCurrentElement(RedisModuleKey *key, double *score) {
     RedisModuleString *str;
 
-    if (key->zcurrent == NULL) return NULL;
+    if (!key->value || key->value->type != OBJ_ZSET) return NULL;
+    if (key->u.zset.current == NULL) return NULL;
     if (key->value->encoding == OBJ_ENCODING_ZIPLIST) {
         unsigned char *eptr, *sptr;
-        eptr = key->zcurrent;
+        eptr = key->u.zset.current;
         sds ele = ziplistGetObject(eptr);
         if (score) {
             sptr = ziplistNext(key->value->ptr,eptr);
@@ -2706,7 +2772,7 @@ RedisModuleString *RM_ZsetRangeCurrentElement(RedisModuleKey *key, double *score
         }
         str = createObject(OBJ_STRING,ele);
     } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) {
-        zskiplistNode *ln = key->zcurrent;
+        zskiplistNode *ln = key->u.zset.current;
         if (score) *score = ln->score;
         str = createStringObject(ln->ele,sdslen(ln->ele));
     } else {
@@ -2720,58 +2786,59 @@ RedisModuleString *RM_ZsetRangeCurrentElement(RedisModuleKey *key, double *score
  * a next element, 0 if we are already at the latest element or the range
  * does not include any item at all. */
 int RM_ZsetRangeNext(RedisModuleKey *key) {
-    if (!key->ztype || !key->zcurrent) return 0; /* No active iterator. */
+    if (!key->value || key->value->type != OBJ_ZSET) return 0;
+    if (!key->u.zset.type || !key->u.zset.current) return 0; /* No active iterator. */
 
     if (key->value->encoding == OBJ_ENCODING_ZIPLIST) {
         unsigned char *zl = key->value->ptr;
-        unsigned char *eptr = key->zcurrent;
+        unsigned char *eptr = key->u.zset.current;
         unsigned char *next;
         next = ziplistNext(zl,eptr); /* Skip element. */
         if (next) next = ziplistNext(zl,next); /* Skip score. */
         if (next == NULL) {
-            key->zer = 1;
+            key->u.zset.er = 1;
             return 0;
         } else {
             /* Are we still within the range? */
-            if (key->ztype == REDISMODULE_ZSET_RANGE_SCORE) {
+            if (key->u.zset.type == REDISMODULE_ZSET_RANGE_SCORE) {
                 /* Fetch the next element score for the
                  * range check. */
                 unsigned char *saved_next = next;
                 next = ziplistNext(zl,next); /* Skip next element. */
                 double score = zzlGetScore(next); /* Obtain the next score. */
-                if (!zslValueLteMax(score,&key->zrs)) {
-                    key->zer = 1;
+                if (!zslValueLteMax(score,&key->u.zset.rs)) {
+                    key->u.zset.er = 1;
                     return 0;
                 }
                 next = saved_next;
-            } else if (key->ztype == REDISMODULE_ZSET_RANGE_LEX) {
-                if (!zzlLexValueLteMax(next,&key->zlrs)) {
-                    key->zer = 1;
+            } else if (key->u.zset.type == REDISMODULE_ZSET_RANGE_LEX) {
+                if (!zzlLexValueLteMax(next,&key->u.zset.lrs)) {
+                    key->u.zset.er = 1;
                     return 0;
                 }
             }
-            key->zcurrent = next;
+            key->u.zset.current = next;
             return 1;
         }
     } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) {
-        zskiplistNode *ln = key->zcurrent, *next = ln->level[0].forward;
+        zskiplistNode *ln = key->u.zset.current, *next = ln->level[0].forward;
         if (next == NULL) {
-            key->zer = 1;
+            key->u.zset.er = 1;
             return 0;
         } else {
             /* Are we still within the range? */
-            if (key->ztype == REDISMODULE_ZSET_RANGE_SCORE &&
-                !zslValueLteMax(next->score,&key->zrs))
+            if (key->u.zset.type == REDISMODULE_ZSET_RANGE_SCORE &&
+                !zslValueLteMax(next->score,&key->u.zset.rs))
             {
-                key->zer = 1;
+                key->u.zset.er = 1;
                 return 0;
-            } else if (key->ztype == REDISMODULE_ZSET_RANGE_LEX) {
-                if (!zslLexValueLteMax(next->ele,&key->zlrs)) {
-                    key->zer = 1;
+            } else if (key->u.zset.type == REDISMODULE_ZSET_RANGE_LEX) {
+                if (!zslLexValueLteMax(next->ele,&key->u.zset.lrs)) {
+                    key->u.zset.er = 1;
                     return 0;
                 }
             }
-            key->zcurrent = next;
+            key->u.zset.current = next;
             return 1;
         }
     } else {
@@ -2783,58 +2850,59 @@ int RM_ZsetRangeNext(RedisModuleKey *key) {
  * a previous element, 0 if we are already at the first element or the range
  * does not include any item at all. */
 int RM_ZsetRangePrev(RedisModuleKey *key) {
-    if (!key->ztype || !key->zcurrent) return 0; /* No active iterator. */
+    if (!key->value || key->value->type != OBJ_ZSET) return 0;
+    if (!key->u.zset.type || !key->u.zset.current) return 0; /* No active iterator. */
 
     if (key->value->encoding == OBJ_ENCODING_ZIPLIST) {
         unsigned char *zl = key->value->ptr;
-        unsigned char *eptr = key->zcurrent;
+        unsigned char *eptr = key->u.zset.current;
         unsigned char *prev;
         prev = ziplistPrev(zl,eptr); /* Go back to previous score. */
         if (prev) prev = ziplistPrev(zl,prev); /* Back to previous ele. */
         if (prev == NULL) {
-            key->zer = 1;
+            key->u.zset.er = 1;
             return 0;
         } else {
             /* Are we still within the range? */
-            if (key->ztype == REDISMODULE_ZSET_RANGE_SCORE) {
+            if (key->u.zset.type == REDISMODULE_ZSET_RANGE_SCORE) {
                 /* Fetch the previous element score for the
                  * range check. */
                 unsigned char *saved_prev = prev;
                 prev = ziplistNext(zl,prev); /* Skip element to get the score.*/
                 double score = zzlGetScore(prev); /* Obtain the prev score. */
-                if (!zslValueGteMin(score,&key->zrs)) {
-                    key->zer = 1;
+                if (!zslValueGteMin(score,&key->u.zset.rs)) {
+                    key->u.zset.er = 1;
                     return 0;
                 }
                 prev = saved_prev;
-            } else if (key->ztype == REDISMODULE_ZSET_RANGE_LEX) {
-                if (!zzlLexValueGteMin(prev,&key->zlrs)) {
-                    key->zer = 1;
+            } else if (key->u.zset.type == REDISMODULE_ZSET_RANGE_LEX) {
+                if (!zzlLexValueGteMin(prev,&key->u.zset.lrs)) {
+                    key->u.zset.er = 1;
                     return 0;
                 }
             }
-            key->zcurrent = prev;
+            key->u.zset.current = prev;
             return 1;
         }
     } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) {
-        zskiplistNode *ln = key->zcurrent, *prev = ln->backward;
+        zskiplistNode *ln = key->u.zset.current, *prev = ln->backward;
         if (prev == NULL) {
-            key->zer = 1;
+            key->u.zset.er = 1;
             return 0;
         } else {
             /* Are we still within the range? */
-            if (key->ztype == REDISMODULE_ZSET_RANGE_SCORE &&
-                !zslValueGteMin(prev->score,&key->zrs))
+            if (key->u.zset.type == REDISMODULE_ZSET_RANGE_SCORE &&
+                !zslValueGteMin(prev->score,&key->u.zset.rs))
             {
-                key->zer = 1;
+                key->u.zset.er = 1;
                 return 0;
-            } else if (key->ztype == REDISMODULE_ZSET_RANGE_LEX) {
-                if (!zslLexValueGteMin(prev->ele,&key->zlrs)) {
-                    key->zer = 1;
+            } else if (key->u.zset.type == REDISMODULE_ZSET_RANGE_LEX) {
+                if (!zslLexValueGteMin(prev->ele,&key->u.zset.lrs)) {
+                    key->u.zset.er = 1;
                     return 0;
                 }
             }
-            key->zcurrent = prev;
+            key->u.zset.current = prev;
             return 1;
         }
     } else {
@@ -3049,6 +3117,455 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) {
     return REDISMODULE_OK;
 }
 
+/* --------------------------------------------------------------------------
+ * Key API for the stream type.
+ * -------------------------------------------------------------------------- */
+
+/* Adds an entry to a stream. Like XADD without trimming.
+ *
+ * - `key`: The key where the stream is (or will be) stored
+ * - `flags`: A bit field of
+ *   - `REDISMODULE_STREAM_ADD_AUTOID`: Assign a stream ID automatically, like
+ *     `*` in the XADD command.
+ * - `id`: If the `AUTOID` flag is set, this is where the assigned ID is
+ *   returned. Can be NULL if `AUTOID` is set, if you don't care to receive the
+ *   ID. If `AUTOID` is not set, this is the requested ID.
+ * - `argv`: A pointer to an array of size `numfields * 2` containing the
+ *   fields and values.
+ * - `numfields`: The number of field-value pairs in `argv`.
+ *
+ * Returns REDISMODULE_OK if an entry has been added. On failure,
+ * REDISMODULE_ERR is returned and `errno` is set as follows:
+ *
+ * - EINVAL if called with invalid arguments
+ * - ENOTSUP if the key refers to a value of a type other than stream
+ * - EBADF if the key was not opened for writing
+ * - EDOM if the given ID was 0-0 or not greater than all other IDs in the
+ *   stream (only if the AUTOID flag is unset)
+ * - EFBIG if the stream has reached the last possible ID
+ */
+int RM_StreamAdd(RedisModuleKey *key, int flags, RedisModuleStreamID *id, RedisModuleString **argv, long numfields) {
+    /* Validate args */
+    if (!key || (numfields != 0 && !argv) || /* invalid key or argv */
+        (flags & ~(REDISMODULE_STREAM_ADD_AUTOID)) || /* invalid flags */
+        (!(flags & REDISMODULE_STREAM_ADD_AUTOID) && !id)) { /* id required */
+        errno = EINVAL;
+        return REDISMODULE_ERR;
+    } else if (key->value && key->value->type != OBJ_STREAM) {
+        errno = ENOTSUP; /* wrong type */
+        return REDISMODULE_ERR;
+    } else if (!(key->mode & REDISMODULE_WRITE)) {
+        errno = EBADF; /* key not open for writing */
+        return REDISMODULE_ERR;
+    } else if (!(flags & REDISMODULE_STREAM_ADD_AUTOID) &&
+               id->ms == 0 && id->seq == 0) {
+        errno = EDOM; /* ID out of range */
+        return REDISMODULE_ERR;
+    }
+
+    /* Create key if necessery */
+    int created = 0;
+    if (key->value == NULL) {
+        moduleCreateEmptyKey(key, REDISMODULE_KEYTYPE_STREAM);
+        created = 1;
+    }
+
+    stream *s = key->value->ptr;
+    if (s->last_id.ms == UINT64_MAX && s->last_id.seq == UINT64_MAX) {
+        /* The stream has reached the last possible ID */
+        errno = EFBIG;
+        return REDISMODULE_ERR;
+    }
+
+    streamID added_id;
+    streamID use_id;
+    streamID *use_id_ptr = NULL;
+    if (!(flags & REDISMODULE_STREAM_ADD_AUTOID)) {
+        use_id.ms = id->ms;
+        use_id.seq = id->seq;
+        use_id_ptr = &use_id;
+    }
+    if (streamAppendItem(s, argv, numfields, &added_id, use_id_ptr) == C_ERR) {
+        /* ID not greater than all existing IDs in the stream */
+        errno = EDOM;
+        return REDISMODULE_ERR;
+    }
+    /* Postponed signalKeyAsReady(). Done implicitly by moduleCreateEmptyKey()
+     * so not needed if the stream has just been created. */
+    if (!created) key->u.stream.signalready = 1;
+
+    if (id != NULL) {
+        id->ms = added_id.ms;
+        id->seq = added_id.seq;
+    }
+
+    return REDISMODULE_OK;
+}
+
+/* Deletes an entry from a stream.
+ *
+ * - `key`: A key opened for writing, with no stream iterator started.
+ * - `id`: The stream ID of the entry to delete.
+ *
+ * Returns REDISMODULE_OK on success. On failure, REDISMODULE_ERR is returned
+ * and `errno` is set as follows:
+ *
+ * - EINVAL if called with invalid arguments
+ * - ENOTSUP if the key refers to a value of a type other than stream or if the
+ *   key is empty
+ * - EBADF if the key was not opened for writing or if a stream iterator is
+ *   associated with the key
+ * - ENOENT if no entry with the given stream ID exists
+ *
+ * See also RM_StreamIteratorDelete() for deleting the current entry while
+ * iterating using a stream iterator.
+ */
+int RM_StreamDelete(RedisModuleKey *key, RedisModuleStreamID *id) {
+    if (!key || !id) {
+        errno = EINVAL;
+        return REDISMODULE_ERR;
+    } else if (!key->value || key->value->type != OBJ_STREAM) {
+        errno = ENOTSUP; /* wrong type */
+        return REDISMODULE_ERR;
+    } else if (!(key->mode & REDISMODULE_WRITE) ||
+               key->iter != NULL) {
+        errno = EBADF; /* key not opened for writing or iterator started */
+        return REDISMODULE_ERR;
+    }
+    stream *s = key->value->ptr;
+    streamID streamid = {id->ms, id->seq};
+    if (streamDeleteItem(s, &streamid)) {
+        return REDISMODULE_OK;
+    } else {
+        errno = ENOENT; /* no entry with this id */
+        return REDISMODULE_ERR;
+    }
+}
+
+/* Sets up a stream iterator.
+ *
+ * - `key`: The stream key opened for reading using RedisModule_OpenKey().
+ * - `flags`:
+ *   - `REDISMODULE_STREAM_ITERATOR_EXCLUSIVE`: Don't include `start` and `end`
+ *     in the iterated range.
+ *   - `REDISMODULE_STREAM_ITERATOR_REVERSE`: Iterate in reverse order, starting
+ *     from the `end` of the range.
+ * - `start`: The lower bound of the range. Use NULL for the beginning of the
+ *   stream.
+ * - `end`: The upper bound of the range. Use NULL for the end of the stream.
+ *
+ * Returns REDISMODULE_OK on success. On failure, REDISMODULE_ERR is returned
+ * and `errno` is set as follows:
+ *
+ * - EINVAL if called with invalid arguments
+ * - ENOTSUP if the key refers to a value of a type other than stream or if the
+ *   key is empty
+ * - EBADF if the key was not opened for writing or if a stream iterator is
+ *   already associated with the key
+ * - EDOM if `start` or `end` is outside the valid range
+ *
+ * Returns REDISMODULE_OK on success and REDISMODULE_ERR if the key doesn't
+ * refer to a stream or if invalid arguments were given.
+ *
+ * The stream IDs are retrieved using RedisModule_StreamIteratorNextID() and
+ * for each stream ID, the fields and values are retrieved using
+ * RedisModule_StreamIteratorNextField(). The iterator is freed by calling
+ * RedisModule_StreamIteratorStop().
+ *
+ * Example (error handling omitted):
+ *
+ *     RedisModule_StreamIteratorStart(key, 0, startid_ptr, endid_ptr);
+ *     RedisModuleStreamID id;
+ *     long numfields;
+ *     while (RedisModule_StreamIteratorNextID(key, &id, &numfields) ==
+ *            REDISMODULE_OK) {
+ *         RedisModuleString *field, *value;
+ *         while (RedisModule_StreamIteratorNextField(key, &field, &value) ==
+ *                REDISMODULE_OK) {
+ *             //
+ *             // ... Do stuff ...
+ *             //
+ *             RedisModule_Free(field);
+ *             RedisModule_Free(value);
+ *         }
+ *     }
+ *     RedisModule_StreamIteratorStop(key);
+ */
+int RM_StreamIteratorStart(RedisModuleKey *key, int flags, RedisModuleStreamID *start, RedisModuleStreamID *end) {
+    /* check args */
+    if (!key ||
+        (flags & ~(REDISMODULE_STREAM_ITERATOR_EXCLUSIVE |
+                   REDISMODULE_STREAM_ITERATOR_REVERSE))) {
+        errno = EINVAL; /* key missing or invalid flags */
+        return REDISMODULE_ERR;
+    } else if (!key->value || key->value->type != OBJ_STREAM) {
+        errno = ENOTSUP;
+        return REDISMODULE_ERR; /* not a stream */
+    } else if (key->iter) {
+        errno = EBADF; /* iterator already started */
+        return REDISMODULE_ERR;
+    }
+
+    /* define range for streamIteratorStart() */
+    streamID lower, upper;
+    if (start) lower = (streamID){start->ms, start->seq};
+    if (end)   upper = (streamID){end->ms,   end->seq};
+    if (flags & REDISMODULE_STREAM_ITERATOR_EXCLUSIVE) {
+        if ((start && streamIncrID(&lower) != C_OK) ||
+            (end   && streamDecrID(&upper) != C_OK)) {
+            errno = EDOM; /* end is 0-0 or start is MAX-MAX? */
+            return REDISMODULE_ERR;
+        }
+    }
+
+    /* create iterator */
+    stream *s = key->value->ptr;
+    int rev = flags & REDISMODULE_STREAM_ITERATOR_REVERSE;
+    streamIterator *si = zmalloc(sizeof(*si));
+    streamIteratorStart(si, s, start ? &lower : NULL, end ? &upper : NULL, rev);
+    key->iter = si;
+    key->u.stream.currentid.ms = 0; /* for RM_StreamIteratorDelete() */
+    key->u.stream.currentid.seq = 0;
+    key->u.stream.numfieldsleft = 0; /* for RM_StreamIteratorNextField() */
+    return REDISMODULE_OK;
+}
+
+/* Stops a stream iterator created using RedisModule_StreamIteratorStart() and
+ * reclaims its memory.
+ *
+ * Returns REDISMODULE_OK on success. On failure, REDISMODULE_ERR is returned
+ * and `errno` is set as follows:
+ *
+ * - EINVAL if called with a NULL key
+ * - ENOTSUP if the key refers to a value of a type other than stream or if the
+ *   key is empty
+ * - EBADF if the key was not opened for writing or if no stream iterator is
+ *   associated with the key
+ */
+int RM_StreamIteratorStop(RedisModuleKey *key) {
+    if (!key) {
+        errno = EINVAL;
+        return REDISMODULE_ERR;
+    } else if (!key->value || key->value->type != OBJ_STREAM) {
+        errno = ENOTSUP;
+        return REDISMODULE_ERR;
+    } else if (!key->iter) {
+        errno = EBADF;
+        return REDISMODULE_ERR;
+    }
+    zfree(key->iter);
+    key->iter = NULL;
+    return REDISMODULE_OK;
+}
+
+/* Finds the next stream entry and returns its stream ID and the number of
+ * fields.
+ *
+ * - `key`: Key for which a stream iterator has been started using
+ *   RedisModule_StreamIteratorStart().
+ * - `id`: The stream ID returned. NULL if you don't care.
+ * - `numfields`: The number of fields in the found stream entry. NULL if you
+ *   don't care.
+ *
+ * Returns REDISMODULE_OK and sets `*id` and `*numfields` if an entry was found.
+ * On failure, REDISMODULE_ERR is returned and `errno` is set as follows:
+ *
+ * - EINVAL if called with a NULL key
+ * - ENOTSUP if the key refers to a value of a type other than stream or if the
+ *   key is empty
+ * - EBADF if no stream iterator is associated with the key
+ * - ENOENT if there are no more entries in the range of the iterator
+ *
+ * In practice, if RM_StreamIteratorNextID() is called after a successful call
+ * to RM_StreamIteratorStart() and with the same key, it is safe to assume that
+ * an REDISMODULE_ERR return value means that there are no more entries.
+ *
+ * Use RedisModule_StreamIteratorNextField() to retrieve the fields and values.
+ * See the example at RedisModule_StreamIteratorStart().
+ */
+int RM_StreamIteratorNextID(RedisModuleKey *key, RedisModuleStreamID *id, long *numfields) {
+    if (!key) {
+        errno = EINVAL;
+        return REDISMODULE_ERR;
+    } else if (!key->value || key->value->type != OBJ_STREAM) {
+        errno = ENOTSUP;
+        return REDISMODULE_ERR;
+    } else if (!key->iter) {
+        errno = EBADF;
+        return REDISMODULE_ERR;
+    }
+    streamIterator *si = key->iter;
+    int64_t *num_ptr = &key->u.stream.numfieldsleft;
+    streamID *streamid_ptr = &key->u.stream.currentid;
+    if (streamIteratorGetID(si, streamid_ptr, num_ptr)) {
+        if (id) {
+            id->ms = streamid_ptr->ms;
+            id->seq = streamid_ptr->seq;
+        }
+        if (numfields) *numfields = *num_ptr;
+        return REDISMODULE_OK;
+    } else {
+        /* No entry found. */
+        key->u.stream.currentid.ms = 0; /* for RM_StreamIteratorDelete() */
+        key->u.stream.currentid.seq = 0;
+        key->u.stream.numfieldsleft = 0; /* for RM_StreamIteratorNextField() */
+        errno = ENOENT;
+        return REDISMODULE_ERR;
+    }
+}
+
+/* Retrieves the next field of the current stream ID and its corresponding value
+ * in a stream iteration. This function should be called repeatedly after calling
+ * RedisModule_StreamIteratorNextID() to fetch each field-value pair.
+ *
+ * - `key`: Key where a stream iterator has been started.
+ * - `field_ptr`: This is where the field is returned.
+ * - `value_ptr`: This is where the value is returned.
+ *
+ * Returns REDISMODULE_OK and points `*field_ptr` and `*value_ptr` to freshly
+ * allocated RedisModuleString objects. The string objects are freed
+ * automatically when the callback finishes if automatic memory is enabled. On
+ * failure, REDISMODULE_ERR is returned and `errno` is set as follows:
+ *
+ * - EINVAL if called with a NULL key
+ * - ENOTSUP if the key refers to a value of a type other than stream or if the
+ *   key is empty
+ * - EBADF if no stream iterator is associated with the key
+ * - ENOENT if there are no more fields in the current stream entry
+ *
+ * In practice, if RM_StreamIteratorNextField() is called after a successful
+ * call to RM_StreamIteratorNextID() and with the same key, it is safe to assume
+ * that an REDISMODULE_ERR return value means that there are no more fields.
+ *
+ * See the example at RedisModule_StreamIteratorStart().
+ */
+int RM_StreamIteratorNextField(RedisModuleKey *key, RedisModuleString **field_ptr, RedisModuleString **value_ptr) {
+    if (!key) {
+        errno = EINVAL;
+        return REDISMODULE_ERR;
+    } else if (!key->value || key->value->type != OBJ_STREAM) {
+        errno = ENOTSUP;
+        return REDISMODULE_ERR;
+    } else if (!key->iter) {
+        errno = EBADF;
+        return REDISMODULE_ERR;
+    } else if (key->u.stream.numfieldsleft <= 0) {
+        errno = ENOENT;
+        return REDISMODULE_ERR;
+    }
+    streamIterator *si = key->iter;
+    unsigned char *field, *value;
+    int64_t field_len, value_len;
+    streamIteratorGetField(si, &field, &value, &field_len, &value_len);
+    if (field_ptr) {
+        *field_ptr = createRawStringObject((char *)field, field_len);
+        autoMemoryAdd(key->ctx, REDISMODULE_AM_STRING, *field_ptr);
+    }
+    if (value_ptr) {
+        *value_ptr = createRawStringObject((char *)value, value_len);
+        autoMemoryAdd(key->ctx, REDISMODULE_AM_STRING, *value_ptr);
+    }
+    key->u.stream.numfieldsleft--;
+    return REDISMODULE_OK;
+}
+
+/* Deletes the current stream entry while iterating.
+ *
+ * This function can be called after RM_StreamIteratorNextID() or after any
+ * calls to RM_StreamIteratorNextField().
+ *
+ * Returns REDISMODULE_OK on success. On failure, REDISMODULE_ERR is returned
+ * and `errno` is set as follows:
+ *
+ * - EINVAL if key is NULL
+ * - ENOTSUP if the key is empty or is of another type than stream
+ * - EBADF if the key is not opened for writing, if no iterator has been started
+ * - ENOENT if the iterator has no current stream entry
+ */
+int RM_StreamIteratorDelete(RedisModuleKey *key) {
+    if (!key) {
+        errno = EINVAL;
+        return REDISMODULE_ERR;
+    } else if (!key->value || key->value->type != OBJ_STREAM) {
+        errno = ENOTSUP;
+        return REDISMODULE_ERR;
+    } else if (!(key->mode & REDISMODULE_WRITE) || !key->iter) {
+        errno = EBADF;
+        return REDISMODULE_ERR;
+    } else if (key->u.stream.currentid.ms == 0 &&
+               key->u.stream.currentid.seq == 0) {
+        errno = ENOENT;
+        return REDISMODULE_ERR;
+    }
+    streamIterator *si = key->iter;
+    streamIteratorRemoveEntry(si, &key->u.stream.currentid);
+    key->u.stream.currentid.ms = 0; /* Make sure repeated Delete() fails */
+    key->u.stream.currentid.seq = 0;
+    key->u.stream.numfieldsleft = 0; /* Make sure NextField() fails */
+    return REDISMODULE_OK;
+}
+
+/* Trim a stream by length, similar to XTRIM with MAXLEN.
+ *
+ * - `key`: Key opened for writing.
+ * - `flags`: A bitfield of
+ *   - `REDISMODULE_STREAM_TRIM_APPROX`: Trim less if it improves performance,
+ *     like XTRIM with `~`.
+ * - `length`: The number of stream entries to keep after trimming.
+ *
+ * Returns the number of entries deleted. On failure, a negative value is
+ * returned and `errno` is set as follows:
+ *
+ * - EINVAL if called with invalid arguments
+ * - ENOTSUP if the key is empty or of a type other than stream
+ * - EBADF if the key is not opened for writing
+ */
+long long RM_StreamTrimByLength(RedisModuleKey *key, int flags, long long length) {
+    if (!key || (flags & ~(REDISMODULE_STREAM_TRIM_APPROX)) || length < 0) {
+        errno = EINVAL;
+        return -1;
+    } else if (!key->value || key->value->type != OBJ_STREAM) {
+        errno = ENOTSUP;
+        return -1;
+    } else if (!(key->mode & REDISMODULE_WRITE)) {
+        errno = EBADF;
+        return -1;
+    }
+    int approx = flags & REDISMODULE_STREAM_TRIM_APPROX ? 1 : 0;
+    return streamTrimByLength((stream *)key->value->ptr, length, approx);
+}
+
+/* Trim a stream by ID, similar to XTRIM with MINID.
+ *
+ * - `key`: Key opened for writing.
+ * - `flags`: A bitfield of
+ *   - `REDISMODULE_STREAM_TRIM_APPROX`: Trim less if it improves performance,
+ *     like XTRIM with `~`.
+ * - `id`: The smallest stream ID to keep after trimming.
+ *
+ * Returns the number of entries deleted. On failure, a negative value is
+ * returned and `errno` is set as follows:
+ *
+ * - EINVAL if called with invalid arguments
+ * - ENOTSUP if the key is empty or of a type other than stream
+ * - EBADF if the key is not opened for writing
+ */
+long long RM_StreamTrimByID(RedisModuleKey *key, int flags, RedisModuleStreamID *id) {
+    if (!key || (flags & ~(REDISMODULE_STREAM_TRIM_APPROX)) || !id) {
+        errno = EINVAL;
+        return -1;
+    } else if (!key->value || key->value->type != OBJ_STREAM) {
+        errno = ENOTSUP;
+        return -1;
+    } else if (!(key->mode & REDISMODULE_WRITE)) {
+        errno = EBADF;
+        return -1;
+    }
+    int approx = flags & REDISMODULE_STREAM_TRIM_APPROX ? 1 : 0;
+    streamID minid = (streamID){id->ms, id->seq};
+    return streamTrimByID((stream *)key->value->ptr, minid, approx);
+}
+
 /* --------------------------------------------------------------------------
  * Redis <-> Modules generic Call() API
  * -------------------------------------------------------------------------- */
@@ -8462,6 +8979,7 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(StringToLongLong);
     REGISTER_API(StringToDouble);
     REGISTER_API(StringToLongDouble);
+    REGISTER_API(StringToStreamID);
     REGISTER_API(Call);
     REGISTER_API(CallReplyProto);
     REGISTER_API(FreeCallReply);
@@ -8476,6 +8994,7 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(CreateStringFromDouble);
     REGISTER_API(CreateStringFromLongDouble);
     REGISTER_API(CreateStringFromString);
+    REGISTER_API(CreateStringFromStreamID);
     REGISTER_API(CreateStringPrintf);
     REGISTER_API(FreeString);
     REGISTER_API(StringPtrLen);
@@ -8507,6 +9026,15 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(ZsetRangeEndReached);
     REGISTER_API(HashSet);
     REGISTER_API(HashGet);
+    REGISTER_API(StreamAdd);
+    REGISTER_API(StreamDelete);
+    REGISTER_API(StreamIteratorStart);
+    REGISTER_API(StreamIteratorStop);
+    REGISTER_API(StreamIteratorNextID);
+    REGISTER_API(StreamIteratorNextField);
+    REGISTER_API(StreamIteratorDelete);
+    REGISTER_API(StreamTrimByLength);
+    REGISTER_API(StreamTrimByID);
     REGISTER_API(IsKeysPositionRequest);
     REGISTER_API(KeyAtPos);
     REGISTER_API(GetClientId);
diff --git a/src/redismodule.h b/src/redismodule.h
index 36c566bb3..1b1304172 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -69,6 +69,20 @@
 #define REDISMODULE_HASH_CFIELDS    (1<<2)
 #define REDISMODULE_HASH_EXISTS     (1<<3)
 
+/* StreamID type. */
+typedef struct RedisModuleStreamID {
+    uint64_t ms;
+    uint64_t seq;
+} RedisModuleStreamID;
+
+/* StreamAdd() flags. */
+#define REDISMODULE_STREAM_ADD_AUTOID (1<<0)
+/* StreamIteratorStart() flags. */
+#define REDISMODULE_STREAM_ITERATOR_EXCLUSIVE (1<<0)
+#define REDISMODULE_STREAM_ITERATOR_REVERSE (1<<1)
+/* StreamIteratorTrim*() flags. */
+#define REDISMODULE_STREAM_TRIM_APPROX (1<<0)
+
 /* Context Flags: Info about the current context returned by
  * RM_GetContextFlags(). */
 
@@ -578,6 +592,7 @@ REDISMODULE_API RedisModuleString * (*RedisModule_CreateStringFromLongLong)(Redi
 REDISMODULE_API RedisModuleString * (*RedisModule_CreateStringFromDouble)(RedisModuleCtx *ctx, double d) REDISMODULE_ATTR;
 REDISMODULE_API RedisModuleString * (*RedisModule_CreateStringFromLongDouble)(RedisModuleCtx *ctx, long double ld, int humanfriendly) REDISMODULE_ATTR;
 REDISMODULE_API RedisModuleString * (*RedisModule_CreateStringFromString)(RedisModuleCtx *ctx, const RedisModuleString *str) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_CreateStringFromStreamID)(RedisModuleCtx *ctx, const RedisModuleStreamID *id) REDISMODULE_ATTR;
 REDISMODULE_API RedisModuleString * (*RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...) REDISMODULE_ATTR_PRINTF(2,3) REDISMODULE_ATTR;
 REDISMODULE_API void (*RedisModule_FreeString)(RedisModuleCtx *ctx, RedisModuleString *str) REDISMODULE_ATTR;
 REDISMODULE_API const char * (*RedisModule_StringPtrLen)(const RedisModuleString *str, size_t *len) REDISMODULE_ATTR;
@@ -599,6 +614,7 @@ REDISMODULE_API int (*RedisModule_ReplyWithCallReply)(RedisModuleCtx *ctx, Redis
 REDISMODULE_API int (*RedisModule_StringToLongLong)(const RedisModuleString *str, long long *ll) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_StringToDouble)(const RedisModuleString *str, double *d) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_StringToLongDouble)(const RedisModuleString *str, long double *d) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StringToStreamID)(const RedisModuleString *str, RedisModuleStreamID *id) REDISMODULE_ATTR;
 REDISMODULE_API void (*RedisModule_AutoMemory)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_Replicate)(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_ReplicateVerbatim)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
@@ -629,6 +645,15 @@ REDISMODULE_API int (*RedisModule_ZsetRangePrev)(RedisModuleKey *key) REDISMODUL
 REDISMODULE_API int (*RedisModule_ZsetRangeEndReached)(RedisModuleKey *key) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_HashSet)(RedisModuleKey *key, int flags, ...) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_HashGet)(RedisModuleKey *key, int flags, ...) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StreamAdd)(RedisModuleKey *key, int flags, RedisModuleStreamID *id, RedisModuleString **argv, int64_t numfields) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StreamDelete)(RedisModuleKey *key, RedisModuleStreamID *id) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StreamIteratorStart)(RedisModuleKey *key, int flags, RedisModuleStreamID *startid, RedisModuleStreamID *endid) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StreamIteratorStop)(RedisModuleKey *key) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StreamIteratorNextID)(RedisModuleKey *key, RedisModuleStreamID *id, long *numfields) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StreamIteratorNextField)(RedisModuleKey *key, RedisModuleString **field_ptr, RedisModuleString **value_ptr) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_StreamIteratorDelete)(RedisModuleKey *key) REDISMODULE_ATTR;
+REDISMODULE_API long long (*RedisModule_StreamTrimByLength)(RedisModuleKey *key, int flags, long long length) REDISMODULE_ATTR;
+REDISMODULE_API long long (*RedisModule_StreamTrimByID)(RedisModuleKey *key, int flags, RedisModuleStreamID *id) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_IsKeysPositionRequest)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
 REDISMODULE_API void (*RedisModule_KeyAtPos)(RedisModuleCtx *ctx, int pos) REDISMODULE_ATTR;
 REDISMODULE_API unsigned long long (*RedisModule_GetClientId)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
@@ -842,6 +867,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(StringToLongLong);
     REDISMODULE_GET_API(StringToDouble);
     REDISMODULE_GET_API(StringToLongDouble);
+    REDISMODULE_GET_API(StringToStreamID);
     REDISMODULE_GET_API(Call);
     REDISMODULE_GET_API(CallReplyProto);
     REDISMODULE_GET_API(FreeCallReply);
@@ -856,6 +882,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(CreateStringFromDouble);
     REDISMODULE_GET_API(CreateStringFromLongDouble);
     REDISMODULE_GET_API(CreateStringFromString);
+    REDISMODULE_GET_API(CreateStringFromStreamID);
     REDISMODULE_GET_API(CreateStringPrintf);
     REDISMODULE_GET_API(FreeString);
     REDISMODULE_GET_API(StringPtrLen);
@@ -887,6 +914,15 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(ZsetRangeEndReached);
     REDISMODULE_GET_API(HashSet);
     REDISMODULE_GET_API(HashGet);
+    REDISMODULE_GET_API(StreamAdd);
+    REDISMODULE_GET_API(StreamDelete);
+    REDISMODULE_GET_API(StreamIteratorStart);
+    REDISMODULE_GET_API(StreamIteratorStop);
+    REDISMODULE_GET_API(StreamIteratorNextID);
+    REDISMODULE_GET_API(StreamIteratorNextField);
+    REDISMODULE_GET_API(StreamIteratorDelete);
+    REDISMODULE_GET_API(StreamTrimByLength);
+    REDISMODULE_GET_API(StreamTrimByID);
     REDISMODULE_GET_API(IsKeysPositionRequest);
     REDISMODULE_GET_API(KeyAtPos);
     REDISMODULE_GET_API(GetClientId);
diff --git a/src/stream.h b/src/stream.h
index c7acee719..1f2132365 100644
--- a/src/stream.h
+++ b/src/stream.h
@@ -108,6 +108,7 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
 void streamIteratorStart(streamIterator *si, stream *s, streamID *start, streamID *end, int rev);
 int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields);
 void streamIteratorGetField(streamIterator *si, unsigned char **fieldptr, unsigned char **valueptr, int64_t *fieldlen, int64_t *valuelen);
+void streamIteratorRemoveEntry(streamIterator *si, streamID *current);
 void streamIteratorStop(streamIterator *si);
 streamCG *streamLookupCG(stream *s, sds groupname);
 streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int flags, int *created);
@@ -121,5 +122,11 @@ int streamDecrID(streamID *id);
 void streamPropagateConsumerCreation(client *c, robj *key, robj *groupname, sds consumername);
 robj *streamDup(robj *o);
 int streamValidateListpackIntegrity(unsigned char *lp, size_t size, int deep);
+int streamParseID(const robj *o, streamID *id);
+robj *createObjectFromStreamID(streamID *id);
+int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_id, streamID *use_id);
+int streamDeleteItem(stream *s, streamID *id);
+int64_t streamTrimByLength(stream *s, long long maxlen, int approx);
+int64_t streamTrimByID(stream *s, streamID minid, int approx);
 
 #endif
diff --git a/src/t_stream.c b/src/t_stream.c
index f991765eb..197b7d4f7 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -818,6 +818,28 @@ int64_t streamTrim(stream *s, streamAddTrimArgs *args) {
     return deleted;
 }
 
+/* Trims a stream by length. Returns the number of deleted items. */
+int64_t streamTrimByLength(stream *s, long long maxlen, int approx) {
+    streamAddTrimArgs args = {
+        .trim_strategy = TRIM_STRATEGY_MAXLEN,
+        .approx_trim = approx,
+        .limit = approx ? 100 * server.stream_node_max_entries : 0,
+        .maxlen = maxlen
+    };
+    return streamTrim(s, &args);
+}
+
+/* Trims a stream by minimum ID. Returns the number of deleted items. */
+int64_t streamTrimByID(stream *s, streamID minid, int approx) {
+    streamAddTrimArgs args = {
+        .trim_strategy = TRIM_STRATEGY_MINID,
+        .approx_trim = approx,
+        .limit = approx ? 100 * server.stream_node_max_entries : 0,
+        .minid = minid
+    };
+    return streamTrim(s, &args);
+}
+
 /* Parse the arguements of XADD/XTRIM.
  *
  * See streamAddTrimArgs for more details about the arguments handled.
@@ -1625,7 +1647,7 @@ robj *streamTypeLookupWriteOrCreate(client *c, robj *key, int no_create) {
  * treated as an invalid ID.
  *
  * If 'c' is set to NULL, no reply is sent to the client. */
-int streamGenericParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq, int strict) {
+int streamGenericParseIDOrReply(client *c, const robj *o, streamID *id, uint64_t missing_seq, int strict) {
     char buf[128];
     if (sdslen(o->ptr) > sizeof(buf)-1) goto invalid;
     memcpy(buf,o->ptr,sdslen(o->ptr)+1);
@@ -1661,6 +1683,11 @@ invalid:
     return C_ERR;
 }
 
+/* Wrapper for streamGenericParseIDOrReply() used by module API. */
+int streamParseID(const robj *o, streamID *id) {
+    return streamGenericParseIDOrReply(NULL, o, id, 0, 0);
+}
+
 /* Wrapper for streamGenericParseIDOrReply() with 'strict' argument set to
  * 0, to be used when - and + are acceptable IDs. */
 int streamParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq) {
diff --git a/tests/modules/Makefile b/tests/modules/Makefile
index 7363c98bc..febed82d0 100644
--- a/tests/modules/Makefile
+++ b/tests/modules/Makefile
@@ -27,7 +27,8 @@ TEST_MODULES = \
     getkeys.so \
     test_lazyfree.so \
     timer.so \
-    defragtest.so
+    defragtest.so \
+    stream.so
 
 
 .PHONY: all
diff --git a/tests/modules/stream.c b/tests/modules/stream.c
new file mode 100644
index 000000000..abfbb1faf
--- /dev/null
+++ b/tests/modules/stream.c
@@ -0,0 +1,258 @@
+#include "redismodule.h"
+
+#include <string.h>
+#include <strings.h>
+#include <assert.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Command which adds a stream entry with automatic ID, like XADD *.
+ *
+ * Syntax: STREAM.ADD key field1 value1 [ field2 value2 ... ]
+ *
+ * The response is the ID of the added stream entry or an error message.
+ */
+int stream_add(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc < 2 || argc % 2 != 0) {
+        RedisModule_WrongArity(ctx);
+        return REDISMODULE_OK;
+    }
+
+    RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE);
+    RedisModuleStreamID id;
+    if (RedisModule_StreamAdd(key, REDISMODULE_STREAM_ADD_AUTOID, &id,
+                              &argv[2], (argc-2)/2) == REDISMODULE_OK) {
+        RedisModuleString *id_str = RedisModule_CreateStringFromStreamID(ctx, &id);
+        RedisModule_ReplyWithString(ctx, id_str);
+        RedisModule_FreeString(ctx, id_str);
+    } else {
+        RedisModule_ReplyWithError(ctx, "ERR StreamAdd failed");
+    }
+    RedisModule_CloseKey(key);
+    return REDISMODULE_OK;
+}
+
+/* Command which adds a stream entry N times.
+ *
+ * Syntax: STREAM.ADD key N field1 value1 [ field2 value2 ... ]
+ *
+ * Returns the number of successfully added entries.
+ */
+int stream_addn(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc < 3 || argc % 2 == 0) {
+        RedisModule_WrongArity(ctx);
+        return REDISMODULE_OK;
+    }
+
+    long long n, i;
+    if (RedisModule_StringToLongLong(argv[2], &n) == REDISMODULE_ERR) {
+        RedisModule_ReplyWithError(ctx, "N must be a number");
+        return REDISMODULE_OK;
+    }
+
+    RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE);
+    for (i = 0; i < n; i++) {
+        if (RedisModule_StreamAdd(key, REDISMODULE_STREAM_ADD_AUTOID, NULL,
+                                  &argv[3], (argc-3)/2) == REDISMODULE_ERR)
+            break;
+    }
+    RedisModule_ReplyWithLongLong(ctx, i);
+    RedisModule_CloseKey(key);
+    return REDISMODULE_OK;
+}
+
+/* STREAM.DELETE key stream-id */
+int stream_delete(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 3) return RedisModule_WrongArity(ctx);
+    RedisModuleStreamID id;
+    if (RedisModule_StringToStreamID(argv[2], &id) != REDISMODULE_OK) {
+        return RedisModule_ReplyWithError(ctx, "Invalid stream ID");
+    }
+    RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE);
+    if (RedisModule_StreamDelete(key, &id) == REDISMODULE_OK) {
+        RedisModule_ReplyWithSimpleString(ctx, "OK");
+    } else {
+        RedisModule_ReplyWithError(ctx, "ERR StreamDelete failed");
+    }
+    RedisModule_CloseKey(key);
+    return REDISMODULE_OK;
+}
+
+/* STREAM.RANGE key start-id end-id
+ *
+ * Returns an array of stream items. Each item is an array on the form
+ * [stream-id, [field1, value1, field2, value2, ...]].
+ *
+ * A funny side-effect used for testing RM_StreamIteratorDelete() is that if any
+ * entry has a field named "selfdestruct", the stream entry is deleted. It is
+ * however included in the results of this command.
+ */
+int stream_range(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 4) {
+        RedisModule_WrongArity(ctx);
+        return REDISMODULE_OK;
+    }
+
+    RedisModuleStreamID startid, endid;
+    if (RedisModule_StringToStreamID(argv[2], &startid) != REDISMODULE_OK ||
+        RedisModule_StringToStreamID(argv[3], &endid) != REDISMODULE_OK) {
+        RedisModule_ReplyWithError(ctx, "Invalid stream ID");
+        return REDISMODULE_OK;
+    }
+
+    /* If startid > endid, we swap and set the reverse flag. */
+    int flags = 0;
+    if (startid.ms > endid.ms ||
+        (startid.ms == endid.ms && startid.seq > endid.seq)) {
+        RedisModuleStreamID tmp = startid;
+        startid = endid;
+        endid = tmp;
+        flags |= REDISMODULE_STREAM_ITERATOR_REVERSE;
+    }
+
+    /* Open key and start iterator. */
+    int openflags = REDISMODULE_READ | REDISMODULE_WRITE;
+    RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], openflags);
+    if (RedisModule_StreamIteratorStart(key, flags,
+                                        &startid, &endid) != REDISMODULE_OK) {
+        /* Key is not a stream, etc. */
+        RedisModule_ReplyWithError(ctx, "ERR StreamIteratorStart failed");
+        RedisModule_CloseKey(key);
+        return REDISMODULE_OK;
+    }
+
+    /* Check error handling: Delete current entry when no current entry. */
+    assert(RedisModule_StreamIteratorDelete(key) ==
+           REDISMODULE_ERR);
+    assert(errno == ENOENT);
+
+    /* Check error handling: Fetch fields when no current entry. */
+    assert(RedisModule_StreamIteratorNextField(key, NULL, NULL) ==
+           REDISMODULE_ERR);
+    assert(errno == ENOENT);
+
+    /* Return array. */
+    RedisModule_ReplyWithArray(ctx, REDISMODULE_POSTPONED_ARRAY_LEN);
+    RedisModule_AutoMemory(ctx);
+    RedisModuleStreamID id;
+    long numfields;
+    long len = 0;
+    while (RedisModule_StreamIteratorNextID(key, &id,
+                                            &numfields) == REDISMODULE_OK) {
+        RedisModule_ReplyWithArray(ctx, 2);
+        RedisModuleString *id_str = RedisModule_CreateStringFromStreamID(ctx, &id);
+        RedisModule_ReplyWithString(ctx, id_str);
+        RedisModule_ReplyWithArray(ctx, numfields * 2);
+        int delete = 0;
+        RedisModuleString *field, *value;
+        for (long i = 0; i < numfields; i++) {
+            assert(RedisModule_StreamIteratorNextField(key, &field, &value) ==
+                   REDISMODULE_OK);
+            RedisModule_ReplyWithString(ctx, field);
+            RedisModule_ReplyWithString(ctx, value);
+            /* check if this is a "selfdestruct" field */
+            size_t field_len;
+            const char *field_str = RedisModule_StringPtrLen(field, &field_len);
+            if (!strncmp(field_str, "selfdestruct", field_len)) delete = 1;
+        }
+        if (delete) {
+            assert(RedisModule_StreamIteratorDelete(key) == REDISMODULE_OK);
+        }
+        /* check error handling: no more fields to fetch */
+        assert(RedisModule_StreamIteratorNextField(key, &field, &value) ==
+               REDISMODULE_ERR);
+        assert(errno == ENOENT);
+        len++;
+    }
+    RedisModule_ReplySetArrayLength(ctx, len);
+    RedisModule_StreamIteratorStop(key);
+    RedisModule_CloseKey(key);
+    return REDISMODULE_OK;
+}
+
+/*
+ * STREAM.TRIM key (MAXLEN (=|~) length | MINID (=|~) id)
+ */
+int stream_trim(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 5) {
+        RedisModule_WrongArity(ctx);
+        return REDISMODULE_OK;
+    }
+
+    /* Parse args */
+    int trim_by_id = 0; /* 0 = maxlen, 1 = minid */
+    long long maxlen;
+    RedisModuleStreamID minid;
+    size_t arg_len;
+    const char *arg = RedisModule_StringPtrLen(argv[2], &arg_len);
+    if (!strcasecmp(arg, "minid")) {
+        trim_by_id = 1;
+        if (RedisModule_StringToStreamID(argv[4], &minid) != REDISMODULE_OK) {
+            RedisModule_ReplyWithError(ctx, "ERR Invalid stream ID");
+            return REDISMODULE_OK;
+        }
+    } else if (!strcasecmp(arg, "maxlen")) {
+        if (RedisModule_StringToLongLong(argv[4], &maxlen) == REDISMODULE_ERR) {
+            RedisModule_ReplyWithError(ctx, "ERR Maxlen must be a number");
+            return REDISMODULE_OK;
+        }
+    } else {
+        RedisModule_ReplyWithError(ctx, "ERR Invalid arguments");
+        return REDISMODULE_OK;
+    }
+
+    /* Approx or exact */
+    int flags;
+    arg = RedisModule_StringPtrLen(argv[3], &arg_len);
+    if (arg_len == 1 && arg[0] == '~') {
+        flags = REDISMODULE_STREAM_TRIM_APPROX;
+    } else if (arg_len == 1 && arg[0] == '=') {
+        flags = 0;
+    } else {
+        RedisModule_ReplyWithError(ctx, "ERR Invalid approx-or-exact mark");
+        return REDISMODULE_OK;
+    }
+
+    /* Trim */
+    RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE);
+    long long trimmed;
+    if (trim_by_id) {
+        trimmed = RedisModule_StreamTrimByID(key, flags, &minid);
+    } else {
+        trimmed = RedisModule_StreamTrimByLength(key, flags, maxlen);
+    }
+
+    /* Return result */
+    if (trimmed < 0) {
+        RedisModule_ReplyWithError(ctx, "ERR Trimming failed");
+    } else {
+        RedisModule_ReplyWithLongLong(ctx, trimmed);
+    }
+    RedisModule_CloseKey(key);
+    return REDISMODULE_OK;
+}
+
+int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+    if (RedisModule_Init(ctx, "stream", 1, REDISMODULE_APIVER_1) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx, "stream.add", stream_add, "",
+                                  1, 1, 1) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+    if (RedisModule_CreateCommand(ctx, "stream.addn", stream_addn, "",
+                                  1, 1, 1) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+    if (RedisModule_CreateCommand(ctx, "stream.delete", stream_delete, "",
+                                  1, 1, 1) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+    if (RedisModule_CreateCommand(ctx, "stream.range", stream_range, "",
+                                  1, 1, 1) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+    if (RedisModule_CreateCommand(ctx, "stream.trim", stream_trim, "",
+                                  1, 1, 1) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    return REDISMODULE_OK;
+}
diff --git a/tests/unit/moduleapi/stream.tcl b/tests/unit/moduleapi/stream.tcl
new file mode 100644
index 000000000..15e97c183
--- /dev/null
+++ b/tests/unit/moduleapi/stream.tcl
@@ -0,0 +1,155 @@
+set testmodule [file normalize tests/modules/stream.so]
+
+start_server {tags {"modules"}} {
+    r module load $testmodule
+
+    test {Module stream add and delete} {
+        r del mystream
+        # add to empty key
+        set streamid1 [r stream.add mystream item 1 value a]
+        # add to existing stream
+        set streamid2 [r stream.add mystream item 2 value b]
+        # check result
+        assert { [string match "*-*" $streamid1] }
+        set items [r XRANGE mystream - +]
+        assert_equal $items \
+            "{$streamid1 {item 1 value a}} {$streamid2 {item 2 value b}}"
+        # delete one of them and try deleting non-existing ID
+        assert_equal OK [r stream.delete mystream $streamid1]
+        assert_error "ERR StreamDelete*" {r stream.delete mystream 123-456}
+        assert_error "Invalid stream ID*" {r stream.delete mystream foo}
+        assert_equal "{$streamid2 {item 2 value b}}" [r XRANGE mystream - +]
+        # check error condition: wrong type
+        r del mystream
+        r set mystream mystring
+        assert_error "ERR StreamAdd*" {r stream.add mystream item 1 value a}
+        assert_error "ERR StreamDelete*" {r stream.delete mystream 123-456}
+    }
+
+    test {Module stream add unblocks blocking xread} {
+        r del mystream
+
+        # Blocking XREAD on an empty key
+        set rd1 [redis_deferring_client]
+        $rd1 XREAD BLOCK 3000 STREAMS mystream $
+        # wait until client is actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Client is not blocked"
+        }
+        set id [r stream.add mystream field 1 value a]
+        assert_equal "{mystream {{$id {field 1 value a}}}}" [$rd1 read]
+
+        # Blocking XREAD on an existing stream
+        set rd2 [redis_deferring_client]
+        $rd2 XREAD BLOCK 3000 STREAMS mystream $
+        # wait until client is actually blocked
+        wait_for_condition 50 100 {
+            [s 0 blocked_clients] eq {1}
+        } else {
+            fail "Client is not blocked"
+        }
+        set id [r stream.add mystream field 2 value b]
+        assert_equal "{mystream {{$id {field 2 value b}}}}" [$rd2 read]
+    }
+
+    test {Module stream add benchmark (1M stream add)} {
+        set n 1000000
+        r del mystream
+        set result [r stream.addn mystream $n field value]
+        assert_equal $result $n
+    }
+
+    test {Module stream iterator} {
+        r del mystream
+        set streamid1 [r xadd mystream * item 1 value a]
+        set streamid2 [r xadd mystream * item 2 value b]
+        # range result
+        set result1 [r stream.range mystream "-" "+"]
+        set expect1 [r xrange mystream "-" "+"]
+        assert_equal $result1 $expect1
+        # reverse range
+        set result_rev [r stream.range mystream "+" "-"]
+        set expect_rev [r xrevrange mystream "+" "-"]
+        assert_equal $result_rev $expect_rev
+
+        # only one item: range with startid = endid
+        set result2 [r stream.range mystream "-" $streamid1]
+        assert_equal $result2 "{$streamid1 {item 1 value a}}"
+        assert_equal $result2 [list [list $streamid1 {item 1 value a}]]
+        # only one item: range with startid = endid
+        set result3 [r stream.range mystream $streamid2 $streamid2]
+        assert_equal $result3 "{$streamid2 {item 2 value b}}"
+        assert_equal $result3 [list [list $streamid2 {item 2 value b}]]
+    }
+
+    test {Module stream iterator delete} {
+        r del mystream
+        set id1 [r xadd mystream * normal item]
+        set id2 [r xadd mystream * selfdestruct yes]
+        set id3 [r xadd mystream * another item]
+        # stream.range deletes the "selfdestruct" item after returning it
+        assert_equal \
+            "{$id1 {normal item}} {$id2 {selfdestruct yes}} {$id3 {another item}}" \
+            [r stream.range mystream - +]
+        # now, the "selfdestruct" item is gone
+        assert_equal \
+            "{$id1 {normal item}} {$id3 {another item}}" \
+            [r stream.range mystream - +]
+    }
+
+    test {Module stream trim by length} {
+        r del mystream
+        # exact maxlen
+        r xadd mystream * item 1 value a
+        r xadd mystream * item 2 value b
+        r xadd mystream * item 3 value c
+        assert_equal 3 [r xlen mystream]
+        assert_equal 0 [r stream.trim mystream maxlen = 5]
+        assert_equal 3 [r xlen mystream]
+        assert_equal 2 [r stream.trim mystream maxlen = 1]
+        assert_equal 1 [r xlen mystream]
+        assert_equal 1 [r stream.trim mystream maxlen = 0]
+        # check that there is no limit for exact maxlen
+        r stream.addn mystream 20000 item x value y
+        assert_equal 20000 [r stream.trim mystream maxlen = 0]
+        # approx maxlen (100 items per node implies default limit 10K items)
+        r stream.addn mystream 20000 item x value y
+        assert_equal 20000 [r xlen mystream]
+        assert_equal 10000 [r stream.trim mystream maxlen ~ 2]
+        assert_equal 9900  [r stream.trim mystream maxlen ~ 2]
+        assert_equal 0     [r stream.trim mystream maxlen ~ 2]
+        assert_equal 100   [r xlen mystream]
+        assert_equal 100   [r stream.trim mystream maxlen ~ 0]
+        assert_equal 0     [r xlen mystream]
+    }
+
+    test {Module stream trim by ID} {
+        r del mystream
+        # exact minid
+        r xadd mystream * item 1 value a
+        r xadd mystream * item 2 value b
+        set minid [r xadd mystream * item 3 value c]
+        assert_equal 3 [r xlen mystream]
+        assert_equal 0 [r stream.trim mystream minid = -]
+        assert_equal 3 [r xlen mystream]
+        assert_equal 2 [r stream.trim mystream minid = $minid]
+        assert_equal 1 [r xlen mystream]
+        assert_equal 1 [r stream.trim mystream minid = +]
+        # check that there is no limit for exact minid
+        r stream.addn mystream 20000 item x value y
+        assert_equal 20000 [r stream.trim mystream minid = +]
+        # approx minid (100 items per node implies default limit 10K items)
+        r stream.addn mystream 19980 item x value y
+        set minid [r xadd mystream * item x value y]
+        r stream.addn mystream 19 item x value y
+        assert_equal 20000 [r xlen mystream]
+        assert_equal 10000 [r stream.trim mystream minid ~ $minid]
+        assert_equal 9900  [r stream.trim mystream minid ~ $minid]
+        assert_equal 0     [r stream.trim mystream minid ~ $minid]
+        assert_equal 100   [r xlen mystream]
+        assert_equal 100   [r stream.trim mystream minid ~ +]
+        assert_equal 0     [r xlen mystream]
+    }
+}

From a16739a3ac330b277d9e10c72af0d3b9176cc181 Mon Sep 17 00:00:00 2001
From: Wang Yuan <wangyuan21@baidu.com>
Date: Thu, 28 Jan 2021 22:25:35 +0800
Subject: [PATCH 1010/1098] Redis exit for fsync error when the AOF fsync
 policy is 'always' (#8347)

With AOF policy of fsync "always", redis should respect the contract with the user
that on acknowledged write data is already synced on disk.

Redis was already exiting for AOF write error,  but don't care about fsync failure.
So to guarantee data safe, redis should exit for fsync error too (when the AOF fsync
policy is 'always').
---
 src/aof.c | 32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index 5518f0206..f07f7ca2e 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -451,10 +451,11 @@ void flushAppendOnlyFile(int force) {
 
         /* Handle the AOF write error. */
         if (server.aof_fsync == AOF_FSYNC_ALWAYS) {
-            /* We can't recover when the fsync policy is ALWAYS since the
-             * reply for the client is already in the output buffers, and we
-             * have the contract with the user that on acknowledged write data
-             * is synced on disk. */
+            /* We can't recover when the fsync policy is ALWAYS since the reply
+             * for the client is already in the output buffers (both writes and
+             * reads), and the changes to the db can't be rolled back. Since we
+             * have a contract with the user that on acknowledged or observed
+             * writes are is synced on disk, we must exit. */
             serverLog(LL_WARNING,"Can't recover from AOF write error when the AOF fsync policy is 'always'. Exiting...");
             exit(1);
         } else {
@@ -502,7 +503,14 @@ try_fsync:
         /* redis_fsync is defined as fdatasync() for Linux in order to avoid
          * flushing metadata. */
         latencyStartMonitor(latency);
-        redis_fsync(server.aof_fd); /* Let's try to get this data on the disk */
+        /* Let's try to get this data on the disk. To guarantee data safe when
+         * the AOF fsync policy is 'always', we should exit if failed to fsync
+         * AOF (see comment next to the exit(1) after write error above). */
+        if (redis_fsync(server.aof_fd) == -1) {
+            serverLog(LL_WARNING,"Can't persist AOF for fsync error when the "
+              "AOF fsync policy is 'always': %s. Exiting...", strerror(errno));
+            exit(1);
+        }
         latencyEndMonitor(latency);
         latencyAddSampleIfNeeded("aof-fsync-always",latency);
         server.aof_fsync_offset = server.aof_current_size;
@@ -1847,6 +1855,16 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
             close(newfd);
             goto cleanup;
         }
+        if (server.aof_fsync == AOF_FSYNC_EVERYSEC) {
+            aof_background_fsync(newfd);
+        } else if (server.aof_fsync == AOF_FSYNC_ALWAYS) {
+            if (redis_fsync(newfd) == -1) {
+                serverLog(LL_WARNING,
+                    "Error trying to fsync the parent diff to the rewritten AOF: %s", strerror(errno));
+                close(newfd);
+                goto cleanup;
+            }
+        }
         latencyEndMonitor(latency);
         latencyAddSampleIfNeeded("aof-rewrite-diff-write",latency);
 
@@ -1916,10 +1934,6 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
             /* AOF enabled, replace the old fd with the new one. */
             oldfd = server.aof_fd;
             server.aof_fd = newfd;
-            if (server.aof_fsync == AOF_FSYNC_ALWAYS)
-                redis_fsync(newfd);
-            else if (server.aof_fsync == AOF_FSYNC_EVERYSEC)
-                aof_background_fsync(newfd);
             server.aof_selected_db = -1; /* Make sure SELECT is re-issued */
             aofUpdateCurrentSize();
             server.aof_rewrite_base_size = server.aof_current_size;

From 01cbf17ba2c0cd8ef7cbb232c0173deaf346788f Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Thu, 28 Jan 2021 15:38:49 +0100
Subject: [PATCH 1011/1098] Modules: Add event for fork child birth and
 termination (#8289)

Useful to avoid doing background jobs that can cause excessive COW
---
 src/module.c      | 10 ++++++++++
 src/redismodule.h | 13 ++++++++++---
 src/server.c      |  6 ++++++
 3 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/src/module.c b/src/module.c
index cabccfb7a..f3191bbc7 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7899,6 +7899,14 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) {
  *     * `REDISMODULE_SUBEVENT_REPL_BACKUP_RESTORE`
  *     * `REDISMODULE_SUBEVENT_REPL_BACKUP_DISCARD`
  *
+ * * RedisModuleEvent_ForkChild
+ *
+ *     Called when a fork child (AOFRW, RDBSAVE, module fork...) is born/dies
+ *     The following sub events are available:
+ *
+ *     * `REDISMODULE_SUBEVENT_FORK_CHILD_BORN`
+ *     * `REDISMODULE_SUBEVENT_FORK_CHILD_DIED`
+ *
  * The function returns REDISMODULE_OK if the module was successfully subscribed
  * for the specified event. If the API is called from a wrong context or unsupported event
  * is given then REDISMODULE_ERR is returned. */
@@ -7971,6 +7979,8 @@ int RM_IsSubEventSupported(RedisModuleEvent event, int64_t subevent) {
         return subevent < _REDISMODULE_SUBEVENT_SWAPDB_NEXT;
     case REDISMODULE_EVENT_REPL_BACKUP:
         return subevent < _REDISMODULE_SUBEVENT_REPL_BACKUP_NEXT;
+    case REDISMODULE_EVENT_FORK_CHILD:
+        return subevent < _REDISMODULE_SUBEVENT_FORK_CHILD_NEXT;
     default:
         break;
     }
diff --git a/src/redismodule.h b/src/redismodule.h
index 1b1304172..5d46b22f9 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -230,9 +230,8 @@ typedef uint64_t RedisModuleTimerID;
 #define REDISMODULE_EVENT_LOADING_PROGRESS 10
 #define REDISMODULE_EVENT_SWAPDB 11
 #define REDISMODULE_EVENT_REPL_BACKUP 12
-
-/* Next event flag, should be updated if a new event added. */
-#define _REDISMODULE_EVENT_NEXT 13
+#define REDISMODULE_EVENT_FORK_CHILD 13
+#define _REDISMODULE_EVENT_NEXT 14 /* Next event flag, should be updated if a new event added. */
 
 typedef struct RedisModuleEvent {
     uint64_t id;        /* REDISMODULE_EVENT_... defines. */
@@ -295,6 +294,10 @@ static const RedisModuleEvent
     RedisModuleEvent_ReplBackup = {
         REDISMODULE_EVENT_REPL_BACKUP,
         1
+    },
+    RedisModuleEvent_ForkChild = {
+        REDISMODULE_EVENT_FORK_CHILD,
+        1
     };
 
 /* Those are values that are used for the 'subevent' callback argument. */
@@ -345,6 +348,10 @@ static const RedisModuleEvent
 #define REDISMODULE_SUBEVENT_REPL_BACKUP_DISCARD 2
 #define _REDISMODULE_SUBEVENT_REPL_BACKUP_NEXT 3
 
+#define REDISMODULE_SUBEVENT_FORK_CHILD_BORN 0
+#define REDISMODULE_SUBEVENT_FORK_CHILD_DIED 1
+#define _REDISMODULE_SUBEVENT_FORK_CHILD_NEXT 2
+
 #define _REDISMODULE_SUBEVENT_SHUTDOWN_NEXT 0
 #define _REDISMODULE_SUBEVENT_CRON_LOOP_NEXT 0
 #define _REDISMODULE_SUBEVENT_SWAPDB_NEXT 0
diff --git a/src/server.c b/src/server.c
index ce34af3df..9135c0867 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1600,6 +1600,9 @@ void resetChildState() {
     server.stat_current_cow_bytes = 0;
     updateDictResizePolicy();
     closeChildInfoPipe();
+    moduleFireServerEvent(REDISMODULE_EVENT_FORK_CHILD,
+                          REDISMODULE_SUBEVENT_FORK_CHILD_DIED,
+                          NULL);
 }
 
 /* Return if child type is mutual exclusive with other fork children */
@@ -5567,6 +5570,9 @@ int redisFork(int purpose) {
         }
 
         updateDictResizePolicy();
+        moduleFireServerEvent(REDISMODULE_EVENT_FORK_CHILD,
+                              REDISMODULE_SUBEVENT_FORK_CHILD_BORN,
+                              NULL);
     }
     return childpid;
 }

From 4bb5ccbefbde8dffe4597de3be9e4b6730857177 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 28 Jan 2021 18:17:39 +0200
Subject: [PATCH 1012/1098] Add proc-title-template option. (#8397)

Make it possible to customize the process title, i.e. include custom
strings, immutable configuration like port, tls-port, unix socket name,
etc.
---
 redis.conf           | 17 ++++++++
 src/config.c         | 20 ++++++++++
 src/sds.c            | 92 ++++++++++++++++++++++++++++++++++++++++++++
 src/sds.h            |  8 ++++
 src/server.c         | 70 +++++++++++++++++++++++++++------
 src/server.h         |  5 ++-
 tests/unit/other.tcl | 44 +++++++++++++++++++++
 7 files changed, 244 insertions(+), 12 deletions(-)

diff --git a/redis.conf b/redis.conf
index 370542344..465d56fc0 100644
--- a/redis.conf
+++ b/redis.conf
@@ -330,6 +330,23 @@ always-show-logo no
 # the process name as executed by setting the following to no.
 set-proc-title yes
 
+# When changing the process title, Redis uses the following template to construct
+# the modified title.
+#
+# Template variables are specified in curly brackets. The following variables are
+# supported:
+#
+# {title}           Name of process as executed if parent, or type of child process.
+# {listen-addr}     Bind address or '*' followed by TCP or TLS port listening on, or
+#                   Unix socket if only that's available.
+# {server-mode}     Special mode, i.e. "[sentinel]" or "[cluster]".
+# {port}            TCP port listening on, or 0.
+# {tls-port}        TLS port listening on, or 0.
+# {unixsocket}      Unix domain socket listening on, or "".
+# {config-file}     Name of configuration file used.
+#
+proc-title-template "{title} {listen-addr} {server-mode}"
+
 ################################ SNAPSHOTTING  ################################
 
 # Save the DB to disk.
diff --git a/src/config.c b/src/config.c
index 20d432573..0bd89c2b9 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2236,6 +2236,25 @@ static int isValidAOFfilename(char *val, const char **err) {
     return 1;
 }
 
+/* Validate specified string is a valid proc-title-template */
+static int isValidProcTitleTemplate(char *val, const char **err) {
+    if (!validateProcTitleTemplate(val)) {
+        *err = "template format is invalid or contains unknown variables";
+        return 0;
+    }
+    return 1;
+}
+
+static int updateProcTitleTemplate(char *val, char *prev, const char **err) {
+    UNUSED(val);
+    UNUSED(prev);
+    if (redisSetProcTitle(NULL) == C_ERR) {
+        *err = "failed to set process title";
+        return 0;
+    }
+    return 1;
+}
+
 static int updateHZ(long long val, long long prev, const char **err) {
     UNUSED(prev);
     UNUSED(err);
@@ -2435,6 +2454,7 @@ standardConfig configs[] = {
     createStringConfig("aof_rewrite_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.aof_rewrite_cpulist, NULL, NULL, NULL),
     createStringConfig("bgsave_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bgsave_cpulist, NULL, NULL, NULL),
     createStringConfig("ignore-warnings", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.ignore_warnings, "", NULL, NULL),
+    createStringConfig("proc-title-template", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.proc_title_template, CONFIG_DEFAULT_PROC_TITLE_TEMPLATE, isValidProcTitleTemplate, updateProcTitleTemplate),
 
     /* SDS Configs */
     createSDSConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL, NULL, NULL),
diff --git a/src/sds.c b/src/sds.c
index f16114471..ad30e2ad4 100644
--- a/src/sds.c
+++ b/src/sds.c
@@ -1157,12 +1157,80 @@ void *sds_malloc(size_t size) { return s_malloc(size); }
 void *sds_realloc(void *ptr, size_t size) { return s_realloc(ptr,size); }
 void sds_free(void *ptr) { s_free(ptr); }
 
+/* Perform expansion of a template string and return the result as a newly
+ * allocated sds.
+ *
+ * Template variables are specified using curly brackets, e.g. {variable}.
+ * An opening bracket can be quoted by repeating it twice.
+ */
+sds sdstemplate(const char *template, sdstemplate_callback_t cb_func, void *cb_arg)
+{
+    sds res = sdsempty();
+    const char *p = template;
+
+    while (*p) {
+        /* Find next variable, copy everything until there */
+        const char *sv = strchr(p, '{');
+        if (!sv) {
+            /* Not found: copy till rest of template and stop */
+            res = sdscat(res, p);
+            break;
+        } else if (sv > p) {
+            /* Found: copy anything up to the begining of the variable */
+            res = sdscatlen(res, p, sv - p);
+        }
+
+        /* Skip into variable name, handle premature end or quoting */
+        sv++;
+        if (!*sv) goto error;       /* Premature end of template */
+        if (*sv == '{') {
+            /* Quoted '{' */
+            p = sv + 1;
+            res = sdscat(res, "{");
+            continue;
+        }
+
+        /* Find end of variable name, handle premature end of template */
+        const char *ev = strchr(sv, '}');
+        if (!ev) goto error;
+
+        /* Pass variable name to callback and obtain value. If callback failed,
+         * abort. */
+        sds varname = sdsnewlen(sv, ev - sv);
+        sds value = cb_func(varname, cb_arg);
+        sdsfree(varname);
+        if (!value) goto error;
+
+        /* Append value to result and continue */
+        res = sdscat(res, value);
+        sdsfree(value);
+        p = ev + 1;
+    }
+
+    return res;
+
+error:
+    sdsfree(res);
+    return NULL;
+}
+
 #ifdef REDIS_TEST
 #include <stdio.h>
 #include <limits.h>
 #include "testhelp.h"
 
 #define UNUSED(x) (void)(x)
+
+static sds sdsTestTemplateCallback(sds varname, void *arg) {
+    UNUSED(arg);
+    static const char *_var1 = "variable1";
+    static const char *_var2 = "variable2";
+
+    if (!strcmp(varname, _var1)) return sdsnew("value1");
+    else if (!strcmp(varname, _var2)) return sdsnew("value2");
+    else return NULL;
+}
+
 int sdsTest(int argc, char **argv) {
     UNUSED(argc);
     UNUSED(argv);
@@ -1342,6 +1410,30 @@ int sdsTest(int argc, char **argv) {
 
             sdsfree(x);
         }
+
+        /* Simple template */
+        x = sdstemplate("v1={variable1} v2={variable2}", sdsTestTemplateCallback, NULL);
+        test_cond("sdstemplate() normal flow",
+                  memcmp(x,"v1=value1 v2=value2",19) == 0);
+        sdsfree(x);
+
+        /* Template with callback error */
+        x = sdstemplate("v1={variable1} v3={doesnotexist}", sdsTestTemplateCallback, NULL);
+        test_cond("sdstemplate() with callback error", x == NULL);
+
+        /* Template with empty var name */
+        x = sdstemplate("v1={", sdsTestTemplateCallback, NULL);
+        test_cond("sdstemplate() with empty var name", x == NULL);
+
+        /* Template with truncated var name */
+        x = sdstemplate("v1={start", sdsTestTemplateCallback, NULL);
+        test_cond("sdstemplate() with truncated var name", x == NULL);
+
+        /* Template with quoting */
+        x = sdstemplate("v1={{{variable1}} {{} v2={variable2}", sdsTestTemplateCallback, NULL);
+        test_cond("sdstemplate() with quoting",
+                  memcmp(x,"v1={value1} {} v2=value2",24) == 0);
+        sdsfree(x);
     }
     test_report();
     return 0;
diff --git a/src/sds.h b/src/sds.h
index 3a9e4cefe..85dc0b680 100644
--- a/src/sds.h
+++ b/src/sds.h
@@ -253,6 +253,14 @@ sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen);
 sds sdsjoin(char **argv, int argc, char *sep);
 sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen);
 
+/* Callback for sdstemplate. The function gets called by sdstemplate
+ * every time a variable needs to be expanded. The variable name is
+ * provided as variable, and the callback is expected to return a
+ * substitution value. Returning a NULL indicates an error.
+ */
+typedef sds (*sdstemplate_callback_t)(const sds variable, void *arg);
+sds sdstemplate(const char *template, sdstemplate_callback_t cb_func, void *cb_arg);
+
 /* Low level functions exposed to the user API */
 sds sdsMakeRoomFor(sds s, size_t addlen);
 void sdsIncrLen(sds s, ssize_t incr);
diff --git a/src/server.c b/src/server.c
index 9135c0867..8b8da8c89 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5648,20 +5648,68 @@ void redisOutOfMemoryHandler(size_t allocation_size) {
         allocation_size);
 }
 
-void redisSetProcTitle(char *title) {
-#ifdef USE_SETPROCTITLE
-    char *server_mode = "";
-    if (server.cluster_enabled) server_mode = " [cluster]";
-    else if (server.sentinel_mode) server_mode = " [sentinel]";
+/* Callback for sdstemplate on proc-title-template. See redis.conf for
+ * supported variables.
+ */
+static sds redisProcTitleGetVariable(const sds varname, void *arg)
+{
+    if (!strcmp(varname, "title")) {
+        return sdsnew(arg);
+    } else if (!strcmp(varname, "listen-addr")) {
+        if (server.port || server.tls_port)
+            return sdscatprintf(sdsempty(), "%s:%u",
+                                server.bindaddr_count ? server.bindaddr[0] : "*",
+                                server.port ? server.port : server.tls_port);
+        else
+            return sdscatprintf(sdsempty(), "unixsocket:%s", server.unixsocket);
+    } else if (!strcmp(varname, "server-mode")) {
+        if (server.cluster_enabled) return sdsnew("[cluster]");
+        else if (server.sentinel_mode) return sdsnew("[sentinel]");
+        else return sdsempty();
+    } else if (!strcmp(varname, "config-file")) {
+        return sdsnew(server.configfile ? server.configfile : "-");
+    } else if (!strcmp(varname, "port")) {
+        return sdscatprintf(sdsempty(), "%u", server.port);
+    } else if (!strcmp(varname, "tls-port")) {
+        return sdscatprintf(sdsempty(), "%u", server.tls_port);
+    } else if (!strcmp(varname, "unixsocket")) {
+        return sdsnew(server.unixsocket);
+    } else
+        return NULL;    /* Unknown variable name */
+}
 
-    setproctitle("%s %s:%d%s",
-        title,
-        server.bindaddr_count ? server.bindaddr[0] : "*",
-        server.port ? server.port : server.tls_port,
-        server_mode);
+/* Expand the specified proc-title-template string and return a newly
+ * allocated sds, or NULL. */
+static sds expandProcTitleTemplate(const char *template, const char *title) {
+    sds res = sdstemplate(template, redisProcTitleGetVariable, (void *) title);
+    if (!res)
+        return NULL;
+    return sdstrim(res, " ");
+}
+/* Validate the specified template, returns 1 if valid or 0 otherwise. */
+int validateProcTitleTemplate(const char *template) {
+    int ok = 1;
+    sds res = expandProcTitleTemplate(template, "");
+    if (!res)
+        return 0;
+    if (sdslen(res) == 0) ok = 0;
+    sdsfree(res);
+    return ok;
+}
+
+int redisSetProcTitle(char *title) {
+#ifdef USE_SETPROCTITLE
+    if (!title) title = server.exec_argv[0];
+    sds proc_title = expandProcTitleTemplate(server.proc_title_template, title);
+    if (!proc_title) return C_ERR;  /* Not likely, proc_title_template is validated */
+
+    setproctitle("%s", proc_title);
+    sdsfree(proc_title);
 #else
     UNUSED(title);
 #endif
+
+    return C_OK;
 }
 
 void redisSetCpuAffinity(const char *cpulist) {
@@ -5925,7 +5973,7 @@ int main(int argc, char **argv) {
     readOOMScoreAdj();
     initServer();
     if (background || server.pidfile) createPidFile();
-    if (server.set_proc_title) redisSetProcTitle(argv[0]);
+    if (server.set_proc_title) redisSetProcTitle(NULL);
     redisAsciiArt();
     checkTcpBacklogSettings();
 
diff --git a/src/server.h b/src/server.h
index b72d7bd0d..349c887cb 100644
--- a/src/server.h
+++ b/src/server.h
@@ -115,6 +115,7 @@ typedef long long ustime_t; /* microsecond time type. */
 #define NET_ADDR_STR_LEN (NET_IP_STR_LEN+32) /* Must be enough for ip:port */
 #define CONFIG_BINDADDR_MAX 16
 #define CONFIG_MIN_RESERVED_FDS 32
+#define CONFIG_DEFAULT_PROC_TITLE_TEMPLATE "{title} {listen-addr} {server-mode}"
 
 #define ACTIVE_EXPIRE_CYCLE_SLOW 0
 #define ACTIVE_EXPIRE_CYCLE_FAST 1
@@ -1298,6 +1299,7 @@ struct redisServer {
     int supervised_mode;            /* See SUPERVISED_* */
     int daemonize;                  /* True if running as a daemon */
     int set_proc_title;             /* True if change proc title */
+    char *proc_title_template;      /* Process title template format */
     clientBufferLimitsConfig client_obuf_limits[CLIENT_TYPE_OBUF_COUNT];
     /* AOF persistence */
     int aof_enabled;                /* AOF configuration */
@@ -1749,7 +1751,8 @@ void getRandomBytes(unsigned char *p, size_t len);
 uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l);
 void exitFromChild(int retcode);
 size_t redisPopcount(void *s, long count);
-void redisSetProcTitle(char *title);
+int redisSetProcTitle(char *title);
+int validateProcTitleTemplate(const char *template);
 int redisCommunicateSystemd(const char *sd_notify_msg);
 void redisSetCpuAffinity(const char *cpulist);
 
diff --git a/tests/unit/other.tcl b/tests/unit/other.tcl
index d98dc1bd4..a35ac1752 100644
--- a/tests/unit/other.tcl
+++ b/tests/unit/other.tcl
@@ -321,3 +321,47 @@ start_server {tags {"other"}} {
         assert_match "*table size: 8192*" [r debug HTSTATS 9]
     }
 }
+
+proc read_proc_title {pid} {
+    set fd [open "/proc/$pid/cmdline" "r"]
+    set cmdline [read $fd 1024]
+    close $fd
+
+    return $cmdline
+}
+
+start_server {tags {"other"}} {
+    test {Process title set as expected} {
+        # Test only on Linux where it's easy to get cmdline without relying on tools.
+        # Skip valgrind as it messes up the arguments.
+        set os [exec uname]
+        if {$os == "Linux" && !$::valgrind} {
+            # Set a custom template
+            r config set "proc-title-template" "TEST {title} {listen-addr} {port} {tls-port} {unixsocket} {config-file}"
+            set cmdline [read_proc_title [srv 0 pid]]
+
+            assert_equal "TEST" [lindex $cmdline 0]
+            assert_match "*/redis-server" [lindex $cmdline 1]
+            
+            if {$::tls} {
+                set expect_port 0
+                set expect_tls_port [srv 0 port]
+            } else {
+                set expect_port [srv 0 port]
+                set expect_tls_port 0
+            }
+            set port [srv 0 port]
+
+            assert_equal "$::host:$port" [lindex $cmdline 2]
+            assert_equal $expect_port [lindex $cmdline 3]
+            assert_equal $expect_tls_port [lindex $cmdline 4]
+            assert_match "*/tests/tmp/server.*/socket" [lindex $cmdline 5]
+            assert_match "*/tests/tmp/redis.conf.*" [lindex $cmdline 6]
+
+            # Try setting a bad template
+            catch {r config set "proc-title-template" "{invalid-var}"} err
+            assert_match {*template format is invalid*} $err
+        }
+    }
+}
+

From 26301897d0f298446f6cc9c21e72484c4488ebe5 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 28 Jan 2021 20:49:46 +0200
Subject: [PATCH 1013/1098] Update CLIENT HELP regarding KILL options. (#8417)

* Indicate address can also be a unix socket path name.
* Document the LADDR option as well.
---
 src/networking.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 26e4017c5..893359ec5 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2439,8 +2439,10 @@ void clientCommand(client *c) {
 "    Kill connection made from <ip:port>.",
 "KILL <option> <value> [<option> <value> [...]]",
 "    Kill connections. Options are:",
-"    * ADDR <ip:port>",
-"      Kill connection made from <ip:port>",
+"    * ADDR (<ip:port>|<unixsocket>:0)",
+"      Kill connections made from the specified address",
+"    * LADDR (<ip:port>|<unixsocket>:0)",
+"      Kill connections made to specified local address",
 "    * TYPE (normal|master|replica|pubsub)",
 "      Kill connections by type.",
 "    * USER <username>",

From 0d18a1e85fdb9967419dc9f7a5905452d065c84a Mon Sep 17 00:00:00 2001
From: Allen Farris <farrisa@amazon.com>
Date: Thu, 28 Jan 2021 13:18:05 -0800
Subject: [PATCH 1014/1098] implement FAILOVER command (#8315)

Implement FAILOVER command, which coordinates failover
between the server and one of its replicas.
---
 src/networking.c               |   6 +-
 src/replication.c              | 336 ++++++++++++++++++++++++++++++++-
 src/server.c                   |  31 ++-
 src/server.h                   |  21 +++
 tests/integration/failover.tcl | 290 ++++++++++++++++++++++++++++
 tests/support/util.tcl         |  10 +-
 tests/test_helper.tcl          |   1 +
 7 files changed, 679 insertions(+), 16 deletions(-)
 create mode 100644 tests/integration/failover.tcl

diff --git a/src/networking.c b/src/networking.c
index 893359ec5..da611675c 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -2680,7 +2680,7 @@ NULL
                                                         c->argc == 4))
     {
         /* CLIENT PAUSE TIMEOUT [WRITE|ALL] */
-        long long duration;
+        mstime_t end;
         int type = CLIENT_PAUSE_ALL;
         if (c->argc == 4) {
             if (!strcasecmp(c->argv[3]->ptr,"write")) {
@@ -2694,9 +2694,9 @@ NULL
             }
         }
 
-        if (getTimeoutFromObjectOrReply(c,c->argv[2],&duration,
+        if (getTimeoutFromObjectOrReply(c,c->argv[2],&end,
             UNIT_MILLISECONDS) != C_OK) return;
-        pauseClients(duration, type);
+        pauseClients(end, type);
         addReply(c,shared.ok);
     } else if (!strcasecmp(c->argv[1]->ptr,"tracking") && c->argc >= 3) {
         /* CLIENT TRACKING (on|off) [REDIRECT <id>] [BCAST] [PREFIX first]
diff --git a/src/replication.c b/src/replication.c
index a9ec30e74..f23fcb6de 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -721,6 +721,36 @@ void syncCommand(client *c) {
     /* ignore SYNC if already slave or in monitor mode */
     if (c->flags & CLIENT_SLAVE) return;
 
+    /* Check if this is a failover request to a replica with the same replid and
+     * become a master if so. */
+    if (c->argc > 3 && !strcasecmp(c->argv[0]->ptr,"psync") && 
+        !strcasecmp(c->argv[3]->ptr,"failover"))
+    {
+        serverLog(LL_WARNING, "Failover request received for replid %s.",
+            (unsigned char *)c->argv[1]->ptr);
+        if (!server.masterhost) {
+            addReplyError(c, "PSYNC FAILOVER can't be sent to a master.");
+            return;
+        }
+
+        if (!strcasecmp(c->argv[1]->ptr,server.replid)) {
+            replicationUnsetMaster();
+            sds client = catClientInfoString(sdsempty(),c);
+            serverLog(LL_NOTICE,
+                "MASTER MODE enabled (failover request from '%s')",client);
+            sdsfree(client);
+        } else {
+            addReplyError(c, "PSYNC FAILOVER replid must match my replid.");
+            return;            
+        }
+    }
+
+    /* Don't let replicas sync with us while we're failing over */
+    if (server.failover_state != NO_FAILOVER) {
+        addReplyError(c,"-NOMASTERLINK Can't SYNC while failing over");
+        return;
+    }
+
     /* Refuse SYNC requests if we are a slave but the link with our master
      * is not ok... */
     if (server.masterhost && server.repl_state != REPL_STATE_CONNECTED) {
@@ -2031,8 +2061,15 @@ int slaveTryPartialResynchronization(connection *conn, int read_reply) {
             memcpy(psync_offset,"-1",3);
         }
 
-        /* Issue the PSYNC command */
-        reply = sendCommand(conn,"PSYNC",psync_replid,psync_offset,NULL);
+        /* Issue the PSYNC command, if this is a master with a failover in
+         * progress then send the failover argument to the replica to cause it
+         * to become a master */
+        if (server.failover_state == FAILOVER_IN_PROGRESS) {
+            reply = sendCommand(conn,"PSYNC",psync_replid,psync_offset,"FAILOVER",NULL);
+        } else {
+            reply = sendCommand(conn,"PSYNC",psync_replid,psync_offset,NULL);
+        }
+
         if (reply != NULL) {
             serverLog(LL_WARNING,"Unable to send PSYNC to master: %s",reply);
             sdsfree(reply);
@@ -2356,6 +2393,7 @@ void syncWithMaster(connection *conn) {
     if (server.repl_state == REPL_STATE_SEND_PSYNC) {
         if (slaveTryPartialResynchronization(conn,0) == PSYNC_WRITE_ERROR) {
             err = sdsnew("Write error sending the PSYNC command.");
+            abortFailover("Write error to failover target");
             goto write_error;
         }
         server.repl_state = REPL_STATE_RECEIVE_PSYNC_REPLY;
@@ -2373,6 +2411,18 @@ void syncWithMaster(connection *conn) {
     psync_result = slaveTryPartialResynchronization(conn,1);
     if (psync_result == PSYNC_WAIT_REPLY) return; /* Try again later... */
 
+    /* Check the status of the planned failover. We expect PSYNC_CONTINUE,
+     * but there is nothing technically wrong with a full resync which
+     * could happen in edge cases. */
+    if (server.failover_state == FAILOVER_IN_PROGRESS) {
+        if (psync_result == PSYNC_CONTINUE || psync_result == PSYNC_FULLRESYNC) {
+            clearFailoverState();
+        } else {
+            abortFailover("Failover target rejected psync request");
+            return;
+        }
+    }
+
     /* If the master is in an transient error, we should try to PSYNC
      * from scratch later, so go to the error path. This happens when
      * the server is loading the dataset or is not connected with its
@@ -2678,6 +2728,11 @@ void replicaofCommand(client *c) {
         return;
     }
 
+    if (server.failover_state != NO_FAILOVER) {
+        addReplyError(c,"REPLICAOF not allowed while failing over.");
+        return;
+    }
+
     /* The special host/port combination "NO" "ONE" turns the instance
      * into a master. Otherwise the new master address is set. */
     if (!strcasecmp(c->argv[1]->ptr,"no") &&
@@ -3211,6 +3266,10 @@ long long replicationGetSlaveOffset(void) {
 void replicationCron(void) {
     static long long replication_cron_loops = 0;
 
+    /* Check failover status first, to see if we need to start
+     * handling the failover. */
+    updateFailoverStatus();
+
     /* Non blocking connection timeout? */
     if (server.masterhost &&
         (server.repl_state == REPL_STATE_CONNECTING ||
@@ -3268,8 +3327,9 @@ void replicationCron(void) {
          * alter the replication offsets of master and slave, and will no longer
          * match the one stored into 'mf_master_offset' state. */
         int manual_failover_in_progress =
-            server.cluster_enabled &&
-            server.cluster->mf_end &&
+            ((server.cluster_enabled &&
+              server.cluster->mf_end) ||
+            server.failover_end_time) &&
             checkClientPauseTimeoutAndReturnIfPaused();
 
         if (!manual_failover_in_progress) {
@@ -3423,3 +3483,271 @@ void replicationStartPendingFork(void) {
         }
     }
 }
+
+/* Find replica at IP:PORT from replica list */
+static client *findReplica(char *host, int port) {
+    listIter li;
+    listNode *ln;
+    client *replica;
+
+    listRewind(server.slaves,&li);
+    while((ln = listNext(&li))) {
+        replica = ln->value;
+        char ip[NET_IP_STR_LEN], *replicaip = replica->slave_ip;
+
+        if (replicaip[0] == '\0') {
+            if (connPeerToString(replica->conn, ip, sizeof(ip), NULL) == -1)
+                continue;
+            replicaip = ip;
+        }
+
+        if (!strcasecmp(host, replicaip) &&
+                (port == replica->slave_listening_port))
+            return replica;
+    }
+
+    return NULL;
+}
+
+const char *getFailoverStateString() {
+    switch(server.failover_state) {
+        case NO_FAILOVER: return "no-failover";
+        case FAILOVER_IN_PROGRESS: return "failover-in-progress";
+        case FAILOVER_WAIT_FOR_SYNC: return "waiting-for-sync";
+        default: return "unknown";
+    }
+}
+
+/* Resets the internal failover configuration, this needs
+ * to be called after a failover either succeeds or fails
+ * as it includes the client unpause. */
+void clearFailoverState() {
+    server.failover_end_time = 0;
+    server.force_failover = 0;
+    zfree(server.target_replica_host);
+    server.target_replica_host = NULL;
+    server.target_replica_port = 0;
+    server.failover_state = NO_FAILOVER;
+    unpauseClients();
+}
+
+/* Abort an ongoing failover if one is going on. */
+void abortFailover(const char *err) {
+    if (server.failover_state == NO_FAILOVER) return;
+
+    if (server.target_replica_host) {
+        serverLog(LL_NOTICE,"FAILOVER to %s:%d aborted: %s",
+            server.target_replica_host,server.target_replica_port,err);  
+    } else {
+        serverLog(LL_NOTICE,"FAILOVER to any replica aborted: %s",err);  
+    }
+    if (server.failover_state == FAILOVER_IN_PROGRESS) {
+        replicationUnsetMaster();
+    }
+    clearFailoverState();
+}
+
+/* 
+ * FAILOVER [TO <HOST> <IP> [FORCE]] [ABORT] [TIMEOUT <timeout>] 
+ * 
+ * This command will coordinate a failover between the master and one
+ * of its replicas. The happy path contains the following steps:
+ * 1) The master will initiate a client pause write, to stop replication
+ * traffic.
+ * 2) The master will periodically check if any of its replicas has
+ * consumed the entire replication stream through acks. 
+ * 3) Once any replica has caught up, the master will itself become a replica.
+ * 4) The master will send a PSYNC FAILOVER request to the target replica, which
+ * if accepted will cause the replica to become the new master and start a sync.
+ * 
+ * FAILOVER ABORT is the only way to abort a failover command, as replicaof
+ * will be disabled. This may be needed if the failover is unable to progress. 
+ * 
+ * The optional arguments [TO <HOST> <IP>] allows designating a specific replica
+ * to be failed over to.
+ * 
+ * FORCE flag indicates that even if the target replica is not caught up,
+ * failover to it anyway. This must be specified with a timeout and a target
+ * HOST and IP.
+ * 
+ * TIMEOUT <timeout> indicates how long should the primary wait for 
+ * a replica to sync up before aborting. If not specified, the failover
+ * will attempt forever and must be manually aborted.
+ */
+void failoverCommand(client *c) {
+    if (server.cluster_enabled) {
+        addReplyError(c,"FAILOVER not allowed in cluster mode. "
+                        "Use CLUSTER FAILOVER command instead.");
+        return;
+    }
+    
+    /* Handle special case for abort */
+    if ((c->argc == 2) && !strcasecmp(c->argv[1]->ptr,"abort")) {
+        if (server.failover_state == NO_FAILOVER) {
+            addReplyError(c, "No failover in progress.");
+            return;
+        }
+
+        abortFailover("Failover manually aborted");
+        addReply(c,shared.ok);
+        return;
+    }
+
+    long timeout_in_ms = 0;
+    int force_flag = 0;
+    long port = 0;
+    char *host = NULL;
+
+    /* Parse the command for syntax and arguments. */
+    for (int j = 1; j < c->argc; j++) {
+        if (!strcasecmp(c->argv[j]->ptr,"timeout") && (j + 1 < c->argc) &&
+            timeout_in_ms == 0)
+        {
+            if (getLongFromObjectOrReply(c,c->argv[j + 1],
+                        &timeout_in_ms,NULL) != C_OK) return;
+            if (timeout_in_ms <= 0) {
+                addReplyError(c,"FAILOVER timeout must be greater than 0");
+                return;
+            }
+            j++;
+        } else if (!strcasecmp(c->argv[j]->ptr,"to") && (j + 2 < c->argc) &&
+            !host) 
+        {
+            if (getLongFromObjectOrReply(c,c->argv[j + 2],&port,NULL) != C_OK)
+                return;
+            host = c->argv[j + 1]->ptr;
+            j += 2;
+        } else if (!strcasecmp(c->argv[j]->ptr,"force") && !force_flag) {
+            force_flag = 1;
+        } else {
+            addReplyErrorObject(c,shared.syntaxerr);
+            return;
+        }
+    }
+
+    if (server.failover_state != NO_FAILOVER) {
+        addReplyError(c,"FAILOVER already in progress.");
+        return;
+    }
+
+    if (server.masterhost) {
+        addReplyError(c,"FAILOVER is not valid when server is a replica.");
+        return;
+    }
+
+    if (listLength(server.slaves) == 0) {
+        addReplyError(c,"FAILOVER requires connected replicas.");
+        return; 
+    }
+
+    if (force_flag && (!timeout_in_ms || !host)) {
+        addReplyError(c,"FAILOVER with force option requires both a timeout "
+            "and target HOST and IP.");
+        return;     
+    }
+
+    /* If a replica address was provided, validate that it is connected. */
+    if (host) {
+        client *replica = findReplica(host, port);
+
+        if (replica == NULL) {
+            addReplyError(c,"FAILOVER target HOST and IP is not "
+                            "a replica.");
+            return;
+        }
+
+        /* Check if requested replica is online */
+        if (replica->replstate != SLAVE_STATE_ONLINE) {
+            addReplyError(c,"FAILOVER target replica is not online.");
+            return;
+        }
+
+        server.target_replica_host = zstrdup(host);
+        server.target_replica_port = port;
+        serverLog(LL_NOTICE,"FAILOVER requested to %s:%ld.",host,port);
+    } else {
+        serverLog(LL_NOTICE,"FAILOVER requested to any replica.");
+    }
+
+    mstime_t now = mstime();
+    if (timeout_in_ms) {
+        server.failover_end_time = now + timeout_in_ms;
+    }
+    
+    server.force_failover = force_flag;
+    server.failover_state = FAILOVER_WAIT_FOR_SYNC;
+    /* Cluster failover will unpause eventually */
+    pauseClients(LLONG_MAX,CLIENT_PAUSE_WRITE);
+    addReply(c,shared.ok);
+}
+
+/* Failover cron function, checks coordinated failover state. 
+ *
+ * Implementation note: The current implementation calls replicationSetMaster()
+ * to start the failover request, this has some unintended side effects if the
+ * failover doesn't work like blocked clients will be unblocked and replicas will
+ * be disconnected. This could be optimized further.
+ */
+void updateFailoverStatus(void) {
+    if (server.failover_state != FAILOVER_WAIT_FOR_SYNC) return;
+    mstime_t now = server.mstime;
+
+    /* Check if failover operation has timed out */
+    if (server.failover_end_time && server.failover_end_time <= now) {
+        if (server.force_failover) {
+            serverLog(LL_NOTICE,
+                "FAILOVER to %s:%d time out exceeded, failing over.",
+                server.target_replica_host, server.target_replica_port);
+            server.failover_state = FAILOVER_IN_PROGRESS;
+            /* If timeout has expired force a failover if requested. */
+            replicationSetMaster(server.target_replica_host,
+                server.target_replica_port);
+            return;
+        } else {
+            /* Force was not requested, so timeout. */
+            abortFailover("Replica never caught up before timeout");
+            return;
+        }
+    }
+
+    /* Check to see if the replica has caught up so failover can start */
+    client *replica = NULL;
+    if (server.target_replica_host) {
+        replica = findReplica(server.target_replica_host, 
+            server.target_replica_port);
+    } else {
+        listIter li;
+        listNode *ln;
+
+        listRewind(server.slaves,&li);
+        /* Find any replica that has matched our repl_offset */
+        while((ln = listNext(&li))) {
+            replica = ln->value;
+            if (replica->repl_ack_off == server.master_repl_offset) {
+                char ip[NET_IP_STR_LEN], *replicaip = replica->slave_ip;
+
+                if (replicaip[0] == '\0') {
+                    if (connPeerToString(replica->conn,ip,sizeof(ip),NULL) == -1)
+                        continue;
+                    replicaip = ip;
+                }
+
+                /* We are now failing over to this specific node */
+                server.target_replica_host = zstrdup(replicaip);
+                server.target_replica_port = replica->slave_listening_port;
+                break;
+            }
+        }
+    }
+
+    /* We've found a replica that is caught up */
+    if (replica && (replica->repl_ack_off == server.master_repl_offset)) {
+        server.failover_state = FAILOVER_IN_PROGRESS;
+        serverLog(LL_NOTICE,
+                "Failover target %s:%d is synced, failing over.",
+                server.target_replica_host, server.target_replica_port);
+        /* Designated replica is caught up, failover to it. */
+        replicationSetMaster(server.target_replica_host,
+            server.target_replica_port);
+    }
+}
diff --git a/src/server.c b/src/server.c
index 8b8da8c89..1166c4b3e 100644
--- a/src/server.c
+++ b/src/server.c
@@ -752,7 +752,7 @@ struct redisCommand redisCommandTable[] = {
      "admin no-script",
      0,NULL,0,0,0,0,0,0},
 
-    {"psync",syncCommand,3,
+    {"psync",syncCommand,-3,
      "admin no-script",
      0,NULL,0,0,0,0,0,0},
 
@@ -1092,6 +1092,10 @@ struct redisCommand redisCommandTable[] = {
 
     {"reset",resetCommand,1,
      "no-script ok-stale ok-loading fast @connection",
+     0,NULL,0,0,0,0,0,0},
+
+    {"failover",failoverCommand,-1,
+     "admin no-script ok-stale",
      0,NULL,0,0,0,0,0,0}
 };
 
@@ -2185,8 +2189,15 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
     checkClientPauseTimeoutAndReturnIfPaused();
 
     /* Replication cron function -- used to reconnect to master,
-     * detect transfer failures, start background RDB transfers and so forth. */
-    run_with_period(1000) replicationCron();
+     * detect transfer failures, start background RDB transfers and so forth. 
+     * 
+     * If Redis is trying to failover then run the replication cron faster so
+     * progress on the handshake happens more quickly. */
+    if (server.failover_state != NO_FAILOVER) {
+        run_with_period(100) replicationCron();
+    } else {
+        run_with_period(1000) replicationCron();
+    }
 
     /* Run the Redis Cluster cron. */
     run_with_period(100) {
@@ -2397,6 +2408,11 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
         server.get_ack_from_slaves = 0;
     }
 
+    /* We may have recieved updates from clients about their current offset. NOTE:
+     * this can't be done where the ACK is recieved since failover will disconnect 
+     * our clients. */
+    updateFailoverStatus();
+
     /* Send the invalidation messages to clients participating to the
      * client side caching protocol in broadcasting (BCAST) mode. */
     trackingBroadcastInvalidationMessages();
@@ -2651,6 +2667,13 @@ void initServerConfig(void) {
     server.repl_backlog_off = 0;
     server.repl_no_slaves_since = time(NULL);
 
+    /* Failover related */
+    server.failover_end_time = 0;
+    server.force_failover = 0;
+    server.target_replica_host = NULL;
+    server.target_replica_port = 0;
+    server.failover_state = NO_FAILOVER;
+
     /* Client output buffer limits */
     for (j = 0; j < CLIENT_TYPE_OBUF_COUNT; j++)
         server.client_obuf_limits[j] = clientBufferLimitsDefaults[j];
@@ -4991,6 +5014,7 @@ sds genRedisInfoString(const char *section) {
             }
         }
         info = sdscatprintf(info,
+            "master_failover_state:%s\r\n"
             "master_replid:%s\r\n"
             "master_replid2:%s\r\n"
             "master_repl_offset:%lld\r\n"
@@ -4999,6 +5023,7 @@ sds genRedisInfoString(const char *section) {
             "repl_backlog_size:%lld\r\n"
             "repl_backlog_first_byte_offset:%lld\r\n"
             "repl_backlog_histlen:%lld\r\n",
+            getFailoverStateString(),
             server.replid,
             server.replid2,
             server.master_repl_offset,
diff --git a/src/server.h b/src/server.h
index 349c887cb..02ba8f7b9 100644
--- a/src/server.h
+++ b/src/server.h
@@ -320,6 +320,14 @@ typedef enum {
     REPL_STATE_CONNECTED,       /* Connected to master */
 } repl_state;
 
+/* The state of an in progress coordinated failover */
+typedef enum {
+    NO_FAILOVER = 0,        /* No failover in progress */
+    FAILOVER_WAIT_FOR_SYNC, /* Waiting for target replica to catch up */
+    FAILOVER_IN_PROGRESS    /* Waiting for target replica to accept
+                             * PSYNC FAILOVER request. */
+} failover_state;
+
 /* State of slaves from the POV of the master. Used in client->replstate.
  * In SEND_BULK and ONLINE state the slave receives new updates
  * in its output queue. In the WAIT_BGSAVE states instead the server is waiting
@@ -1577,6 +1585,14 @@ struct redisServer {
     char *bgsave_cpulist; /* cpu affinity list of bgsave process. */
     /* Sentinel config */
     struct sentinelConfig *sentinel_config; /* sentinel config to load at startup time. */
+    /* Coordinate failover info */
+    mstime_t failover_end_time; /* Deadline for failover command. */
+    int force_failover; /* If true then failover will be foreced at the
+                         * deadline, otherwise failover is aborted. */
+    char *target_replica_host; /* Failover target host. If null during a
+                                * failover then any replica can be used. */
+    int target_replica_port; /* Failover target port */
+    int failover_state; /* Failover state */
 };
 
 typedef struct pubsubPattern {
@@ -1997,6 +2013,10 @@ void feedReplicationBacklog(void *ptr, size_t len);
 void showLatestBacklog(void);
 void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask);
 void rdbPipeWriteHandlerConnRemoved(struct connection *conn);
+void clearFailoverState(void);
+void updateFailoverStatus(void);
+void abortFailover(const char *err);
+const char *getFailoverStateString();
 
 /* Generic persistence functions */
 void startLoadingFile(FILE* fp, char* filename, int rdbflags);
@@ -2637,6 +2657,7 @@ void lolwutCommand(client *c);
 void aclCommand(client *c);
 void stralgoCommand(client *c);
 void resetCommand(client *c);
+void failoverCommand(client *c);
 
 #if defined(__GNUC__)
 void *calloc(size_t count, size_t size) __attribute__ ((deprecated));
diff --git a/tests/integration/failover.tcl b/tests/integration/failover.tcl
new file mode 100644
index 000000000..c6818700d
--- /dev/null
+++ b/tests/integration/failover.tcl
@@ -0,0 +1,290 @@
+start_server {tags {"failover"}} {
+start_server {} {
+start_server {} {
+    set node_0 [srv 0 client]
+    set node_0_host [srv 0 host]
+    set node_0_port [srv 0 port]
+    set node_0_pid [srv 0 pid]
+
+    set node_1 [srv -1 client]
+    set node_1_host [srv -1 host]
+    set node_1_port [srv -1 port]
+    set node_1_pid [srv -1 pid]
+
+    set node_2 [srv -2 client]
+    set node_2_host [srv -2 host]
+    set node_2_port [srv -2 port]
+    set node_2_pid [srv -2 pid]
+
+    proc assert_digests_match {n1 n2 n3} {
+        assert_equal [$n1 debug digest] [$n2 debug digest]
+        assert_equal [$n2 debug digest] [$n3 debug digest]
+    }
+
+    test {failover command fails without connected replica} {
+        catch { $node_0 failover to $node_1_host $node_1_port } err
+        if {! [string match "ERR*" $err]} {
+            fail "failover command succeeded when replica not connected"
+        }
+    }
+
+    test {setup replication for following tests} {
+        $node_1 replicaof $node_0_host $node_0_port
+        $node_2 replicaof $node_0_host $node_0_port
+        wait_for_sync $node_1
+        wait_for_sync $node_2
+    }
+
+    test {failover command fails with invalid host} {
+        catch { $node_0 failover to invalidhost $node_1_port } err
+        assert_match "ERR*" $err
+    }
+
+    test {failover command fails with invalid port} {
+        catch { $node_0 failover to $node_1_host invalidport } err
+        assert_match "ERR*" $err
+    }
+
+    test {failover command fails with just force and timeout} {
+        catch { $node_0 FAILOVER FORCE TIMEOUT 100} err
+        assert_match "ERR*" $err
+    }
+
+    test {failover command fails when sent to a replica} {
+        catch { $node_1 failover to $node_1_host $node_1_port } err
+        assert_match "ERR*" $err
+    }
+
+    test {failover command fails with force without timeout} {
+        catch { $node_0 failover to $node_1_host $node_1_port FORCE } err
+        assert_match "ERR*" $err
+    }
+
+    test {failover command to specific replica works} {
+        set initial_psyncs [s -1 sync_partial_ok]
+        set initial_syncs [s -1 sync_full]
+
+        # Generate a delta between primary and replica
+        set load_handler [start_write_load $node_0_host $node_0_port 5]
+        exec kill -SIGSTOP [srv -1 pid]
+        wait_for_condition 50 100 {
+            [s 0 total_commands_processed] > 100
+        } else {
+            fail "Node 0 did not accept writes"
+        }
+        exec kill -SIGCONT [srv -1 pid]
+
+        # Execute the failover
+        $node_0 failover to $node_1_host $node_1_port
+
+        # Wait for failover to end
+        wait_for_condition 50 100 {
+            [s 0 master_failover_state] == "no-failover"
+        } else {
+            fail "Failover from node 0 to node 1 did not finish"
+        }
+        stop_write_load $load_handler
+        $node_2 replicaof $node_1_host $node_1_port
+        wait_for_sync $node_0
+        wait_for_sync $node_2
+
+        assert_match *slave* [$node_0 role]
+        assert_match *master* [$node_1 role]
+        assert_match *slave* [$node_2 role]
+
+        # We should accept psyncs from both nodes
+        assert_equal [expr [s -1 sync_partial_ok] - $initial_psyncs] 2
+        assert_equal [expr [s -1 sync_full] - $initial_psyncs] 0
+        assert_digests_match $node_0 $node_1 $node_2
+    }
+
+    test {failover command to any replica works} {
+        set initial_psyncs [s -2 sync_partial_ok]
+        set initial_syncs [s -2 sync_full]
+
+        wait_for_ofs_sync $node_1 $node_2
+        # We stop node 0 to and make sure node 2 is selected
+        exec kill -SIGSTOP $node_0_pid
+        $node_1 set CASE 1
+        $node_1 FAILOVER
+
+        # Wait for failover to end
+        wait_for_condition 50 100 {
+            [s -1 master_failover_state] == "no-failover"
+        } else {
+            fail "Failover from node 1 to node 2 did not finish"
+        }
+        exec kill -SIGCONT $node_0_pid
+        $node_0 replicaof $node_2_host $node_2_port
+
+        wait_for_sync $node_0
+        wait_for_sync $node_1
+
+        assert_match *slave* [$node_0 role]
+        assert_match *slave* [$node_1 role]
+        assert_match *master* [$node_2 role]
+
+        # We should accept Psyncs from both nodes
+        assert_equal [expr [s -2 sync_partial_ok] - $initial_psyncs] 2
+        assert_equal [expr [s -1 sync_full] - $initial_psyncs] 0
+        assert_digests_match $node_0 $node_1 $node_2
+    }
+
+    test {failover to a replica with force works} {
+        set initial_psyncs [s 0 sync_partial_ok]
+        set initial_syncs [s 0 sync_full]
+
+        exec kill -SIGSTOP $node_0_pid
+        # node 0 will never acknowledge this write
+        $node_2 set case 2
+        $node_2 failover to $node_0_host $node_0_port TIMEOUT 100 FORCE
+
+        # Wait for node 0 to give up on sync attempt and start failover
+        wait_for_condition 50 100 {
+            [s -2 master_failover_state] == "failover-in-progress"
+        } else {
+            fail "Failover from node 2 to node 0 did not timeout"
+        }
+
+        # Quick check that everyone is a replica, we never want a 
+        # state where there are two masters.
+        assert_match *slave* [$node_1 role]
+        assert_match *slave* [$node_2 role]
+
+        exec kill -SIGCONT $node_0_pid
+
+        # Wait for failover to end
+        wait_for_condition 50 100 {
+            [s -2 master_failover_state] == "no-failover"
+        } else {
+            fail "Failover from node 2 to node 0 did not finish"
+        }
+        $node_1 replicaof $node_0_host $node_0_port
+
+        wait_for_sync $node_1
+        wait_for_sync $node_2
+
+        assert_match *master* [$node_0 role]
+        assert_match *slave* [$node_1 role]
+        assert_match *slave* [$node_2 role]
+
+        assert_equal [count_log_message -2 "time out exceeded, failing over."] 1
+
+        # We should accept both psyncs, although this is the condition we might not
+        # since we didn't catch up.
+        assert_equal [expr [s 0 sync_partial_ok] - $initial_psyncs] 2
+        assert_equal [expr [s 0 sync_full] - $initial_syncs] 0
+        assert_digests_match $node_0 $node_1 $node_2
+    }
+
+    test {failover with timeout aborts if replica never catches up} {
+        set initial_psyncs [s 0 sync_partial_ok]
+        set initial_syncs [s 0 sync_full]
+
+        # Stop replica so it never catches up
+        exec kill -SIGSTOP [srv -1 pid]
+        $node_0 SET CASE 1
+        
+        $node_0 failover to [srv -1 host] [srv -1 port] TIMEOUT 500
+        # Wait for failover to end
+        wait_for_condition 50 20 {
+            [s 0 master_failover_state] == "no-failover"
+        } else {
+            fail "Failover from node_0 to replica did not finish"
+        }
+
+        exec kill -SIGCONT [srv -1 pid]
+
+        # We need to make sure the nodes actually sync back up
+        wait_for_ofs_sync $node_0 $node_1
+        wait_for_ofs_sync $node_0 $node_2
+
+        assert_match *master* [$node_0 role]
+        assert_match *slave* [$node_1 role]
+        assert_match *slave* [$node_2 role]
+
+        # Since we never caught up, there should be no syncs
+        assert_equal [expr [s 0 sync_partial_ok] - $initial_psyncs] 0
+        assert_equal [expr [s 0 sync_full] - $initial_syncs] 0
+        assert_digests_match $node_0 $node_1 $node_2
+    }
+
+    test {failovers can be aborted} {
+        set initial_psyncs [s 0 sync_partial_ok]
+        set initial_syncs [s 0 sync_full]
+    
+        # Stop replica so it never catches up
+        exec kill -SIGSTOP [srv -1 pid]
+        $node_0 SET CASE 2
+        
+        $node_0 failover to [srv -1 host] [srv -1 port] TIMEOUT 60000
+        assert_match [s 0 master_failover_state] "waiting-for-sync"
+
+        # Sanity check that read commands are still accepted
+        $node_0 GET CASE
+
+        $node_0 failover abort
+        assert_match [s 0 master_failover_state] "no-failover"
+
+        exec kill -SIGCONT [srv -1 pid]
+
+        # Just make sure everything is still synced
+        wait_for_ofs_sync $node_0 $node_1
+        wait_for_ofs_sync $node_0 $node_2
+
+        assert_match *master* [$node_0 role]
+        assert_match *slave* [$node_1 role]
+        assert_match *slave* [$node_2 role]
+
+        # Since we never caught up, there should be no syncs
+        assert_equal [expr [s 0 sync_partial_ok] - $initial_psyncs] 0
+        assert_equal [expr [s 0 sync_full] - $initial_syncs] 0
+        assert_digests_match $node_0 $node_1 $node_2
+    }
+
+    test {failover aborts if target rejects sync request} {
+        set initial_psyncs [s 0 sync_partial_ok]
+        set initial_syncs [s 0 sync_full]
+
+        # We block psync, so the failover will fail
+        $node_1 acl setuser default -psync
+
+        # We pause the target long enough to send a write command
+        # during the pause. This write will not be interrupted.
+        exec kill -SIGSTOP [srv -1 pid]
+        set rd [redis_deferring_client]
+        $rd SET FOO BAR
+        $node_0 failover to $node_1_host $node_1_port
+        exec kill -SIGCONT [srv -1 pid]
+
+        # Wait for failover to end
+        wait_for_condition 50 100 {
+            [s 0 master_failover_state] == "no-failover"
+        } else {
+            fail "Failover from node_0 to replica did not finish"
+        }
+
+        assert_equal [$rd read] "OK"
+        $rd close
+
+        # restore access to psync
+        $node_1 acl setuser default +psync
+
+        # We need to make sure the nodes actually sync back up
+        wait_for_sync $node_1
+        wait_for_sync $node_2
+
+        assert_match *master* [$node_0 role]
+        assert_match *slave* [$node_1 role]
+        assert_match *slave* [$node_2 role]
+
+        # We will cycle all of our replicas here and force a psync.
+        assert_equal [expr [s 0 sync_partial_ok] - $initial_psyncs] 2
+        assert_equal [expr [s 0 sync_full] - $initial_syncs] 0
+
+        assert_equal [count_log_message 0 "Failover target rejected psync request"] 1
+        assert_digests_match $node_0 $node_1 $node_2
+    }
+}
+}
+}
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index 86f2753c2..e69395994 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -86,12 +86,10 @@ proc waitForBgrewriteaof r {
 }
 
 proc wait_for_sync r {
-    while 1 {
-        if {[status $r master_link_status] eq "down"} {
-            after 10
-        } else {
-            break
-        }
+    wait_for_condition 50 100 {
+        [status $r master_link_status] eq "up"
+    } else {
+        fail "replica didn't sync in time"
     }
 }
 
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index f947c024a..2b7854780 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -52,6 +52,7 @@ set ::all_tests {
     integration/psync2
     integration/psync2-reg
     integration/psync2-pingoff
+    integration/failover
     integration/redis-cli
     integration/redis-benchmark
     unit/pubsub

From 49b36633324074edd3e06c334d23562edc49159d Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <276441700@qq.com>
Date: Fri, 29 Jan 2021 14:35:10 +0800
Subject: [PATCH 1015/1098] AOF: recover from last write error after turn on
 appendonly again (#8030)

The key point is how to recover from last AOF write error, for example:

1. start redis with appendonly yes, and append some write commands

2. short write or something else error happen, `server.aof_last_write_status` changed to `C_ERR`, now redis doesn't accept write commands

3. execute `CONFIG SET appendonly no` to avoid the above problem, now redis can accept write commands again

4. disk error resolved, and execute `CONFIG SET appendonly yes` to reopen AOF, but `server.aof_last_write_status` cannot be changed to `C_OK` (if background aof rewrite run less then 1 second, it will free `server.aof_buf` and then serverCron cannot fix `aof_last_write_status`), then redis cannot accept write commands forever.

This PR use a simple way to fix it:

1. just free `server.aof_buf` when stop appendonly to save memory, if error happens in `flushAppendOnlyFile(1)`, the `server.aof_buf` may contains some data which has not be written to aof, I think we can ignore it because we turn off the appendonly.

2. reset fsync status after stop appendonly and call `flushAppendOnlyFile` only when `aof_state` is ON

3. reset `server.last_write_status` when reopen aof to accept write commands
---
 src/aof.c    | 19 +++++++++++++++++--
 src/server.c |  8 +++++---
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index f07f7ca2e..6753e8bcc 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -235,6 +235,8 @@ void stopAppendOnly(void) {
     serverAssert(server.aof_state != AOF_OFF);
     flushAppendOnlyFile(1);
     redis_fsync(server.aof_fd);
+    server.aof_fsync_offset = server.aof_current_size;
+    server.aof_last_fsync = server.unixtime;
     close(server.aof_fd);
 
     server.aof_fd = -1;
@@ -242,6 +244,8 @@ void stopAppendOnly(void) {
     server.aof_state = AOF_OFF;
     server.aof_rewrite_scheduled = 0;
     killAppendOnlyChild();
+    sdsfree(server.aof_buf);
+    server.aof_buf = sdsempty();
 }
 
 /* Called when the user switches from "appendonly no" to "appendonly yes"
@@ -285,6 +289,12 @@ int startAppendOnly(void) {
     server.aof_state = AOF_WAIT_REWRITE;
     server.aof_last_fsync = server.unixtime;
     server.aof_fd = newfd;
+
+    /* If AOF was in error state, we just ignore it and log the event. */
+    if (server.aof_last_write_status == C_ERR) {
+        serverLog(LL_WARNING,"AOF reopen, just ignore the last error.");
+        server.aof_last_write_status = C_OK;
+    }
     return C_OK;
 }
 
@@ -1855,18 +1865,22 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
             close(newfd);
             goto cleanup;
         }
+        latencyEndMonitor(latency);
+        latencyAddSampleIfNeeded("aof-rewrite-diff-write",latency);
+  
         if (server.aof_fsync == AOF_FSYNC_EVERYSEC) {
             aof_background_fsync(newfd);
         } else if (server.aof_fsync == AOF_FSYNC_ALWAYS) {
+            latencyStartMonitor(latency);
             if (redis_fsync(newfd) == -1) {
                 serverLog(LL_WARNING,
                     "Error trying to fsync the parent diff to the rewritten AOF: %s", strerror(errno));
                 close(newfd);
                 goto cleanup;
             }
+            latencyEndMonitor(latency);
+            latencyAddSampleIfNeeded("aof-rewrite-done-fsync",latency);
         }
-        latencyEndMonitor(latency);
-        latencyAddSampleIfNeeded("aof-rewrite-diff-write",latency);
 
         serverLog(LL_NOTICE,
             "Residual parent diff successfully flushed to the rewritten AOF (%.2f MB)", (double) aofRewriteBufferSize() / (1024*1024));
@@ -1938,6 +1952,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
             aofUpdateCurrentSize();
             server.aof_rewrite_base_size = server.aof_current_size;
             server.aof_fsync_offset = server.aof_current_size;
+            server.aof_last_fsync = server.unixtime;
 
             /* Clear regular AOF buffer since its contents was just written to
              * the new AOF from the background rewrite buffer. */
diff --git a/src/server.c b/src/server.c
index 1166c4b3e..0a12e2e31 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2174,14 +2174,15 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
 
     /* AOF postponed flush: Try at every cron cycle if the slow fsync
      * completed. */
-    if (server.aof_flush_postponed_start) flushAppendOnlyFile(0);
+    if (server.aof_state == AOF_ON && server.aof_flush_postponed_start)
+        flushAppendOnlyFile(0);
 
     /* AOF write errors: in this case we have a buffer to flush as well and
      * clear the AOF error in case of success to make the DB writable again,
      * however to try every second is enough in case of 'hz' is set to
      * a higher frequency. */
     run_with_period(1000) {
-        if (server.aof_last_write_status == C_ERR)
+        if (server.aof_state == AOF_ON && server.aof_last_write_status == C_ERR)
             flushAppendOnlyFile(0);
     }
 
@@ -2418,7 +2419,8 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     trackingBroadcastInvalidationMessages();
 
     /* Write the AOF buffer on disk */
-    flushAppendOnlyFile(0);
+    if (server.aof_state == AOF_ON)
+        flushAppendOnlyFile(0);
 
     /* Handle writes with pending output buffers. */
     handleClientsWithPendingWritesUsingThreads();

From b9a0500f16d0cd016398133cc7ac256ad927b679 Mon Sep 17 00:00:00 2001
From: Yang Bodong <bodong.ybd@alibaba-inc.com>
Date: Fri, 29 Jan 2021 16:47:28 +0800
Subject: [PATCH 1016/1098] Add HRANDFIELD and ZRANDMEMBER. improvements to
 SRANDMEMBER (#8297)

New commands:
`HRANDFIELD [<count> [WITHVALUES]]`
`ZRANDMEMBER [<count> [WITHSCORES]]`
Algorithms are similar to the one in SRANDMEMBER.

Both return a simple bulk response when no arguments are given, and an array otherwise.
In case values/scores are requested, RESP2 returns a long array, and RESP3 a nested array.
note: in all 3 commands, the only option that also provides random order is the one with negative count.

Changes to SRANDMEMBER
* Optimization when count is 1, we can use the more efficient algorithm of non-unique random
* optimization: work with sds strings rather than robj

Other changes:
* zzlGetScore: when zset needs to convert string to double, we use safer memcpy (in
  case the buffer is too small)
* Solve a "bug" in SRANDMEMBER test: it intended to test a positive count (case 3 or
  case 4) and by accident used a negative count

Co-authored-by: xinluton <xinluton@qq.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/server.c             |  19 +++
 src/server.h             |   3 +
 src/t_hash.c             | 253 ++++++++++++++++++++++++++++++++++++
 src/t_set.c              |  36 +++---
 src/t_zset.c             | 269 ++++++++++++++++++++++++++++++++++++++-
 src/ziplist.c            |  77 +++++++++++
 src/ziplist.h            |  11 ++
 tests/support/util.tcl   |   4 +-
 tests/unit/type/hash.tcl | 175 +++++++++++++++++++++++++
 tests/unit/type/set.tcl  |   2 +-
 tests/unit/type/zset.tcl | 176 +++++++++++++++++++++++++
 11 files changed, 1002 insertions(+), 23 deletions(-)

diff --git a/src/server.c b/src/server.c
index 0a12e2e31..79d6d813d 100644
--- a/src/server.c
+++ b/src/server.c
@@ -555,6 +555,10 @@ struct redisCommand redisCommandTable[] = {
      "write no-script fast @sortedset @blocking",
      0,NULL,1,-2,1,0,0,0},
 
+    {"zrandmember",zrandmemberCommand,-2,
+     "read-only random @sortedset",
+     0,NULL,1,1,1,0,0,0},
+
     {"hset",hsetCommand,-4,
      "write use-memory fast @hash",
      0,NULL,1,1,1,0,0,0},
@@ -611,6 +615,10 @@ struct redisCommand redisCommandTable[] = {
      "read-only fast @hash",
      0,NULL,1,1,1,0,0,0},
 
+    {"hrandfield",hrandfieldCommand,-2,
+     "read-only random @hash",
+     0,NULL,1,1,1,0,0,0},
+
     {"hscan",hscanCommand,-3,
      "read-only random @hash",
      0,NULL,1,1,1,0,0,0},
@@ -1456,6 +1464,17 @@ dictType hashDictType = {
     NULL                        /* allow to expand */
 };
 
+/* Dict type without destructor */
+dictType sdsReplyDictType = {
+    dictSdsHash,                /* hash function */
+    NULL,                       /* key dup */
+    NULL,                       /* val dup */
+    dictSdsKeyCompare,          /* key compare */
+    NULL,                       /* key destructor */
+    NULL,                       /* val destructor */
+    NULL                        /* allow to expand */
+};
+
 /* Keylist hash table type has unencoded redis objects as keys and
  * lists as values. It's used for blocking operations (BLPOP) and to
  * map swapped keys to a list of clients waiting for this keys to be loaded. */
diff --git a/src/server.h b/src/server.h
index 02ba8f7b9..1506c12a8 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1718,6 +1718,7 @@ extern dictType hashDictType;
 extern dictType replScriptCacheDictType;
 extern dictType dbExpiresDictType;
 extern dictType modulesDictType;
+extern dictType sdsReplyDictType;
 
 /*-----------------------------------------------------------------------------
  * Functions prototypes
@@ -2555,6 +2556,7 @@ void zpopminCommand(client *c);
 void zpopmaxCommand(client *c);
 void bzpopminCommand(client *c);
 void bzpopmaxCommand(client *c);
+void zrandmemberCommand(client *c);
 void multiCommand(client *c);
 void execCommand(client *c);
 void discardCommand(client *c);
@@ -2588,6 +2590,7 @@ void hvalsCommand(client *c);
 void hgetallCommand(client *c);
 void hexistsCommand(client *c);
 void hscanCommand(client *c);
+void hrandfieldCommand(client *c);
 void configCommand(client *c);
 void hincrbyCommand(client *c);
 void hincrbyfloatCommand(client *c);
diff --git a/src/t_hash.c b/src/t_hash.c
index 51c7d6758..9f7540a72 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -598,6 +598,42 @@ int hashZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep) {
     return ret;
 }
 
+/* Create a new sds string from the ziplist entry. */
+sds hashSdsFromZiplistEntry(ziplistEntry *e) {
+    return e->sval ? sdsnewlen(e->sval, e->slen) : sdsfromlonglong(e->lval);
+}
+
+/* Reply with bulk string from the ziplist entry. */
+void hashReplyFromZiplistEntry(client *c, ziplistEntry *e) {
+    if (e->sval)
+        addReplyBulkCBuffer(c, e->sval, e->slen);
+    else
+        addReplyBulkLongLong(c, e->lval);
+}
+
+/* Return random element from a non empty hash.
+ * 'key' and 'val' will be set to hold the element.
+ * The memory in them is not to be freed or modified by the caller.
+ * 'val' can be NULL in which case it's not extracted. */
+void hashTypeRandomElement(robj *hashobj, unsigned long hashsize, ziplistEntry *key, ziplistEntry *val) {
+    if (hashobj->encoding == OBJ_ENCODING_HT) {
+        dictEntry *de = dictGetFairRandomKey(hashobj->ptr);
+        sds s = dictGetKey(de);
+        key->sval = (unsigned char*)s;
+        key->slen = sdslen(s);
+        if (val) {
+            sds s = dictGetVal(de);
+            val->sval = (unsigned char*)s;
+            val->slen = sdslen(s);
+        }
+    } else if (hashobj->encoding == OBJ_ENCODING_ZIPLIST) {
+        ziplistRandomPair(hashobj->ptr, hashsize, key, val);
+    } else {
+        serverPanic("Unknown hash encoding");
+    }
+}
+
+
 /*-----------------------------------------------------------------------------
  * Hash type commands
  *----------------------------------------------------------------------------*/
@@ -922,3 +958,220 @@ void hscanCommand(client *c) {
         checkType(c,o,OBJ_HASH)) return;
     scanGenericCommand(c,o,cursor);
 }
+
+/* How many times bigger should be the hash compared to the requested size
+ * for us to not use the "remove elements" strategy? Read later in the
+ * implementation for more info. */
+#define HRANDFIELD_SUB_STRATEGY_MUL 3
+
+void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
+    unsigned long count, size;
+    int uniq = 1;
+    robj *hash;
+
+    if ((hash = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp]))
+        == NULL || checkType(c,hash,OBJ_HASH)) return;
+    size = hashTypeLength(hash);
+
+    if(l >= 0) {
+        count = (unsigned long) l;
+    } else {
+        count = -l;
+        uniq = 0;
+    }
+
+    /* If count is zero, serve it ASAP to avoid special cases later. */
+    if (count == 0) {
+        addReply(c,shared.emptyarray);
+        return;
+    }
+
+    /* CASE 1: The count was negative, so the extraction method is just:
+     * "return N random elements" sampling the whole set every time.
+     * This case is trivial and can be served without auxiliary data
+     * structures. This case is the only one that also needs to return the
+     * elements in random order. */
+    if (!uniq || count == 1) {
+        if (withvalues && c->resp == 2)
+            addReplyArrayLen(c, count*2);
+        else
+            addReplyArrayLen(c, count);
+        if (hash->encoding == OBJ_ENCODING_HT) {
+            sds key, value;
+            while (count--) {
+                dictEntry *de = dictGetRandomKey(hash->ptr);
+                key = dictGetKey(de);
+                value = dictGetVal(de);
+                if (withvalues && c->resp > 2)
+                    addReplyArrayLen(c,2);
+                addReplyBulkCBuffer(c, key, sdslen(key));
+                if (withvalues)
+                    addReplyBulkCBuffer(c, value, sdslen(value));
+            }
+        } else if (hash->encoding == OBJ_ENCODING_ZIPLIST) {
+            ziplistEntry *keys, *vals = NULL;
+            keys = zmalloc(sizeof(ziplistEntry)*count);
+            if (withvalues)
+                vals = zmalloc(sizeof(ziplistEntry)*count);
+            ziplistRandomPairs(hash->ptr, count, keys, vals);
+            for (unsigned long i = 0; i < count; i++) {
+                if (withvalues && c->resp > 2)
+                    addReplyArrayLen(c,2);
+                if (keys[i].sval)
+                    addReplyBulkCBuffer(c, keys[i].sval, keys[i].slen);
+                else
+                    addReplyBulkLongLong(c, keys[i].lval);
+                if (withvalues) {
+                    if (vals[i].sval)
+                        addReplyBulkCBuffer(c, vals[i].sval, vals[i].slen);
+                    else
+                        addReplyBulkLongLong(c, vals[i].lval);
+                }
+            }
+            zfree(keys);
+            zfree(vals);
+        }
+        return;
+    }
+
+    /* Initiate reply count, RESP3 responds with nested array, RESP2 with flat one. */
+    long reply_size = count < size ? count : size;
+    if (withvalues && c->resp == 2)
+        addReplyArrayLen(c, reply_size*2);
+    else
+        addReplyArrayLen(c, reply_size);
+
+    /* CASE 2:
+    * The number of requested elements is greater than the number of
+    * elements inside the hash: simply return the whole hash. */
+    if(count >= size) {
+        hashTypeIterator *hi = hashTypeInitIterator(hash);
+        while (hashTypeNext(hi) != C_ERR) {
+            if (withvalues && c->resp > 2)
+                addReplyArrayLen(c,2);
+            addHashIteratorCursorToReply(c, hi, OBJ_HASH_KEY);
+            if (withvalues)
+                addHashIteratorCursorToReply(c, hi, OBJ_HASH_VALUE);
+        }
+        hashTypeReleaseIterator(hi);
+        return;
+    }
+
+    /* CASE 3:
+     * The number of elements inside the hash is not greater than
+     * HRANDFIELD_SUB_STRATEGY_MUL times the number of requested elements.
+     * In this case we create a hash from scratch with all the elements, and
+     * subtract random elements to reach the requested number of elements.
+     *
+     * This is done because if the number of requested elements is just
+     * a bit less than the number of elements in the hash, the natural approach
+     * used into CASE 4 is highly inefficient. */
+    if (count*HRANDFIELD_SUB_STRATEGY_MUL > size) {
+        dict *d = dictCreate(&sdsReplyDictType, NULL);
+        hashTypeIterator *hi = hashTypeInitIterator(hash);
+
+        /* Add all the elements into the temporary dictionary. */
+        while ((hashTypeNext(hi)) != C_ERR) {
+            int ret = DICT_ERR;
+            sds key, value = NULL;
+
+            key = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_KEY);
+            if (withvalues)
+                value = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_VALUE);
+            ret = dictAdd(d, key, value);
+
+            serverAssert(ret == DICT_OK);
+        }
+        serverAssert(dictSize(d) == size);
+        hashTypeReleaseIterator(hi);
+
+        /* Remove random elements to reach the right count. */
+        while (size > count) {
+            dictEntry *de;
+            de = dictGetRandomKey(d);
+            dictUnlink(d,dictGetKey(de));
+            sdsfree(dictGetKey(de));
+            sdsfree(dictGetVal(de));
+            dictFreeUnlinkedEntry(d,de);
+            size--;
+        }
+
+        /* Reply with what's in the dict and release memory */
+        dictIterator *di;
+        dictEntry *de;
+        di = dictGetIterator(d);
+        while ((de = dictNext(di)) != NULL) {
+            sds key = dictGetKey(de);
+            sds value = dictGetVal(de);
+            if (withvalues && c->resp > 2)
+                addReplyArrayLen(c,2);
+            addReplyBulkSds(c, key);
+            if (withvalues)
+                addReplyBulkSds(c, value);
+        }
+
+        dictReleaseIterator(di);
+        dictRelease(d);
+    }
+
+    /* CASE 4: We have a big hash compared to the requested number of elements.
+     * In this case we can simply get random elements from the hash and add
+     * to the temporary hash, trying to eventually get enough unique elements
+     * to reach the specified count. */
+    else {
+        unsigned long added = 0;
+        ziplistEntry key, value;
+        dict *d = dictCreate(&hashDictType, NULL);
+        while(added < count) {
+            hashTypeRandomElement(hash, size, &key, withvalues? &value : NULL);
+
+            /* Try to add the object to the dictionary. If it already exists
+            * free it, otherwise increment the number of objects we have
+            * in the result dictionary. */
+            sds skey = hashSdsFromZiplistEntry(&key);
+            if (dictAdd(d,skey,NULL) != DICT_OK) {
+                sdsfree(skey);
+                continue;
+            }
+            added++;
+
+            /* We can reply right away, so that we don't need to store the value in the dict. */
+            if (withvalues && c->resp > 2)
+                addReplyArrayLen(c,2);
+            hashReplyFromZiplistEntry(c, &key);
+            if (withvalues)
+                hashReplyFromZiplistEntry(c, &value);
+        }
+
+        /* Release memory */
+        dictRelease(d);
+    }
+}
+
+/* HRANDFIELD [<count> WITHVALUES] */
+void hrandfieldCommand(client *c) {
+    long l;
+    int withvalues = 0;
+    robj *hash;
+    ziplistEntry ele;
+
+    if (c->argc >= 3) {
+        if (getLongFromObjectOrReply(c,c->argv[2],&l,NULL) != C_OK) return;
+        if (c->argc > 4 || (c->argc == 4 && strcasecmp(c->argv[3]->ptr,"withvalues"))) {
+            addReplyErrorObject(c,shared.syntaxerr);
+            return;
+        } else if (c->argc == 4)
+            withvalues = 1;
+        hrandfieldWithCountCommand(c, l, withvalues);
+        return;
+    }
+
+    /* Handle variant without <count> argument. Reply with simple bulk string */
+    if ((hash = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp]))== NULL ||
+        checkType(c,hash,OBJ_HASH)) {
+        return;
+    }
+
+    hashTypeRandomElement(hash,hashTypeLength(hash),&ele,NULL);
+    hashReplyFromZiplistEntry(c, &ele);
+}
diff --git a/src/t_set.c b/src/t_set.c
index 64bbbd3a0..de0a9f954 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -690,8 +690,9 @@ void srandmemberWithCountCommand(client *c) {
     /* CASE 1: The count was negative, so the extraction method is just:
      * "return N random elements" sampling the whole set every time.
      * This case is trivial and can be served without auxiliary data
-     * structures. */
-    if (!uniq) {
+     * structures. This case is the only one that also needs to return the
+     * elements in random order. */
+    if (!uniq || count == 1) {
         addReplySetLen(c,count);
         while(count--) {
             encoding = setTypeRandomElement(set,&ele,&llele);
@@ -713,7 +714,7 @@ void srandmemberWithCountCommand(client *c) {
     }
 
     /* For CASE 3 and CASE 4 we need an auxiliary dictionary. */
-    d = dictCreate(&objectKeyPointerValueDictType,NULL);
+    d = dictCreate(&sdsReplyDictType,NULL);
 
     /* CASE 3:
      * The number of elements inside the set is not greater than
@@ -729,13 +730,13 @@ void srandmemberWithCountCommand(client *c) {
 
         /* Add all the elements into the temporary dictionary. */
         si = setTypeInitIterator(set);
-        while((encoding = setTypeNext(si,&ele,&llele)) != -1) {
+        while ((encoding = setTypeNext(si,&ele,&llele)) != -1) {
             int retval = DICT_ERR;
 
             if (encoding == OBJ_ENCODING_INTSET) {
-                retval = dictAdd(d,createStringObjectFromLongLong(llele),NULL);
+                retval = dictAdd(d,sdsfromlonglong(llele),NULL);
             } else {
-                retval = dictAdd(d,createStringObject(ele,sdslen(ele)),NULL);
+                retval = dictAdd(d,sdsdup(ele),NULL);
             }
             serverAssert(retval == DICT_OK);
         }
@@ -743,11 +744,12 @@ void srandmemberWithCountCommand(client *c) {
         serverAssert(dictSize(d) == size);
 
         /* Remove random elements to reach the right count. */
-        while(size > count) {
+        while (size > count) {
             dictEntry *de;
-
             de = dictGetRandomKey(d);
-            dictDelete(d,dictGetKey(de));
+            dictUnlink(d,dictGetKey(de));
+            sdsfree(dictGetKey(de));
+            dictFreeUnlinkedEntry(d,de);
             size--;
         }
     }
@@ -758,22 +760,22 @@ void srandmemberWithCountCommand(client *c) {
      * to reach the specified count. */
     else {
         unsigned long added = 0;
-        robj *objele;
+        sds sdsele;
 
-        while(added < count) {
+        while (added < count) {
             encoding = setTypeRandomElement(set,&ele,&llele);
             if (encoding == OBJ_ENCODING_INTSET) {
-                objele = createStringObjectFromLongLong(llele);
+                sdsele = sdsfromlonglong(llele);
             } else {
-                objele = createStringObject(ele,sdslen(ele));
+                sdsele = sdsdup(ele);
             }
             /* Try to add the object to the dictionary. If it already exists
              * free it, otherwise increment the number of objects we have
              * in the result dictionary. */
-            if (dictAdd(d,objele,NULL) == DICT_OK)
+            if (dictAdd(d,sdsele,NULL) == DICT_OK)
                 added++;
             else
-                decrRefCount(objele);
+                sdsfree(sdsele);
         }
     }
 
@@ -785,12 +787,13 @@ void srandmemberWithCountCommand(client *c) {
         addReplySetLen(c,count);
         di = dictGetIterator(d);
         while((de = dictNext(di)) != NULL)
-            addReplyBulk(c,dictGetKey(de));
+            addReplyBulkSds(c,dictGetKey(de));
         dictReleaseIterator(di);
         dictRelease(d);
     }
 }
 
+/* SRANDMEMBER [<count>] */
 void srandmemberCommand(client *c) {
     robj *set;
     sds ele;
@@ -805,6 +808,7 @@ void srandmemberCommand(client *c) {
         return;
     }
 
+    /* Handle variant without <count> argument. Reply with simple bulk string */
     if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp]))
         == NULL || checkType(c,set,OBJ_SET)) return;
 
diff --git a/src/t_zset.c b/src/t_zset.c
index 6851ac86c..b55fc169e 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -721,20 +721,26 @@ zskiplistNode *zslLastInLexRange(zskiplist *zsl, zlexrangespec *range) {
  * Ziplist-backed sorted set API
  *----------------------------------------------------------------------------*/
 
+double zzlStrtod(unsigned char *vstr, unsigned int vlen) {
+    char buf[128];
+    if (vlen > sizeof(buf))
+        vlen = sizeof(buf);
+    memcpy(buf,vstr,vlen);
+    buf[vlen] = '\0';
+    return strtod(buf,NULL);
+ }
+
 double zzlGetScore(unsigned char *sptr) {
     unsigned char *vstr;
     unsigned int vlen;
     long long vlong;
-    char buf[128];
     double score;
 
     serverAssert(sptr != NULL);
     serverAssert(ziplistGet(sptr,&vstr,&vlen,&vlong));
 
     if (vstr) {
-        memcpy(buf,vstr,vlen);
-        buf[vlen] = '\0';
-        score = strtod(buf,NULL);
+        score = zzlStrtod(vstr,vlen);
     } else {
         score = vlong;
     }
@@ -1653,6 +1659,48 @@ int zsetZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep) {
     return ret;
 }
 
+/* Create a new sds string from the ziplist entry. */
+sds zsetSdsFromZiplistEntry(ziplistEntry *e) {
+    return e->sval ? sdsnewlen(e->sval, e->slen) : sdsfromlonglong(e->lval);
+}
+
+/* Reply with bulk string from the ziplist entry. */
+void zsetReplyFromZiplistEntry(client *c, ziplistEntry *e) {
+    if (e->sval)
+        addReplyBulkCBuffer(c, e->sval, e->slen);
+    else
+        addReplyBulkLongLong(c, e->lval);
+}
+
+
+/* Return random element from a non empty zset.
+ * 'key' and 'val' will be set to hold the element.
+ * The memory in `key` is not to be freed or modified by the caller.
+ * 'score' can be NULL in which case it's not extracted. */
+void zsetTypeRandomElement(robj *zsetobj, unsigned long zsetsize, ziplistEntry *key, double *score) {
+    if (zsetobj->encoding == OBJ_ENCODING_SKIPLIST) {
+        zset *zs = zsetobj->ptr;
+        dictEntry *de = dictGetFairRandomKey(zs->dict);
+        sds s = dictGetKey(de);
+        key->sval = (unsigned char*)s;
+        key->slen = sdslen(s);
+        if (score)
+            *score = *(double*)dictGetVal(de);
+    } else if (zsetobj->encoding == OBJ_ENCODING_ZIPLIST) {
+        ziplistEntry val;
+        ziplistRandomPair(zsetobj->ptr, zsetsize, key, &val);
+        if (score) {
+            if (val.sval) {
+                *score = zzlStrtod(val.sval,val.slen);
+            } else {
+                *score = (double)val.lval;
+            }
+        }
+    } else {
+        serverPanic("Unknown zset encoding");
+    }
+}
+
 /*-----------------------------------------------------------------------------
  * Sorted set commands
  *----------------------------------------------------------------------------*/
@@ -3907,3 +3955,216 @@ void bzpopminCommand(client *c) {
 void bzpopmaxCommand(client *c) {
     blockingGenericZpopCommand(c,ZSET_MAX);
 }
+
+/* How many times bigger should be the zset compared to the requested size
+ * for us to not use the "remove elements" strategy? Read later in the
+ * implementation for more info. */
+#define ZRANDMEMBER_SUB_STRATEGY_MUL 3
+
+void zrandmemberWithCountCommand(client *c, long l, int withscores) {
+    unsigned long count, size;
+    int uniq = 1;
+    robj *zsetobj;
+
+    if ((zsetobj = lookupKeyReadOrReply(c, c->argv[1], shared.null[c->resp]))
+        == NULL || checkType(c, zsetobj, OBJ_ZSET)) return;
+    size = zsetLength(zsetobj);
+
+    if(l >= 0) {
+        count = (unsigned long) l;
+    } else {
+        count = -l;
+        uniq = 0;
+    }
+
+    /* If count is zero, serve it ASAP to avoid special cases later. */
+    if (count == 0) {
+        addReply(c,shared.emptyarray);
+        return;
+    }
+
+    /* CASE 1: The count was negative, so the extraction method is just:
+     * "return N random elements" sampling the whole set every time.
+     * This case is trivial and can be served without auxiliary data
+     * structures. This case is the only one that also needs to return the
+     * elements in random order. */
+    if (!uniq || count == 1) {
+        if (withscores && c->resp == 2)
+            addReplyArrayLen(c, count*2);
+        else
+            addReplyArrayLen(c, count);
+        if (zsetobj->encoding == OBJ_ENCODING_SKIPLIST) {
+            zset *zs = zsetobj->ptr;
+            while (count--) {
+                dictEntry *de = dictGetFairRandomKey(zs->dict);
+                sds key = dictGetKey(de);
+                if (withscores && c->resp > 2)
+                    addReplyArrayLen(c,2);
+                addReplyBulkCBuffer(c, key, sdslen(key));
+                if (withscores)
+                    addReplyDouble(c, dictGetDoubleVal(de));
+            }
+        } else if (zsetobj->encoding == OBJ_ENCODING_ZIPLIST) {
+            ziplistEntry *keys, *vals = NULL;
+            keys = zmalloc(sizeof(ziplistEntry)*count);
+            if (withscores)
+                vals = zmalloc(sizeof(ziplistEntry)*count);
+            ziplistRandomPairs(zsetobj->ptr, count, keys, vals);
+            for (unsigned long i = 0; i < count; i++) {
+                if (withscores && c->resp > 2)
+                    addReplyArrayLen(c,2);
+                if (keys[i].sval)
+                    addReplyBulkCBuffer(c, keys[i].sval, keys[i].slen);
+                else
+                    addReplyBulkLongLong(c, keys[i].lval);
+                if (withscores) {
+                    if (vals[i].sval) {
+                        addReplyDouble(c, zzlStrtod(vals[i].sval,vals[i].slen));
+                    } else
+                        addReplyDouble(c, vals[i].lval);
+                }
+            }
+            zfree(keys);
+            zfree(vals);
+        }
+        return;
+    }
+
+    zsetopsrc src;
+    zsetopval zval;
+    src.subject = zsetobj;
+    src.type = zsetobj->type;
+    src.encoding = zsetobj->encoding;
+    zuiInitIterator(&src);
+    memset(&zval, 0, sizeof(zval));
+
+    /* Initiate reply count, RESP3 responds with nested array, RESP2 with flat one. */
+    long reply_size = count < size ? count : size;
+    if (withscores && c->resp == 2)
+        addReplyArrayLen(c, reply_size*2);
+    else
+        addReplyArrayLen(c, reply_size);
+
+    /* CASE 2:
+    * The number of requested elements is greater than the number of
+    * elements inside the zset: simply return the whole zset. */
+    if (count >= size) {
+        while (zuiNext(&src, &zval)) {
+            if (withscores && c->resp > 2)
+                addReplyArrayLen(c,2);
+            addReplyBulkSds(c, zuiNewSdsFromValue(&zval));
+            if (withscores)
+                addReplyDouble(c, zval.score);
+        }
+        return;
+    }
+
+    /* CASE 3:
+     * The number of elements inside the zset is not greater than
+     * ZRANDMEMBER_SUB_STRATEGY_MUL times the number of requested elements.
+     * In this case we create a dict from scratch with all the elements, and
+     * subtract random elements to reach the requested number of elements.
+     *
+     * This is done because if the number of requested elements is just
+     * a bit less than the number of elements in the set, the natural approach
+     * used into CASE 4 is highly inefficient. */
+    if (count*ZRANDMEMBER_SUB_STRATEGY_MUL > size) {
+        dict *d = dictCreate(&sdsReplyDictType, NULL);
+        /* Add all the elements into the temporary dictionary. */
+        while (zuiNext(&src, &zval)) {
+            sds key = zuiNewSdsFromValue(&zval);
+            dictEntry *de = dictAddRaw(d, key, NULL);
+            serverAssert(de);
+            if (withscores)
+                dictSetDoubleVal(de, zval.score);
+        }
+        serverAssert(dictSize(d) == size);
+
+        /* Remove random elements to reach the right count. */
+        while (size > count) {
+            dictEntry *de;
+            de = dictGetRandomKey(d);
+            dictUnlink(d,dictGetKey(de));
+            sdsfree(dictGetKey(de));
+            dictFreeUnlinkedEntry(d,de);
+            size--;
+        }
+
+        /* Reply with what's in the dict and release memory */
+        dictIterator *di;
+        dictEntry *de;
+        di = dictGetIterator(d);
+        while ((de = dictNext(di)) != NULL) {
+            if (withscores && c->resp > 2)
+                addReplyArrayLen(c,2);
+            addReplyBulkSds(c, dictGetKey(de));
+            if (withscores)
+                addReplyDouble(c, dictGetDoubleVal(de));
+        }
+
+        dictReleaseIterator(di);
+        dictRelease(d);
+    }
+
+    /* CASE 4: We have a big zset compared to the requested number of elements.
+     * In this case we can simply get random elements from the zset and add
+     * to the temporary set, trying to eventually get enough unique elements
+     * to reach the specified count. */
+    else {
+        unsigned long added = 0;
+        dict *d = dictCreate(&hashDictType, NULL);
+
+        while (added < count) {
+            ziplistEntry key;
+            double score;
+            zsetTypeRandomElement(zsetobj, size, &key, withscores ? &score: NULL);
+
+            /* Try to add the object to the dictionary. If it already exists
+            * free it, otherwise increment the number of objects we have
+            * in the result dictionary. */
+            sds skey = zsetSdsFromZiplistEntry(&key);
+            if (dictAdd(d,skey,NULL) != DICT_OK) {
+                sdsfree(skey);
+                continue;
+            }
+            added++;
+
+            if (withscores && c->resp > 2)
+                addReplyArrayLen(c,2);
+            zsetReplyFromZiplistEntry(c, &key);
+            if (withscores)
+                addReplyDouble(c, score);
+        }
+
+        /* Release memory */
+        dictRelease(d);
+    }
+}
+
+/* ZRANDMEMBER [<count> WITHSCORES] */
+void zrandmemberCommand(client *c) {
+    long l;
+    int withscores = 0;
+    robj *zset;
+    ziplistEntry ele;
+
+    if (c->argc >= 3) {
+        if (getLongFromObjectOrReply(c,c->argv[2],&l,NULL) != C_OK) return;
+        if (c->argc > 4 || (c->argc == 4 && strcasecmp(c->argv[3]->ptr,"withscores"))) {
+            addReplyErrorObject(c,shared.syntaxerr);
+            return;
+        } else if (c->argc == 4)
+            withscores = 1;
+        zrandmemberWithCountCommand(c, l, withscores);
+        return;
+    }
+
+    /* Handle variant without <count> argument. Reply with simple bulk string */
+    if ((zset = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp]))== NULL ||
+        checkType(c,zset,OBJ_ZSET)) {
+        return;
+    }
+
+    zsetTypeRandomElement(zset, zsetLength(zset), &ele,NULL);
+    zsetReplyFromZiplistEntry(c,&ele);
+}
diff --git a/src/ziplist.c b/src/ziplist.c
index a4f38c5e8..62ffcb93d 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -1498,6 +1498,83 @@ int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep,
     return 1;
 }
 
+/* Randomly select a pair of key and value.
+ * total_count is a pre-computed length/2 of the ziplist (to avoid calls to ziplistLen)
+ * 'key' and 'val' are used to store the result key value pair.
+ * 'val' can be NULL if the value is not needed. */
+void ziplistRandomPair(unsigned char *zl, unsigned long total_count, ziplistEntry *key, ziplistEntry *val) {
+    int ret;
+    unsigned char *p;
+
+    /* Generate even numbers, because ziplist saved K-V pair */
+    int r = (rand() % total_count) * 2;
+    p = ziplistIndex(zl, r);
+    ret = ziplistGet(p, &key->sval, &key->slen, &key->lval);
+    assert(ret != 0);
+
+    if (!val)
+        return;
+    p = ziplistNext(zl, p);
+    ret = ziplistGet(p, &val->sval, &val->slen, &val->lval);
+    assert(ret != 0);
+}
+
+/* int compare for qsort */
+int intCompare(const void *a, const void *b) {
+    return (*(int *) a - *(int *) b);
+}
+
+/* Helper method to store a string into from val or lval into dest */
+static inline void ziplistSaveValue(unsigned char *val, unsigned int len, long long lval, ziplistEntry *dest) {
+    dest->sval = val;
+    dest->slen = len;
+    dest->lval = lval;
+}
+
+/* Randomly select unique count of key value pairs and store into 'keys' and
+ * 'vals' args. The order of the picked entries is random.
+ * The 'vals' arg can be NULL in which case we skip these. */
+void ziplistRandomPairs(unsigned char *zl, int count, ziplistEntry *keys, ziplistEntry *vals) {
+    unsigned char *p, *key, *value;
+    unsigned int klen, vlen;
+    long long klval, vlval;
+    typedef struct {
+        int index;
+        int order;
+    } rand_pick;
+    rand_pick *picks = zmalloc(sizeof(rand_pick)*count);
+    unsigned long total_size = ziplistLen(zl)/2;
+
+    /* create a pool of random indexes (some may be duplicate). */
+    for (int i = 0; i < count; i++) {
+        picks[i].index = (rand() % total_size) * 2; /* Generate even indexes */
+        /* keep track of the order we picked them */
+        picks[i].order = i;
+    }
+
+    /* sort by indexes. */
+    qsort(picks, count, sizeof(rand_pick), intCompare);
+
+    /* fetch the elements form the ziplist into a output array respecting the original order. */
+    int zipindex = 0, pickindex = 0;
+    p = ziplistIndex(zl, 0);
+    while (ziplistGet(p, &key, &klen, &klval) && pickindex < count) {
+        p = ziplistNext(zl, p);
+        ziplistGet(p, &value, &vlen, &vlval);
+        while (pickindex < count && zipindex == picks[pickindex].index) {
+            int storeorder = picks[pickindex].order;
+            ziplistSaveValue(key, klen, klval, &keys[storeorder]);
+            if (vals)
+                ziplistSaveValue(value, vlen, vlval, &vals[storeorder]);
+             pickindex++;
+        }
+        zipindex += 2;
+        p = ziplistNext(zl, p);
+    }
+
+    zfree(picks);
+}
+
 #ifdef REDIS_TEST
 #include <sys/time.h>
 #include "adlist.h"
diff --git a/src/ziplist.h b/src/ziplist.h
index 5153951dc..5fb8fd46a 100644
--- a/src/ziplist.h
+++ b/src/ziplist.h
@@ -34,6 +34,15 @@
 #define ZIPLIST_HEAD 0
 #define ZIPLIST_TAIL 1
 
+/* Each entry in the ziplist is either a string or an integer. */
+typedef struct {
+    /* When string is used, it is provided with the length (slen). */
+    unsigned char *sval;
+    unsigned int slen;
+    /* When integer is used, 'sval' is NULL, and lval holds the value. */
+    long long lval;
+} ziplistEntry;
+
 unsigned char *ziplistNew(void);
 unsigned char *ziplistMerge(unsigned char **first, unsigned char **second);
 unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where);
@@ -52,6 +61,8 @@ void ziplistRepr(unsigned char *zl);
 typedef int (*ziplistValidateEntryCB)(unsigned char* p, void* userdata);
 int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep,
                              ziplistValidateEntryCB entry_cb, void *cb_userdata);
+void ziplistRandomPair(unsigned char *zl, unsigned long total_count, ziplistEntry *key, ziplistEntry *val);
+void ziplistRandomPairs(unsigned char *zl, int count, ziplistEntry *keys, ziplistEntry *vals);
 
 #ifdef REDIS_TEST
 int ziplistTest(int argc, char *argv[]);
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index e69395994..943f45a2b 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -569,8 +569,8 @@ proc generate_fuzzy_traffic_on_key {key duration} {
     # Commands per type, blocking commands removed
     # TODO: extract these from help.h or elsewhere, and improve to include other types
     set string_commands {APPEND BITCOUNT BITFIELD BITOP BITPOS DECR DECRBY GET GETBIT GETRANGE GETSET INCR INCRBY INCRBYFLOAT MGET MSET MSETNX PSETEX SET SETBIT SETEX SETNX SETRANGE STRALGO STRLEN}
-    set hash_commands {HDEL HEXISTS HGET HGETALL HINCRBY HINCRBYFLOAT HKEYS HLEN HMGET HMSET HSCAN HSET HSETNX HSTRLEN HVALS}
-    set zset_commands {ZADD ZCARD ZCOUNT ZINCRBY ZINTERSTORE ZLEXCOUNT ZPOPMAX ZPOPMIN ZRANGE ZRANGEBYLEX ZRANGEBYSCORE ZRANK ZREM ZREMRANGEBYLEX ZREMRANGEBYRANK ZREMRANGEBYSCORE ZREVRANGE ZREVRANGEBYLEX ZREVRANGEBYSCORE ZREVRANK ZSCAN ZSCORE ZUNIONSTORE}
+    set hash_commands {HDEL HEXISTS HGET HGETALL HINCRBY HINCRBYFLOAT HKEYS HLEN HMGET HMSET HSCAN HSET HSETNX HSTRLEN HVALS HRANDFIELD}
+    set zset_commands {ZADD ZCARD ZCOUNT ZINCRBY ZINTERSTORE ZLEXCOUNT ZPOPMAX ZPOPMIN ZRANGE ZRANGEBYLEX ZRANGEBYSCORE ZRANK ZREM ZREMRANGEBYLEX ZREMRANGEBYRANK ZREMRANGEBYSCORE ZREVRANGE ZREVRANGEBYLEX ZREVRANGEBYSCORE ZREVRANK ZSCAN ZSCORE ZUNIONSTORE ZRANDMEMBER}
     set list_commands {LINDEX LINSERT LLEN LPOP LPOS LPUSH LPUSHX LRANGE LREM LSET LTRIM RPOP RPOPLPUSH RPUSH RPUSHX}
     set set_commands {SADD SCARD SDIFF SDIFFSTORE SINTER SINTERSTORE SISMEMBER SMEMBERS SMOVE SPOP SRANDMEMBER SREM SSCAN SUNION SUNIONSTORE}
     set stream_commands {XACK XADD XCLAIM XDEL XGROUP XINFO XLEN XPENDING XRANGE XREAD XREADGROUP XREVRANGE XTRIM}
diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl
index 79e58301a..2f3ea37c2 100644
--- a/tests/unit/type/hash.tcl
+++ b/tests/unit/type/hash.tcl
@@ -18,6 +18,181 @@ start_server {tags {"hash"}} {
         assert_encoding ziplist smallhash
     }
 
+    proc create_hash {key entries} {
+        r del $key
+        foreach entry $entries {
+            r hset $key [lindex $entry 0] [lindex $entry 1]
+        }
+    }
+
+    proc get_keys {l} {
+        set res {}
+        foreach entry $l {
+            set key [lindex $entry 0]
+            lappend res $key
+        }
+        return $res
+    }
+
+    foreach {type contents} "ziplist {{a 1} {b 2} {c 3}} hashtable {{a 1} {b 2} {[randstring 70 90 alpha] 3}}" {
+        set original_max_value [lindex [r config get hash-max-ziplist-value] 1]
+        r config set hash-max-ziplist-value 10
+        create_hash myhash $contents
+        assert_encoding $type myhash
+
+        test "HRANDFIELD - $type" {
+            unset -nocomplain myhash
+            array set myhash {}
+            for {set i 0} {$i < 100} {incr i} {
+                set key [r hrandfield myhash]
+                set myhash($key) 1
+            }
+            assert_equal [lsort [get_keys $contents]] [lsort [array names myhash]]
+        }
+        r config set hash-max-ziplist-value $original_max_value
+    }
+
+    test "HRANDFIELD with RESP3" {
+        r hello 3
+        set res [r hrandfield myhash 3 withvalues]
+        assert_equal [llength $res] 3
+        assert_equal [llength [lindex $res 1]] 2
+
+        set res [r hrandfield myhash 3]
+        assert_equal [llength $res] 3
+        assert_equal [llength [lindex $res 1]] 1
+    }
+    r hello 2
+
+    test "HRANDFIELD count of 0 is handled correctly" {
+        r hrandfield myhash 0
+    } {}
+
+    test "HRANDFIELD with <count> against non existing key" {
+        r hrandfield nonexisting_key 100
+    } {}
+
+    foreach {type contents} "
+        hashtable {{a 1} {b 2} {c 3} {d 4} {e 5} {6 f} {7 g} {8 h} {9 i} {[randstring 70 90 alpha] 10}}
+        ziplist {{a 1} {b 2} {c 3} {d 4} {e 5} {6 f} {7 g} {8 h} {9 i} {10 j}} " {
+        test "HRANDFIELD with <count> - $type" {
+            set original_max_value [lindex [r config get hash-max-ziplist-value] 1]
+            r config set hash-max-ziplist-value 10
+            create_hash myhash $contents
+            assert_encoding $type myhash
+
+            # create a dict for easy lookup
+            unset -nocomplain mydict
+            foreach {k v} [r hgetall myhash] {
+                dict append mydict $k $v
+            }
+
+            # We'll stress different parts of the code, see the implementation
+            # of HRANDFIELD for more information, but basically there are
+            # four different code paths.
+
+            # PATH 1: Use negative count.
+
+            # 1) Check that it returns repeated elements with and without values.
+            set res [r hrandfield myhash -20]
+            assert_equal [llength $res] 20
+            # again with WITHVALUES
+            set res [r hrandfield myhash -20 withvalues]
+            assert_equal [llength $res] 40
+
+            # 2) Check that all the elements actually belong to the original hash.
+            foreach {key val} $res {
+                assert {[dict exists $mydict $key]}
+            }
+
+            # 3) Check that eventually all the elements are returned.
+            #    Use both WITHVALUES and without
+            unset -nocomplain auxset
+            set iterations 1000
+            while {$iterations != 0} {
+                incr iterations -1
+                if {[expr {$iterations % 2}] == 0} {
+                    set res [r hrandfield myhash -3 withvalues]
+                    foreach {key val} $res {
+                        dict append auxset $key $val
+                    }
+                } else {
+                    set res [r hrandfield myhash -3]
+                    foreach key $res {
+                        dict append auxset $key $val
+                    }
+                }
+                if {[lsort [dict keys $mydict]] eq
+                    [lsort [dict keys $auxset]]} {
+                    break;
+                }
+            }
+            assert {$iterations != 0}
+
+            # PATH 2: positive count (unique behavior) with requested size
+            # equal or greater than set size.
+            foreach size {10 20} {
+                set res [r hrandfield myhash $size]
+                assert_equal [llength $res] 10
+                assert_equal [lsort $res] [lsort [dict keys $mydict]]
+
+                # again with WITHVALUES
+                set res [r hrandfield myhash $size withvalues]
+                assert_equal [llength $res] 20
+                assert_equal [lsort $res] [lsort $mydict]
+            }
+
+            # PATH 3: Ask almost as elements as there are in the set.
+            # In this case the implementation will duplicate the original
+            # set and will remove random elements up to the requested size.
+            #
+            # PATH 4: Ask a number of elements definitely smaller than
+            # the set size.
+            #
+            # We can test both the code paths just changing the size but
+            # using the same code.
+            foreach size {8 2} {
+                set res [r hrandfield myhash $size]
+                assert_equal [llength $res] $size
+                # again with WITHVALUES
+                set res [r hrandfield myhash $size withvalues]
+                assert_equal [llength $res] [expr {$size * 2}]
+
+                # 1) Check that all the elements actually belong to the
+                # original set.
+                foreach ele [dict keys $res] {
+                    assert {[dict exists $mydict $ele]}
+                }
+
+                # 2) Check that eventually all the elements are returned.
+                #    Use both WITHVALUES and without
+                unset -nocomplain auxset
+                set iterations 1000
+                while {$iterations != 0} {
+                    incr iterations -1
+                    if {[expr {$iterations % 2}] == 0} {
+                        set res [r hrandfield myhash $size withvalues]
+                        foreach {key value} $res {
+                            dict append auxset $key $value
+                        }
+                    } else {
+                        set res [r hrandfield myhash $size]
+                        foreach key $res {
+                            dict append auxset $key
+                        }
+                    }
+                    if {[lsort [dict keys $mydict]] eq
+                        [lsort [dict keys $auxset]]} {
+                        break;
+                    }
+                }
+                assert {$iterations != 0}
+            }
+        }
+        r config set hash-max-ziplist-value $original_max_value
+    }
+
+
     test {HSET/HLEN - Big hash creation} {
         array set bighash {}
         for {set i 0} {$i < 1024} {incr i} {
diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl
index 84c31c4e4..091ef7f0f 100644
--- a/tests/unit/type/set.tcl
+++ b/tests/unit/type/set.tcl
@@ -501,7 +501,7 @@ start_server {
                 set iterations 1000
                 while {$iterations != 0} {
                     incr iterations -1
-                    set res [r srandmember myset -10]
+                    set res [r srandmember myset $size]
                     foreach ele $res {
                         set auxset($ele) 1
                     }
diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl
index f3c952f0e..c657c1e4e 100644
--- a/tests/unit/type/zset.tcl
+++ b/tests/unit/type/zset.tcl
@@ -7,6 +7,8 @@ start_server {tags {"zset"}} {
     }
 
     proc basics {encoding} {
+        set original_max_entries [lindex [r config get zset-max-ziplist-entries] 1]
+        set original_max_value [lindex [r config get zset-max-ziplist-value] 1]
         if {$encoding == "ziplist"} {
             r config set zset-max-ziplist-entries 128
             r config set zset-max-ziplist-value 64
@@ -925,6 +927,9 @@ start_server {tags {"zset"}} {
             assert_equal 0 [r zcard z1]
             assert_equal 1 [r zcard z2]
         }
+
+        r config set zset-max-ziplist-entries $original_max_entries
+        r config set zset-max-ziplist-value $original_max_value
     }
 
     basics ziplist
@@ -1022,6 +1027,8 @@ start_server {tags {"zset"}} {
     }
 
     proc stressers {encoding} {
+        set original_max_entries [lindex [r config get zset-max-ziplist-entries] 1]
+        set original_max_value [lindex [r config get zset-max-ziplist-value] 1]
         if {$encoding == "ziplist"} {
             # Little extra to allow proper fuzzing in the sorting stresser
             r config set zset-max-ziplist-entries 256
@@ -1446,6 +1453,8 @@ start_server {tags {"zset"}} {
             r zadd zset 0 foo
             assert_equal {zset foo 0} [$rd read]
         }
+        r config set zset-max-ziplist-entries $original_max_entries
+        r config set zset-max-ziplist-value $original_max_value
     }
 
     tags {"slow"} {
@@ -1566,4 +1575,171 @@ start_server {tags {"zset"}} {
         catch {r zrangebyscore z1 0 -1 REV} err
         assert_match "*syntax*" $err
     }
+
+    proc get_keys {l} {
+        set res {}
+        foreach {score key} $l {
+            lappend res $key
+        }
+        return $res
+    }
+
+    foreach {type contents} "ziplist {1 a 2 b 3 c} skiplist {1 a 2 b 3 [randstring 70 90 alpha]}" {
+        set original_max_value [lindex [r config get zset-max-ziplist-value] 1]
+        r config set zset-max-ziplist-value 10
+        create_zset myzset $contents
+        assert_encoding $type myzset
+
+        test "ZRANDMEMBER - $type" {
+            unset -nocomplain myzset
+            array set myzset {}
+            for {set i 0} {$i < 100} {incr i} {
+                set key [r zrandmember myzset]
+                set myzset($key) 1
+            }
+            assert_equal [lsort [get_keys $contents]] [lsort [array names myzset]]
+        }
+        r config set zset-max-ziplist-value $original_max_value
+    }
+
+    test "ZRANDMEMBER with RESP3" {
+        r hello 3
+        set res [r zrandmember myzset 3 withscores]
+        assert_equal [llength $res] 3
+        assert_equal [llength [lindex $res 1]] 2
+
+        set res [r zrandmember myzset 3]
+        assert_equal [llength $res] 3
+        assert_equal [llength [lindex $res 1]] 1
+    }
+    r hello 2
+
+    test "ZRANDMEMBER count of 0 is handled correctly" {
+        r zrandmember myzset 0
+    } {}
+
+    test "ZRANDMEMBER with <count> against non existing key" {
+        r zrandmember nonexisting_key 100
+    } {}
+
+    foreach {type contents} "
+        skiplist {1 a 2 b 3 c 4 d 5 e 6 f 7 g 7 h 9 i 10 [randstring 70 90 alpha]}
+        ziplist {1 a 2 b 3 c 4 d 5 e 6 f 7 g 7 h 9 i 10 j} " {
+        test "ZRANDMEMBER with <count> - $type" {
+            set original_max_value [lindex [r config get zset-max-ziplist-value] 1]
+            r config set zset-max-ziplist-value 10
+            create_zset myzset $contents
+            assert_encoding $type myzset
+
+            # create a dict for easy lookup
+            unset -nocomplain mydict
+            foreach {k v} [r zrange myzset 0 -1 withscores] {
+                dict append mydict $k $v
+            }
+
+            # We'll stress different parts of the code, see the implementation
+            # of ZRANDMEMBER for more information, but basically there are
+            # four different code paths.
+
+            # PATH 1: Use negative count.
+
+            # 1) Check that it returns repeated elements with and without values.
+            set res [r zrandmember myzset -20]
+            assert_equal [llength $res] 20
+            # again with WITHSCORES
+            set res [r zrandmember myzset -20 withscores]
+            assert_equal [llength $res] 40
+
+            # 2) Check that all the elements actually belong to the original zset.
+            foreach {key val} $res {
+                assert {[dict exists $mydict $key]}
+            }
+
+            # 3) Check that eventually all the elements are returned.
+            #    Use both WITHSCORES and without
+            unset -nocomplain auxset
+            set iterations 1000
+            while {$iterations != 0} {
+                incr iterations -1
+                if {[expr {$iterations % 2}] == 0} {
+                    set res [r zrandmember myzset -3 withscores]
+                    foreach {key val} $res {
+                        dict append auxset $key $val
+                    }
+                } else {
+                    set res [r zrandmember myzset -3]
+                    foreach key $res {
+                        dict append auxset $key $val
+                    }
+                }
+                if {[lsort [dict keys $mydict]] eq
+                    [lsort [dict keys $auxset]]} {
+                    break;
+                }
+            }
+            assert {$iterations != 0}
+
+            # PATH 2: positive count (unique behavior) with requested size
+            # equal or greater than set size.
+            foreach size {10 20} {
+                set res [r zrandmember myzset $size]
+                assert_equal [llength $res] 10
+                assert_equal [lsort $res] [lsort [dict keys $mydict]]
+
+                # again with WITHSCORES
+                set res [r zrandmember myzset $size withscores]
+                assert_equal [llength $res] 20
+                assert_equal [lsort $res] [lsort $mydict]
+            }
+
+            # PATH 3: Ask almost as elements as there are in the set.
+            # In this case the implementation will duplicate the original
+            # set and will remove random elements up to the requested size.
+            #
+            # PATH 4: Ask a number of elements definitely smaller than
+            # the set size.
+            #
+            # We can test both the code paths just changing the size but
+            # using the same code.
+            foreach size {8 2} {
+                set res [r zrandmember myzset $size]
+                assert_equal [llength $res] $size
+                # again with WITHSCORES
+                set res [r zrandmember myzset $size withscores]
+                assert_equal [llength $res] [expr {$size * 2}]
+
+                # 1) Check that all the elements actually belong to the
+                # original set.
+                foreach ele [dict keys $res] {
+                    assert {[dict exists $mydict $ele]}
+                }
+
+                # 2) Check that eventually all the elements are returned.
+                #    Use both WITHSCORES and without
+                unset -nocomplain auxset
+                set iterations 1000
+                while {$iterations != 0} {
+                    incr iterations -1
+                    if {[expr {$iterations % 2}] == 0} {
+                        set res [r zrandmember myzset $size withscores]
+                        foreach {key value} $res {
+                            dict append auxset $key $value
+                        }
+                    } else {
+                        set res [r zrandmember myzset $size]
+                        foreach key $res {
+                            dict append auxset $key
+                        }
+                    }
+                    if {[lsort [dict keys $mydict]] eq
+                        [lsort [dict keys $auxset]]} {
+                        break;
+                    }
+                }
+                assert {$iterations != 0}
+            }
+        }
+        r config set zset-max-ziplist-value $original_max_value
+    }
+
 }

From f0c5052aa8de084f54b3eda7848f17b122907509 Mon Sep 17 00:00:00 2001
From: filipe oliveira <filipecosta.90@gmail.com>
Date: Fri, 29 Jan 2021 13:38:30 +0000
Subject: [PATCH 1017/1098] Enabled background and reply time tracking on
 blocked on keys/blocked on background work clients (#7491)

This commit enables tracking time of the background tasks and on replies,
opening the door for properly tracking commands that rely on blocking / background
 work via the slowlog, latency history, and commandstats.

Some notes:
- The time spent blocked waiting for key changes, or blocked on synchronous
  replication is not accounted for.

- **This commit does not affect latency tracking of commands that are non-blocking
  or do not have background work.** ( meaning that it all stays the same with exception to
  `BZPOPMIN`,`BZPOPMAX`,`BRPOP`,`BLPOP`, etc... and module's commands that rely
  on background threads ).

-  Specifically for latency history command we've added a new event class named
  `command-unblocking` that will enable latency monitoring on commands that spawn
  background threads to do the work.

- For blocking commands we're now considering the total time of a command as the
  time spent on call() + the time spent on replying when unblocked.

- For Modules commands that rely on background threads we're now considering the
  total time of a command as the time spent on call (main thread) + the time spent on
  the background thread ( if marked within `RedisModule_MeasureTimeStart()` and
  `RedisModule_MeasureTimeEnd()` ) + the time spent on replying (main thread)

To test for this feature we've added a `unit/moduleapi/blockonbackground` test that relies on
a module that blocks the client and sleeps on the background for a given time.
- check blocked command that uses RedisModule_MeasureTimeStart() is tracking background time
- check blocked command that uses RedisModule_MeasureTimeStart() is tracking background time even in timeout
- check blocked command with multiple calls RedisModule_MeasureTimeStart()  is tracking the total background time
- check blocked command without calling RedisModule_MeasureTimeStart() is not reporting background time
---
 runtest-moduleapi                          |   1 +
 src/blocked.c                              |  29 +++
 src/module.c                               |  51 +++++
 src/redismodule.h                          |   4 +
 src/server.c                               |  12 +-
 src/server.h                               |   2 +
 tests/modules/Makefile                     |   1 +
 tests/modules/blockonbackground.c          | 220 +++++++++++++++++++++
 tests/unit/moduleapi/blockonbackground.tcl |  67 +++++++
 9 files changed, 383 insertions(+), 4 deletions(-)
 create mode 100644 tests/modules/blockonbackground.c
 create mode 100644 tests/unit/moduleapi/blockonbackground.tcl

diff --git a/runtest-moduleapi b/runtest-moduleapi
index 878b5c6ad..e554226c1 100755
--- a/runtest-moduleapi
+++ b/runtest-moduleapi
@@ -23,6 +23,7 @@ $TCLSH tests/test_helper.tcl \
 --single unit/moduleapi/hooks \
 --single unit/moduleapi/misc \
 --single unit/moduleapi/blockonkeys \
+--single unit/moduleapi/blockonbackground \
 --single unit/moduleapi/scan \
 --single unit/moduleapi/datatype \
 --single unit/moduleapi/auth \
diff --git a/src/blocked.c b/src/blocked.c
index 1d69ff902..09e17213c 100644
--- a/src/blocked.c
+++ b/src/blocked.c
@@ -61,6 +61,9 @@
  */
 
 #include "server.h"
+#include "slowlog.h"
+#include "latency.h"
+#include "monotonic.h"
 
 int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int wherefrom, int whereto);
 int getListPositionFromObjectOrReply(client *c, robj *arg, int *position);
@@ -97,6 +100,20 @@ void blockClient(client *c, int btype) {
     }
 }
 
+/* This function is called after a client has finished a blocking operation
+ * in order to update the total command duration, log the command into
+ * the Slow log if needed, and log the reply duration event if needed. */
+void updateStatsOnUnblock(client *c, long blocked_us, long reply_us){
+    const ustime_t total_cmd_duration = c->duration + blocked_us + reply_us;
+    c->lastcmd->microseconds += total_cmd_duration;
+    /* Log the command into the Slow log if needed. */
+    if (!(c->lastcmd->flags & CMD_SKIP_SLOWLOG)) {
+        slowlogPushEntryIfNeeded(c,c->argv,c->argc,total_cmd_duration);
+        /* Log the reply duration event. */
+        latencyAddSampleIfNeeded("command-unblocking",reply_us/1000);
+    }
+}
+
 /* This function is called in the beforeSleep() function of the event loop
  * in order to process the pending input buffer of clients that were
  * unblocked after a blocking operation. */
@@ -264,6 +281,8 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) {
                 if (dstkey) incrRefCount(dstkey);
                 unblockClient(receiver);
 
+                monotime replyTimer;
+                elapsedStart(&replyTimer);
                 if (serveClientBlockedOnList(receiver,
                     rl->key,dstkey,rl->db,value,
                     wherefrom, whereto) == C_ERR)
@@ -272,6 +291,7 @@ void serveClientsBlockedOnListKey(robj *o, readyList *rl) {
                      * to also undo the POP operation. */
                     listTypePush(o,value,wherefrom);
                 }
+                updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer));
 
                 if (dstkey) decrRefCount(dstkey);
                 decrRefCount(value);
@@ -316,7 +336,10 @@ void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) {
                          receiver->lastcmd->proc == bzpopminCommand)
                          ? ZSET_MIN : ZSET_MAX;
             unblockClient(receiver);
+            monotime replyTimer;
+            elapsedStart(&replyTimer);
             genericZpopCommand(receiver,&rl->key,1,where,1,NULL);
+            updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer));
             zcard--;
 
             /* Replicate the command. */
@@ -406,6 +429,8 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) {
                     }
                 }
 
+                monotime replyTimer;
+                elapsedStart(&replyTimer);
                 /* Emit the two elements sub-array consisting of
                  * the name of the stream and the data we
                  * extracted from it. Wrapped in a single-item
@@ -425,6 +450,7 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) {
                 streamReplyWithRange(receiver,s,&start,NULL,
                                      receiver->bpop.xread_count,
                                      0, group, consumer, noack, &pi);
+                updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer));
 
                 /* Note that after we unblock the client, 'gt'
                  * and other receiver->bpop stuff are no longer
@@ -471,7 +497,10 @@ void serveClientsBlockedOnKeyByModule(readyList *rl) {
              * different modules with different triggers to consider if a key
              * is ready or not. This means we can't exit the loop but need
              * to continue after the first failure. */
+            monotime replyTimer;
+            elapsedStart(&replyTimer);
             if (!moduleTryServeClientBlockedOnKey(receiver, rl->key)) continue;
+            updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer));
 
             moduleUnblockClient(receiver);
         }
diff --git a/src/module.c b/src/module.c
index f3191bbc7..b04595801 100644
--- a/src/module.c
+++ b/src/module.c
@@ -29,7 +29,9 @@
 
 #include "server.h"
 #include "cluster.h"
+#include "slowlog.h"
 #include "rdb.h"
+#include "monotonic.h"
 #include <dlfcn.h>
 #include <sys/stat.h>
 #include <sys/wait.h>
@@ -262,6 +264,9 @@ typedef struct RedisModuleBlockedClient {
     int dbid;           /* Database number selected by the original client. */
     int blocked_on_keys;    /* If blocked via RM_BlockClientOnKeys(). */
     int unblocked;          /* Already on the moduleUnblocked list. */
+    monotime background_timer; /* Timer tracking the start of background work */
+    uint64_t background_duration; /* Current command background time duration.
+                                     Used for measuring latency of blocking cmds */
 } RedisModuleBlockedClient;
 
 static pthread_mutex_t moduleUnblockedClientsMutex = PTHREAD_MUTEX_INITIALIZER;
@@ -915,6 +920,30 @@ long long RM_Milliseconds(void) {
     return mstime();
 }
 
+/* Mark a point in time that will be used as the start time to calculate
+ * the elapsed execution time when RM_BlockedClientMeasureTimeEnd() is called.
+ * Within the same command, you can call multiple times
+ * RM_BlockedClientMeasureTimeStart() and RM_BlockedClientMeasureTimeEnd()
+ * to accummulate indepedent time intervals to the background duration.
+ * This method always return REDISMODULE_OK. */
+int RM_BlockedClientMeasureTimeStart(RedisModuleBlockedClient *bc) {
+    elapsedStart(&(bc->background_timer));
+    return REDISMODULE_OK;
+}
+
+/* Mark a point in time that will be used as the end time
+ * to calculate the elapsed execution time.
+ * On success REDISMODULE_OK is returned.
+ * This method only returns REDISMODULE_ERR if no start time was
+ * previously defined ( meaning RM_BlockedClientMeasureTimeStart was not called ). */
+int RM_BlockedClientMeasureTimeEnd(RedisModuleBlockedClient *bc) {
+    // If the counter is 0 then we haven't called RM_BlockedClientMeasureTimeStart
+    if (!bc->background_timer)
+        return REDISMODULE_ERR;
+    bc->background_duration += elapsedUs(bc->background_timer);
+    return REDISMODULE_OK;
+}
+
 /* Set flags defining capabilities or behavior bit flags.
  *
  * REDISMODULE_OPTIONS_HANDLE_IO_ERRORS:
@@ -5091,6 +5120,7 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF
     bc->dbid = c->db->id;
     bc->blocked_on_keys = keys != NULL;
     bc->unblocked = 0;
+    bc->background_duration = 0;
     c->bpop.timeout = timeout;
 
     if (islua || ismulti) {
@@ -5164,6 +5194,11 @@ int moduleTryServeClientBlockedOnKey(client *c, robj *key) {
  *
  * In these cases, a call to RedisModule_BlockClient() will **not** block the
  * client, but instead produce a specific error reply.
+ *
+ * Measuring background time: By default the time spent in the blocked command
+ * is not account for the total command duration. To include such time you should
+ * use RM_BlockedClientMeasureTimeStart() and RM_BlockedClientMeasureTimeEnd() one,
+ * or multiple times within the blocking command background work.
  */
 RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms) {
     return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, NULL,0,NULL);
@@ -5358,6 +5393,7 @@ void moduleHandleBlockedClients(void) {
          * was blocked on keys (RM_BlockClientOnKeys()), because we already
          * called such callback in moduleTryServeClientBlockedOnKey() when
          * the key was signaled as ready. */
+        uint64_t reply_us = 0;
         if (c && !bc->blocked_on_keys && bc->reply_callback) {
             RedisModuleCtx ctx = REDISMODULE_CTX_INIT;
             ctx.flags |= REDISMODULE_CTX_BLOCKED_REPLY;
@@ -5366,9 +5402,19 @@ void moduleHandleBlockedClients(void) {
             ctx.module = bc->module;
             ctx.client = bc->client;
             ctx.blocked_client = bc;
+            monotime replyTimer;
+            elapsedStart(&replyTimer);
             bc->reply_callback(&ctx,(void**)c->argv,c->argc);
+            reply_us = elapsedUs(replyTimer);
             moduleFreeContext(&ctx);
         }
+        /* Update stats now that we've finished the blocking operation.
+         * This needs to be out of the reply callback above given that a
+         * module might not define any callback and still do blocking ops.
+         */
+        if (c && !bc->blocked_on_keys) {
+            updateStatsOnUnblock(c, bc->background_duration, reply_us);
+        }
 
         /* Free privdata if any. */
         if (bc->privdata && bc->free_privdata) {
@@ -5432,6 +5478,9 @@ void moduleBlockedClientTimedOut(client *c) {
     ctx.blocked_privdata = bc->privdata;
     bc->timeout_callback(&ctx,(void**)c->argv,c->argc);
     moduleFreeContext(&ctx);
+    if (!bc->blocked_on_keys) {
+        updateStatsOnUnblock(c, bc->background_duration, 0);
+    }
     /* For timeout events, we do not want to call the disconnect callback,
      * because the blocked client will be automatically disconnected in
      * this case, and the user can still hook using the timeout callback. */
@@ -9094,6 +9143,8 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(GetBlockedClientPrivateData);
     REGISTER_API(AbortBlock);
     REGISTER_API(Milliseconds);
+    REGISTER_API(BlockedClientMeasureTimeStart);
+    REGISTER_API(BlockedClientMeasureTimeEnd);
     REGISTER_API(GetThreadSafeContext);
     REGISTER_API(GetDetachedThreadSafeContext);
     REGISTER_API(FreeThreadSafeContext);
diff --git a/src/redismodule.h b/src/redismodule.h
index 5d46b22f9..9d8c6c5ea 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -776,6 +776,8 @@ REDISMODULE_API int (*RedisModule_IsBlockedTimeoutRequest)(RedisModuleCtx *ctx)
 REDISMODULE_API void * (*RedisModule_GetBlockedClientPrivateData)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
 REDISMODULE_API RedisModuleBlockedClient * (*RedisModule_GetBlockedClientHandle)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_AbortBlock)(RedisModuleBlockedClient *bc) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_BlockedClientMeasureTimeStart)(RedisModuleBlockedClient *bc) REDISMODULE_ATTR;
+REDISMODULE_API int (*RedisModule_BlockedClientMeasureTimeEnd)(RedisModuleBlockedClient *bc) REDISMODULE_ATTR;
 REDISMODULE_API RedisModuleCtx * (*RedisModule_GetThreadSafeContext)(RedisModuleBlockedClient *bc) REDISMODULE_ATTR;
 REDISMODULE_API RedisModuleCtx * (*RedisModule_GetDetachedThreadSafeContext)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
 REDISMODULE_API void (*RedisModule_FreeThreadSafeContext)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
@@ -1049,6 +1051,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(GetBlockedClientPrivateData);
     REDISMODULE_GET_API(GetBlockedClientHandle);
     REDISMODULE_GET_API(AbortBlock);
+    REDISMODULE_GET_API(BlockedClientMeasureTimeStart);
+    REDISMODULE_GET_API(BlockedClientMeasureTimeEnd);
     REDISMODULE_GET_API(SetDisconnectCallback);
     REDISMODULE_GET_API(SubscribeToKeyspaceEvents);
     REDISMODULE_GET_API(NotifyKeyspaceEvent);
diff --git a/src/server.c b/src/server.c
index 79d6d813d..faaca7215 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3620,7 +3620,7 @@ void preventCommandReplication(client *c) {
  */
 void call(client *c, int flags) {
     long long dirty;
-    ustime_t start, duration;
+    monotime call_timer;
     int client_old_flags = c->flags;
     struct redisCommand *real_cmd = c->cmd;
     static long long prev_err_count;
@@ -3646,9 +3646,10 @@ void call(client *c, int flags) {
     dirty = server.dirty;
     prev_err_count = server.stat_total_error_replies;
     updateCachedTime(0);
-    start = server.ustime;
+    elapsedStart(&call_timer);
     c->cmd->proc(c);
-    duration = ustime()-start;
+    const long duration = elapsedUs(call_timer);
+    c->duration = duration;
     dirty = server.dirty-dirty;
     if (dirty < 0) dirty = 0;
 
@@ -3692,7 +3693,10 @@ void call(client *c, int flags) {
          * arguments. */
         robj **argv = c->original_argv ? c->original_argv : c->argv;
         int argc = c->original_argv ? c->original_argc : c->argc;
-        slowlogPushEntryIfNeeded(c,argv,argc,duration);
+        /* If the client is blocked we will handle slowlog when it is unblocked . */
+        if (!(c->flags & CLIENT_BLOCKED)) {
+            slowlogPushEntryIfNeeded(c,argv,argc,duration);
+        }
     }
     freeClientOriginalArgv(c);
 
diff --git a/src/server.h b/src/server.h
index 1506c12a8..b293afcee 100644
--- a/src/server.h
+++ b/src/server.h
@@ -881,6 +881,7 @@ typedef struct client {
     size_t sentlen;         /* Amount of bytes already sent in the current
                                buffer or object being sent. */
     time_t ctime;           /* Client creation time. */
+    long duration;          /* Current command duration. Used for measuring latency of blocking/non-blocking cmds */
     time_t lastinteraction; /* Time of the last interaction, used for timeout */
     time_t obuf_soft_limit_reached_time;
     uint64_t flags;         /* Client flags: CLIENT_* macros. */
@@ -2405,6 +2406,7 @@ void disconnectAllBlockedClients(void);
 void handleClientsBlockedOnKeys(void);
 void signalKeyAsReady(redisDb *db, robj *key, int type);
 void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, struct listPos *listpos, streamID *ids);
+void updateStatsOnUnblock(client *c, long blocked_us, long reply_us);
 
 /* timeout.c -- Blocked clients timeout and connections timeout. */
 void addClientToTimeoutTable(client *c);
diff --git a/tests/modules/Makefile b/tests/modules/Makefile
index febed82d0..93b4b022f 100644
--- a/tests/modules/Makefile
+++ b/tests/modules/Makefile
@@ -19,6 +19,7 @@ TEST_MODULES = \
     misc.so \
     hooks.so \
     blockonkeys.so \
+    blockonbackground.so \
     scan.so \
     datatype.so \
     auth.so \
diff --git a/tests/modules/blockonbackground.c b/tests/modules/blockonbackground.c
new file mode 100644
index 000000000..cf7e9c7c1
--- /dev/null
+++ b/tests/modules/blockonbackground.c
@@ -0,0 +1,220 @@
+#define REDISMODULE_EXPERIMENTAL_API
+#include "redismodule.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <pthread.h>
+#include <time.h>
+#include "assert.h"
+
+#define UNUSED(x) (void)(x)
+
+/* Reply callback for blocking command BLOCK.DEBUG */
+int HelloBlock_Reply(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    UNUSED(argv);
+    UNUSED(argc);
+    int *myint = RedisModule_GetBlockedClientPrivateData(ctx);
+    return RedisModule_ReplyWithLongLong(ctx,*myint);
+}
+
+/* Timeout callback for blocking command BLOCK.DEBUG */
+int HelloBlock_Timeout(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    UNUSED(argv);
+    UNUSED(argc);
+    RedisModuleBlockedClient *bc = RedisModule_GetBlockedClientHandle(ctx);
+    assert(RedisModule_BlockedClientMeasureTimeEnd(bc)==REDISMODULE_OK);
+    return RedisModule_ReplyWithSimpleString(ctx,"Request timedout");
+}
+
+/* Private data freeing callback for BLOCK.DEBUG command. */
+void HelloBlock_FreeData(RedisModuleCtx *ctx, void *privdata) {
+    UNUSED(ctx);
+    RedisModule_Free(privdata);
+}
+
+/* The thread entry point that actually executes the blocking part
+ * of the command BLOCK.DEBUG. */
+void *BlockDebug_ThreadMain(void *arg) {
+    void **targ = arg;
+    RedisModuleBlockedClient *bc = targ[0];
+    long long delay = (unsigned long)targ[1];
+    long long enable_time_track = (unsigned long)targ[2];
+    if (enable_time_track)
+        assert(RedisModule_BlockedClientMeasureTimeStart(bc)==REDISMODULE_OK);
+    RedisModule_Free(targ);
+
+    struct timespec ts;
+    ts.tv_sec = delay / 1000;
+    ts.tv_nsec = (delay % 1000) * 1000000;
+    nanosleep(&ts, NULL);
+    int *r = RedisModule_Alloc(sizeof(int));
+    *r = rand();
+    if (enable_time_track)
+        assert(RedisModule_BlockedClientMeasureTimeEnd(bc)==REDISMODULE_OK);
+    RedisModule_UnblockClient(bc,r);
+    return NULL;
+}
+
+/* The thread entry point that actually executes the blocking part
+ * of the command BLOCK.DEBUG. */
+void *DoubleBlock_ThreadMain(void *arg) {
+    void **targ = arg;
+    RedisModuleBlockedClient *bc = targ[0];
+    long long delay = (unsigned long)targ[1];
+    assert(RedisModule_BlockedClientMeasureTimeStart(bc)==REDISMODULE_OK);
+    RedisModule_Free(targ);
+    struct timespec ts;
+    ts.tv_sec = delay / 1000;
+    ts.tv_nsec = (delay % 1000) * 1000000;
+    nanosleep(&ts, NULL);
+    int *r = RedisModule_Alloc(sizeof(int));
+    *r = rand();
+    RedisModule_BlockedClientMeasureTimeEnd(bc);
+    /* call again RedisModule_BlockedClientMeasureTimeStart() and
+     * RedisModule_BlockedClientMeasureTimeEnd and ensure that the
+     * total execution time is 2x the delay. */
+    assert(RedisModule_BlockedClientMeasureTimeStart(bc)==REDISMODULE_OK);
+    nanosleep(&ts, NULL);
+    RedisModule_BlockedClientMeasureTimeEnd(bc);
+
+    RedisModule_UnblockClient(bc,r);
+    return NULL;
+}
+
+void HelloBlock_Disconnected(RedisModuleCtx *ctx, RedisModuleBlockedClient *bc) {
+    RedisModule_Log(ctx,"warning","Blocked client %p disconnected!",
+        (void*)bc);
+}
+
+/* BLOCK.DEBUG <delay_ms> <timeout_ms> -- Block for <count> milliseconds, then reply with
+ * a random number. Timeout is the command timeout, so that you can test
+ * what happens when the delay is greater than the timeout. */
+int HelloBlock_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 3) return RedisModule_WrongArity(ctx);
+    long long delay;
+    long long timeout;
+
+    if (RedisModule_StringToLongLong(argv[1],&delay) != REDISMODULE_OK) {
+        return RedisModule_ReplyWithError(ctx,"ERR invalid count");
+    }
+
+    if (RedisModule_StringToLongLong(argv[2],&timeout) != REDISMODULE_OK) {
+        return RedisModule_ReplyWithError(ctx,"ERR invalid count");
+    }
+
+    pthread_t tid;
+    RedisModuleBlockedClient *bc = RedisModule_BlockClient(ctx,HelloBlock_Reply,HelloBlock_Timeout,HelloBlock_FreeData,timeout);
+
+    /* Here we set a disconnection handler, however since this module will
+     * block in sleep() in a thread, there is not much we can do in the
+     * callback, so this is just to show you the API. */
+    RedisModule_SetDisconnectCallback(bc,HelloBlock_Disconnected);
+
+    /* Now that we setup a blocking client, we need to pass the control
+     * to the thread. However we need to pass arguments to the thread:
+     * the delay and a reference to the blocked client handle. */
+    void **targ = RedisModule_Alloc(sizeof(void*)*3);
+    targ[0] = bc;
+    targ[1] = (void*)(unsigned long) delay;
+    // pass 1 as flag to enable time tracking
+    targ[2] = (void*)(unsigned long) 1;
+
+    if (pthread_create(&tid,NULL,BlockDebug_ThreadMain,targ) != 0) {
+        RedisModule_AbortBlock(bc);
+        return RedisModule_ReplyWithError(ctx,"-ERR Can't start thread");
+    }
+    return REDISMODULE_OK;
+}
+
+/* BLOCK.DEBUG_NOTRACKING <delay_ms> <timeout_ms> -- Block for <count> milliseconds, then reply with
+ * a random number. Timeout is the command timeout, so that you can test
+ * what happens when the delay is greater than the timeout.
+ * this command does not track background time so the background time should no appear in stats*/
+int HelloBlockNoTracking_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 3) return RedisModule_WrongArity(ctx);
+    long long delay;
+    long long timeout;
+
+    if (RedisModule_StringToLongLong(argv[1],&delay) != REDISMODULE_OK) {
+        return RedisModule_ReplyWithError(ctx,"ERR invalid count");
+    }
+
+    if (RedisModule_StringToLongLong(argv[2],&timeout) != REDISMODULE_OK) {
+        return RedisModule_ReplyWithError(ctx,"ERR invalid count");
+    }
+
+    pthread_t tid;
+    RedisModuleBlockedClient *bc = RedisModule_BlockClient(ctx,HelloBlock_Reply,HelloBlock_Timeout,HelloBlock_FreeData,timeout);
+
+    /* Here we set a disconnection handler, however since this module will
+     * block in sleep() in a thread, there is not much we can do in the
+     * callback, so this is just to show you the API. */
+    RedisModule_SetDisconnectCallback(bc,HelloBlock_Disconnected);
+
+    /* Now that we setup a blocking client, we need to pass the control
+     * to the thread. However we need to pass arguments to the thread:
+     * the delay and a reference to the blocked client handle. */
+    void **targ = RedisModule_Alloc(sizeof(void*)*3);
+    targ[0] = bc;
+    targ[1] = (void*)(unsigned long) delay;
+    // pass 0 as flag to enable time tracking
+    targ[2] = (void*)(unsigned long) 0;
+
+    if (pthread_create(&tid,NULL,BlockDebug_ThreadMain,targ) != 0) {
+        RedisModule_AbortBlock(bc);
+        return RedisModule_ReplyWithError(ctx,"-ERR Can't start thread");
+    }
+    return REDISMODULE_OK;
+}
+
+/* BLOCK.DOUBLE_DEBUG <delay_ms> -- Block for 2 x <count> milliseconds,
+ * then reply with a random number.
+ * This command is used to test multiple calls to RedisModule_BlockedClientMeasureTimeStart()
+ * and RedisModule_BlockedClientMeasureTimeEnd() within the same execution. */
+int HelloDoubleBlock_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 2) return RedisModule_WrongArity(ctx);
+    long long delay;
+    long long timeout;
+
+    if (RedisModule_StringToLongLong(argv[1],&delay) != REDISMODULE_OK) {
+        return RedisModule_ReplyWithError(ctx,"ERR invalid count");
+    }
+
+    pthread_t tid;
+    RedisModuleBlockedClient *bc = RedisModule_BlockClient(ctx,HelloBlock_Reply,HelloBlock_Timeout,HelloBlock_FreeData,timeout);
+
+    /* Now that we setup a blocking client, we need to pass the control
+     * to the thread. However we need to pass arguments to the thread:
+     * the delay and a reference to the blocked client handle. */
+    void **targ = RedisModule_Alloc(sizeof(void*)*2);
+    targ[0] = bc;
+    targ[1] = (void*)(unsigned long) delay;
+
+    if (pthread_create(&tid,NULL,DoubleBlock_ThreadMain,targ) != 0) {
+        RedisModule_AbortBlock(bc);
+        return RedisModule_ReplyWithError(ctx,"-ERR Can't start thread");
+    }
+    return REDISMODULE_OK;
+}
+
+
+int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    UNUSED(argv);
+    UNUSED(argc);
+
+    if (RedisModule_Init(ctx,"block",1,REDISMODULE_APIVER_1)
+        == REDISMODULE_ERR) return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx,"block.debug",
+        HelloBlock_RedisCommand,"",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx,"block.double_debug",
+        HelloDoubleBlock_RedisCommand,"",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    if (RedisModule_CreateCommand(ctx,"block.debug_no_track",
+        HelloBlockNoTracking_RedisCommand,"",0,0,0) == REDISMODULE_ERR)
+        return REDISMODULE_ERR;
+
+    return REDISMODULE_OK;
+}
diff --git a/tests/unit/moduleapi/blockonbackground.tcl b/tests/unit/moduleapi/blockonbackground.tcl
new file mode 100644
index 000000000..23111ab73
--- /dev/null
+++ b/tests/unit/moduleapi/blockonbackground.tcl
@@ -0,0 +1,67 @@
+set testmodule [file normalize tests/modules/blockonbackground.so]
+
+source tests/support/util.tcl
+
+start_server {tags {"modules"}} {
+    r module load $testmodule
+
+    test { blocked clients time tracking - check blocked command that uses RedisModule_BlockedClientMeasureTimeStart() is tracking background time} {
+        r slowlog reset
+        r config set slowlog-log-slower-than 200000
+        assert_equal [r slowlog len] 0
+        r block.debug 0 10000
+        assert_equal [r slowlog len] 0
+        r config resetstat
+        r block.debug 200 10000
+        assert_equal [r slowlog len] 1
+
+        set cmdstatline [cmdrstat block.debug r]
+
+        regexp "calls=1,usec=(.*?),usec_per_call=(.*?),rejected_calls=0,failed_calls=0" $cmdstatline usec usec_per_call
+        assert {$usec >= 100000}
+        assert {$usec_per_call >= 100000}
+    }
+
+    test { blocked clients time tracking - check blocked command that uses RedisModule_BlockedClientMeasureTimeStart() is tracking background time even in timeout } {
+        r slowlog reset
+        r config set slowlog-log-slower-than 200000
+        assert_equal [r slowlog len] 0
+        r block.debug 0 20000
+        assert_equal [r slowlog len] 0
+        r config resetstat
+        r block.debug 20000 200
+        assert_equal [r slowlog len] 1
+
+        set cmdstatline [cmdrstat block.debug r]
+
+        regexp "calls=1,usec=(.*?),usec_per_call=(.*?),rejected_calls=0,failed_calls=0" $cmdstatline usec usec_per_call
+        assert {$usec >= 100000}
+        assert {$usec_per_call >= 100000}
+    }
+
+    test { blocked clients time tracking - check blocked command with multiple calls RedisModule_BlockedClientMeasureTimeStart()  is tracking the total background time } {
+        r slowlog reset
+        r config set slowlog-log-slower-than 200000
+        assert_equal [r slowlog len] 0
+        r block.double_debug 0
+        assert_equal [r slowlog len] 0
+        r config resetstat
+        r block.double_debug 100
+        assert_equal [r slowlog len] 1
+
+        set cmdstatline [cmdrstat block.double_debug r]
+
+        regexp "calls=1,usec=(.*?),usec_per_call=(.*?),rejected_calls=0,failed_calls=0" $cmdstatline usec usec_per_call
+        assert {$usec >= 60000}
+        assert {$usec_per_call >= 60000}
+    }
+
+    test { blocked clients time tracking - check blocked command without calling RedisModule_BlockedClientMeasureTimeStart() is not reporting background time } {
+        r slowlog reset
+        r config set slowlog-log-slower-than 200000
+        assert_equal [r slowlog len] 0
+        r block.debug_no_track 200 1000
+        # ensure slowlog is still empty
+        assert_equal [r slowlog len] 0
+    }
+}

From eacccd2acbbebde3e50e14a4d0ade5b1ccc2bc07 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Sat, 30 Jan 2021 04:18:58 -0500
Subject: [PATCH 1018/1098] fix sentinel tests error (#8422)

This commit fixes sentinel announces hostnames test error in certain linux environment
Before this commit, we only check localhost is resolved into 127.0.0.1, however in ubuntu
or some other linux environments "localhost" will be resolved into ::1 ipv6 address first if
the network stack is capable.
---
 tests/sentinel/tests/08-hostname-conf.tcl | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/tests/sentinel/tests/08-hostname-conf.tcl b/tests/sentinel/tests/08-hostname-conf.tcl
index 45bb56687..be6e42cb0 100644
--- a/tests/sentinel/tests/08-hostname-conf.tcl
+++ b/tests/sentinel/tests/08-hostname-conf.tcl
@@ -33,13 +33,18 @@ source "../tests/includes/init-tests.tcl"
 proc verify_hostname_announced {hostname} {
     foreach_sentinel_id id {
         # Master is reported with its hostname
-        assert_equal [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 0] $hostname
+        if {![string equal [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 0] $hostname]} {
+            return 0
+        }
 
         # Replicas are reported with their hostnames
         foreach replica [S $id SENTINEL REPLICAS mymaster] {
-            assert_equal [dict get $replica ip] $hostname
+            if {![string equal [dict get $replica ip] $hostname]} {
+                return 0
+            }
         }
     }
+    return 1
 }
 
 test "Sentinel announces hostnames" {
@@ -48,7 +53,7 @@ test "Sentinel announces hostnames" {
 
     # Disable announce-hostnames and confirm IPs are used
     set_sentinel_config announce-hostnames no
-    verify_hostname_announced "127.0.0.1"
+    assert {[verify_hostname_announced "127.0.0.1"] || [verify_hostname_announced "::1"]}
 }
 
 # We need to revert any special configuration because all tests currently

From 5a7eb9c8810980c4a3e453d9f9fa6d19ab832062 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 31 Jan 2021 12:13:45 +0200
Subject: [PATCH 1019/1098] Fix test issues from introduction of HRANDFIELD
 (#8424)

* The corrupt dump fuzzer found a division by zero.
* in some cases the random fields from the HRANDFIELD tests produced
  fields with newlines and other special chars (due to \ char), this caused
  the TCL tests to see a bulk response that has a newline in it and add {}
  around it, later it can think this is a nested list. in fact the `alpha` random
  string generator isn't using spaces and newlines, so it should not use `\`
  either.
---
 src/ziplist.c                      |  6 ++++++
 tests/integration/corrupt-dump.tcl | 11 +++++++++++
 tests/support/util.tcl             |  6 +++++-
 3 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/ziplist.c b/src/ziplist.c
index 62ffcb93d..0cd20630a 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -1506,6 +1506,9 @@ void ziplistRandomPair(unsigned char *zl, unsigned long total_count, ziplistEntr
     int ret;
     unsigned char *p;
 
+    /* Avoid div by zero on corrupt ziplist */
+    assert(total_count);
+
     /* Generate even numbers, because ziplist saved K-V pair */
     int r = (rand() % total_count) * 2;
     p = ziplistIndex(zl, r);
@@ -1545,6 +1548,9 @@ void ziplistRandomPairs(unsigned char *zl, int count, ziplistEntry *keys, ziplis
     rand_pick *picks = zmalloc(sizeof(rand_pick)*count);
     unsigned long total_size = ziplistLen(zl)/2;
 
+    /* Avoid div by zero on corrupt ziplist */
+    assert(total_size);
+
     /* create a pool of random indexes (some may be duplicate). */
     for (int i = 0; i < count; i++) {
         picks[i].index = (rand() % total_size) * 2; /* Generate even indexes */
diff --git a/tests/integration/corrupt-dump.tcl b/tests/integration/corrupt-dump.tcl
index cc597bb4d..f5079e5ed 100644
--- a/tests/integration/corrupt-dump.tcl
+++ b/tests/integration/corrupt-dump.tcl
@@ -507,5 +507,16 @@ test {corrupt payload: fuzzer findings - valgrind invalid read} {
     }
 }
 
+test {corrupt payload: fuzzer findings - HRANDFIELD on bad ziplist} {
+    start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
+        r config set sanitize-dump-payload yes
+        r debug set-skip-checksum-validation 1
+        r RESTORE _int 0 "\x04\xC0\x01\x09\x00\xF6\x8A\xB6\x7A\x85\x87\x72\x4D"
+        catch {r HRANDFIELD _int}
+        assert_equal [count_log_message 0 "crashed by signal"] 0
+        assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
+    }
+}
+
 } ;# tags
 
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index 943f45a2b..80f8598ce 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -12,7 +12,11 @@ proc randstring {min max {type binary}} {
         set maxval 52
     }
     while {$len} {
-        append output [format "%c" [expr {$minval+int(rand()*($maxval-$minval+1))}]]
+        set rr [expr {$minval+int(rand()*($maxval-$minval+1))}]
+        if {$type eq {alpha} && $rr eq 92} {
+            set rr 90; # avoid putting '\' char in the string, it can mess up TCL processing
+        }
+        append output [format "%c" $rr]
         incr len -1
     }
     return $output

From 8f27578de294aca6e94943976943bc5224e39fd1 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 31 Jan 2021 12:14:36 +0200
Subject: [PATCH 1020/1098] temporarily disable sentinel test FD leak print
 (#8425)

These tests are not yet stable. on github actions they show some false leaks.
---
 tests/instances.tcl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/instances.tcl b/tests/instances.tcl
index b9d1b3fe5..8cb616ae8 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -431,9 +431,9 @@ proc run_tests {} {
 proc end_tests {} {
     set sentinel_fd_leaks_file "sentinel_fd_leaks"
     if { [file exists $sentinel_fd_leaks_file] } {
-        puts [colorstr red "WARNING: sentinel test(s) failed, there are leaked fds in sentinel:"] 
-        puts [exec cat $sentinel_fd_leaks_file]
         # temporarily disabling this error from failing the tests until leaks are fixed.
+        #puts [colorstr red "WARNING: sentinel test(s) failed, there are leaked fds in sentinel:"] 
+        #puts [exec cat $sentinel_fd_leaks_file]
         #exit 1
     }
 

From b57d0eb418dd8622bfc054239320ad7de9ccd752 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 31 Jan 2021 12:16:58 +0200
Subject: [PATCH 1021/1098] update help.h with new commands (#8426)

---
 src/help.h | 159 ++++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 132 insertions(+), 27 deletions(-)

diff --git a/src/help.h b/src/help.h
index 8a12458e1..b8b1efb95 100644
--- a/src/help.h
+++ b/src/help.h
@@ -1,4 +1,4 @@
-/* Automatically generated by generate-command-help.rb, do not edit. */
+/* Automatically generated by ./utils/generate-command-help.rb, do not edit. */
 
 #ifndef __REDIS_HELP_H
 #define __REDIS_HELP_H
@@ -94,7 +94,7 @@ struct commandHelp {
     1,
     "2.0.0" },
     { "AUTH",
-    "password",
+    "[username] password",
     "Authenticate to the server",
     8,
     "1.0.0" },
@@ -128,6 +128,11 @@ struct commandHelp {
     "Find first bit set or clear in a string",
     1,
     "2.8.7" },
+    { "BLMOVE",
+    "source destination LEFT|RIGHT LEFT|RIGHT timeout",
+    "Pop an element from a list, push it to another list and return it; or block until one is available",
+    2,
+    "6.2.0" },
     { "BLPOP",
     "key [key ...] timeout",
     "Remove and get the first element in a list, or block until one is available",
@@ -173,18 +178,23 @@ struct commandHelp {
     "Returns the client ID for the current connection",
     8,
     "5.0.0" },
+    { "CLIENT INFO",
+    "-",
+    "Returns information about the current client connection.",
+    8,
+    "6.2.0" },
     { "CLIENT KILL",
     "[ip:port] [ID client-id] [TYPE normal|master|slave|pubsub] [USER username] [ADDR ip:port] [SKIPME yes/no]",
     "Kill the connection of a client",
     8,
     "2.4.0" },
     { "CLIENT LIST",
-    "[TYPE normal|master|replica|pubsub]",
+    "[TYPE normal|master|replica|pubsub] [ID client-id [client-id ...]]",
     "Get the list of client connections",
     8,
     "2.4.0" },
     { "CLIENT PAUSE",
-    "timeout",
+    "timeout [WRITE|ALL]",
     "Stop processing commands from clients for some time",
     8,
     "2.9.50" },
@@ -203,11 +213,21 @@ struct commandHelp {
     "Enable or disable server assisted client side caching support",
     8,
     "6.0.0" },
+    { "CLIENT TRACKINGINFO",
+    "-",
+    "Return information about server assisted client side caching for the current connection",
+    8,
+    "6.2.0" },
     { "CLIENT UNBLOCK",
     "client-id [TIMEOUT|ERROR]",
     "Unblock a client blocked in a blocking command from a different connection",
     8,
     "5.0.0" },
+    { "CLIENT UNPAUSE",
+    "-",
+    "Resume processing of clients that were paused",
+    8,
+    "6.2.0" },
     { "CLUSTER ADDSLOTS",
     "slot [slot ...]",
     "Assign new hash slots to receiving node",
@@ -358,6 +378,11 @@ struct commandHelp {
     "Set a configuration parameter to the given value",
     9,
     "2.0.0" },
+    { "COPY",
+    "source destination [DB destination-db] [REPLACE]",
+    "Copy a key",
+    0,
+    "6.2.0" },
     { "DBSIZE",
     "-",
     "Return the number of keys in the selected database",
@@ -434,17 +459,17 @@ struct commandHelp {
     0,
     "1.2.0" },
     { "FLUSHALL",
-    "[ASYNC]",
+    "[ASYNC|SYNC]",
     "Remove all keys from all databases",
     9,
     "1.0.0" },
     { "FLUSHDB",
-    "[ASYNC]",
+    "[ASYNC|SYNC]",
     "Remove all keys from the current database",
     9,
     "1.0.0" },
     { "GEOADD",
-    "key longitude latitude member [longitude latitude member ...]",
+    "key [NX|XX] [CH] longitude latitude member [longitude latitude member ...]",
     "Add one or more geospatial items in the geospatial index represented using a sorted set",
     13,
     "3.2.0" },
@@ -464,15 +489,25 @@ struct commandHelp {
     13,
     "3.2.0" },
     { "GEORADIUS",
-    "key longitude latitude radius m|km|ft|mi [WITHCOORD] [WITHDIST] [WITHHASH] [COUNT count] [ASC|DESC] [STORE key] [STOREDIST key]",
+    "key longitude latitude radius m|km|ft|mi [WITHCOORD] [WITHDIST] [WITHHASH] [COUNT count [ANY]] [ASC|DESC] [STORE key] [STOREDIST key]",
     "Query a sorted set representing a geospatial index to fetch members matching a given maximum distance from a point",
     13,
     "3.2.0" },
     { "GEORADIUSBYMEMBER",
-    "key member radius m|km|ft|mi [WITHCOORD] [WITHDIST] [WITHHASH] [COUNT count] [ASC|DESC] [STORE key] [STOREDIST key]",
+    "key member radius m|km|ft|mi [WITHCOORD] [WITHDIST] [WITHHASH] [COUNT count [ANY]] [ASC|DESC] [STORE key] [STOREDIST key]",
     "Query a sorted set representing a geospatial index to fetch members matching a given maximum distance from a member",
     13,
     "3.2.0" },
+    { "GEOSEARCH",
+    "key [FROMMEMBER member] [FROMLONLAT longitude latitude] [BYRADIUS radius m|km|ft|mi] [BYBOX width height m|km|ft|mi] [ASC|DESC] [COUNT count [ANY]] [WITHCOORD] [WITHDIST] [WITHHASH]",
+    "Query a sorted set representing a geospatial index to fetch members inside an area of a box or a circle.",
+    13,
+    "6.2" },
+    { "GEOSEARCHSTORE",
+    "destination source [FROMMEMBER member] [FROMLONLAT longitude latitude] [BYRADIUS radius m|km|ft|mi] [BYBOX width height m|km|ft|mi] [ASC|DESC] [COUNT count [ANY]] [WITHCOORD] [WITHDIST] [WITHHASH] [STOREDIST]",
+    "Query a sorted set representing a geospatial index to fetch members inside an area of a box or a circle, and store the result in another key.",
+    13,
+    "6.2" },
     { "GET",
     "key",
     "Get the value of a key",
@@ -483,6 +518,16 @@ struct commandHelp {
     "Returns the bit value at offset in the string value stored at key",
     1,
     "2.2.0" },
+    { "GETDEL",
+    "key",
+    "Get the value of a key and delete the key",
+    1,
+    "6.2.0" },
+    { "GETEX",
+    "key [EX seconds|PX milliseconds|EXAT timestamp|PXAT milliseconds-timestamp|PERSIST]",
+    "Get the value of a key and optionally set its expiration",
+    1,
+    "6.2.0" },
     { "GETRANGE",
     "key start end",
     "Get a substring of the string stored at a key",
@@ -499,8 +544,8 @@ struct commandHelp {
     5,
     "2.0.0" },
     { "HELLO",
-    "protover [AUTH username password] [SETNAME clientname]",
-    "switch Redis protocol",
+    "[protover [AUTH username password] [SETNAME clientname]]",
+    "Handshake with Redis",
     8,
     "6.0.0" },
     { "HEXISTS",
@@ -548,6 +593,11 @@ struct commandHelp {
     "Set multiple hash fields to multiple values",
     5,
     "2.0.0" },
+    { "HRANDFIELD",
+    "key [count [WITHVALUES]]",
+    "Get one or multiple random fields from a hash",
+    5,
+    "6.2.0" },
     { "HSCAN",
     "key cursor [MATCH pattern] [COUNT count]",
     "Incrementally iterate hash fields and associated values",
@@ -648,18 +698,23 @@ struct commandHelp {
     "Get the length of a list",
     2,
     "1.0.0" },
+    { "LMOVE",
+    "source destination LEFT|RIGHT LEFT|RIGHT",
+    "Pop an element from a list, push it to another list and return it",
+    2,
+    "6.2.0" },
     { "LOLWUT",
     "[VERSION version]",
     "Display some computer art and the Redis version",
     9,
     "5.0.0" },
     { "LPOP",
-    "key",
-    "Remove and get the first element in a list",
+    "key [count]",
+    "Remove and get the first elements in a list",
     2,
     "1.0.0" },
     { "LPOS",
-    "key element [FIRST rank] [COUNT num-matches] [MAXLEN len]",
+    "key element [RANK rank] [COUNT num-matches] [MAXLEN len]",
     "Return the index of matching elements on a list",
     2,
     "6.0.6" },
@@ -729,7 +784,7 @@ struct commandHelp {
     1,
     "1.0.0" },
     { "MIGRATE",
-    "host port key|"" destination-db timeout [COPY] [REPLACE] [AUTH password] [KEYS key]",
+    "host port key|"" destination-db timeout [COPY] [REPLACE] [AUTH password] [AUTH2 username password] [KEYS key]",
     "Atomically transfer a key from a Redis instance to another one.",
     0,
     "2.6.0" },
@@ -883,6 +938,11 @@ struct commandHelp {
     "Make the server a replica of another instance, or promote it as master.",
     9,
     "5.0.0" },
+    { "RESET",
+    "-",
+    "Reset the connection",
+    8,
+    "6.2" },
     { "RESTORE",
     "key ttl serialized-value [REPLACE] [ABSTTL] [IDLETIME seconds] [FREQ frequency]",
     "Create a key using the provided serialized value, previously obtained using DUMP.",
@@ -894,8 +954,8 @@ struct commandHelp {
     9,
     "2.8.12" },
     { "RPOP",
-    "key",
-    "Remove and get the last element in a list",
+    "key [count]",
+    "Remove and get the last elements in a list",
     2,
     "1.0.0" },
     { "RPOPLPUSH",
@@ -944,7 +1004,7 @@ struct commandHelp {
     10,
     "2.6.0" },
     { "SCRIPT FLUSH",
-    "-",
+    "[ASYNC|SYNC]",
     "Remove all the scripts from the script cache.",
     10,
     "2.6.0" },
@@ -974,7 +1034,7 @@ struct commandHelp {
     8,
     "1.0.0" },
     { "SET",
-    "key value [EX seconds|PX milliseconds] [NX|XX] [KEEPTTL]",
+    "key value [EX seconds|PX milliseconds|EXAT timestamp|PXAT milliseconds-timestamp|KEEPTTL] [NX|XX] [GET]",
     "Set the string value of a key",
     1,
     "1.0.0" },
@@ -1033,6 +1093,11 @@ struct commandHelp {
     "Get all the members in a set",
     3,
     "1.0.0" },
+    { "SMISMEMBER",
+    "key member [member ...]",
+    "Returns the membership associated with the given elements for a set",
+    3,
+    "6.2.0" },
     { "SMOVE",
     "source destination member",
     "Move a member from one set to another",
@@ -1149,10 +1214,15 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XADD",
-    "key ID field value [field value ...]",
+    "key [NOMKSTREAM] [MAXLEN|MINID [=|~] threshold [LIMIT count]] *|ID field value [field value ...]",
     "Appends a new entry to a stream",
     14,
     "5.0.0" },
+    { "XAUTOCLAIM",
+    "key group consumer min-idle-time start [COUNT count] [justid]",
+    "Changes (or acquires) ownership of messages in a consumer group, as if the messages were delivered to the specified consumer.",
+    14,
+    "6.2.0" },
     { "XCLAIM",
     "key group consumer min-idle-time ID [ID ...] [IDLE ms] [TIME ms-unix-time] [RETRYCOUNT count] [force] [justid]",
     "Changes (or acquires) ownership of a message in a consumer group, as if the message was delivered to the specified consumer.",
@@ -1164,7 +1234,7 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XGROUP",
-    "[CREATE key groupname id-or-$] [SETID key groupname id-or-$] [DESTROY key groupname] [DELCONSUMER key groupname consumername]",
+    "[CREATE key groupname ID|$ [MKSTREAM]] [SETID key groupname ID|$] [DESTROY key groupname] [CREATECONSUMER key groupname consumername] [DELCONSUMER key groupname consumername]",
     "Create, destroy, and manage consumer groups.",
     14,
     "5.0.0" },
@@ -1179,7 +1249,7 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XPENDING",
-    "key group [start end count] [consumer]",
+    "key group [[IDLE min-idle-time] start end count [consumer]]",
     "Return information and entries from a stream consumer group pending entries list, that are messages fetched but never acknowledged.",
     14,
     "5.0.0" },
@@ -1189,7 +1259,7 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XREAD",
-    "[COUNT count] [BLOCK milliseconds] STREAMS key [key ...] id [id ...]",
+    "[COUNT count] [BLOCK milliseconds] STREAMS key [key ...] ID [ID ...]",
     "Return never seen elements in multiple streams, with IDs greater than the ones reported by the caller for each stream. Can block.",
     14,
     "5.0.0" },
@@ -1204,12 +1274,12 @@ struct commandHelp {
     14,
     "5.0.0" },
     { "XTRIM",
-    "key MAXLEN [~] count",
+    "key MAXLEN|MINID [=|~] threshold [LIMIT count]",
     "Trims the stream to (approximately if '~' is passed) a certain size",
     14,
     "5.0.0" },
     { "ZADD",
-    "key [NX|XX] [CH] [INCR] score member [score member ...]",
+    "key [NX|XX] [GT|LT] [CH] [INCR] score member [score member ...]",
     "Add one or more members to a sorted set, or update its score if it already exists",
     4,
     "1.2.0" },
@@ -1223,11 +1293,26 @@ struct commandHelp {
     "Count the members in a sorted set with scores within the given values",
     4,
     "2.0.0" },
+    { "ZDIFF",
+    "numkeys key [key ...] [WITHSCORES]",
+    "Subtract multiple sorted sets",
+    4,
+    "6.2.0" },
+    { "ZDIFFSTORE",
+    "destination numkeys key [key ...]",
+    "Subtract multiple sorted sets and store the resulting sorted set in a new key",
+    4,
+    "6.2.0" },
     { "ZINCRBY",
     "key increment member",
     "Increment the score of a member in a sorted set",
     4,
     "1.2.0" },
+    { "ZINTER",
+    "numkeys key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX] [WITHSCORES]",
+    "Intersect multiple sorted sets",
+    4,
+    "6.2.0" },
     { "ZINTERSTORE",
     "destination numkeys key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX]",
     "Intersect multiple sorted sets and store the resulting sorted set in a new key",
@@ -1238,6 +1323,11 @@ struct commandHelp {
     "Count the number of members in a sorted set between a given lexicographical range",
     4,
     "2.8.9" },
+    { "ZMSCORE",
+    "key member [member ...]",
+    "Get the score associated with the given members in a sorted set",
+    4,
+    "6.2.0" },
     { "ZPOPMAX",
     "key [count]",
     "Remove and return members with the highest scores in a sorted set",
@@ -1248,9 +1338,14 @@ struct commandHelp {
     "Remove and return members with the lowest scores in a sorted set",
     4,
     "5.0.0" },
+    { "ZRANDMEMBER",
+    "key [count [WITHSCORES]]",
+    "Get one or multiple random elements from a sorted set",
+    4,
+    "6.2.0" },
     { "ZRANGE",
-    "key start stop [WITHSCORES]",
-    "Return a range of members in a sorted set, by index",
+    "key min max [BYSCORE|BYLEX] [REV] [LIMIT offset count] [WITHSCORES]",
+    "Return a range of members in a sorted set",
     4,
     "1.2.0" },
     { "ZRANGEBYLEX",
@@ -1263,6 +1358,11 @@ struct commandHelp {
     "Return a range of members in a sorted set, by score",
     4,
     "1.0.5" },
+    { "ZRANGESTORE",
+    "dst src min max [BYSCORE|BYLEX] [REV] [LIMIT offset count]",
+    "Store a range of members from sorted set into another key",
+    4,
+    "6.2.0" },
     { "ZRANK",
     "key member",
     "Determine the index of a member in a sorted set",
@@ -1318,6 +1418,11 @@ struct commandHelp {
     "Get the score associated with the given member in a sorted set",
     4,
     "1.2.0" },
+    { "ZUNION",
+    "numkeys key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX] [WITHSCORES]",
+    "Add multiple sorted sets",
+    4,
+    "6.2.0" },
     { "ZUNIONSTORE",
     "destination numkeys key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX]",
     "Add multiple sorted sets and store the resulting sorted set in a new key",

From 95338f9cc41fdfd050f122789187db75fda1fe3c Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 31 Jan 2021 17:52:50 +0200
Subject: [PATCH 1022/1098] Redis 6.2 RC3

---
 00-RELEASENOTES | 70 +++++++++++++++++++++++++++++++++++++++++++------
 src/version.h   |  4 +--
 2 files changed, 64 insertions(+), 10 deletions(-)

diff --git a/00-RELEASENOTES b/00-RELEASENOTES
index 3434d4838..f52a22ed9 100644
--- a/00-RELEASENOTES
+++ b/00-RELEASENOTES
@@ -1,3 +1,53 @@
+Redis 6.2 RC3   Released Tue Feb 1 14:00:00 IST 2021
+================================================================================
+
+Upgrade urgency LOW: This is the third Release Candidate of Redis 6.2.
+
+Here is a comprehensive list of changes in this release compared to 6.2 RC2,
+each one includes the PR number that added it, so you can get more details
+at https://github.com/redis/redis/pull/<number>
+
+New commands / args:
+* Add HRANDFIELD and ZRANDMEMBER commands (#8297)
+* Add FAILOVER command (#8315)
+* Add GETEX, GETDEL commands (#8327)
+* Add PXAT/EXAT arguments to SET command (#8327)
+* Add SYNC arg to FLUSHALL and FLUSHDB, and ASYNC/SYNC arg to SCRIPT FLUSH (#8258)
+
+Sentinel:
+* Add hostname support to Sentinel (#8282)
+* Prevent file descriptors from leaking into Sentinel scripts (#8242)
+* Fix config file line order dependency and config rewrite sequence (#8271)
+
+New configuration options:
+* Add set-proc-title config option to disable changes to the process title (#3623)
+* Add proc-title-template option to control what's shown in the process title (#8397)
+* Add lazyfree-lazy-user-flush config option to control FLUSHALL, FLUSHDB and SCRIPT FLUSH (#8258)
+
+Bug fixes:
+* AOF: recover from last write error by turning on/off appendonly config (#8030)
+* Exit on fsync error when the AOF fsync policy is 'always' (#8347)
+* Avoid assertions (on older kernels) when testing arm64 CoW bug (#8405)
+* CONFIG REWRITE should honor umask settings (#8371)
+* Fix firstkey,lastkey,step in COMMAND command for some commands (#8367)
+
+Special considerations:
+* Fix misleading description of the save configuration directive (#8337)
+
+Improvements:
+* A way to get RDB file via replication without excessive replication buffers (#8303)
+* Optimize performance of clusterGenNodesDescription for large clusters (#8182)
+
+Info fields and introspection changes:
+* SLOWLOG and LATENCY monitor include unblocking time of blocked commands (#7491)
+
+Modules:
+* Add modules API for streams (#8288)
+* Add event for fork child birth and termination (#8289)
+* Add RM_BlockedClientMeasureTime* etc, to track background processing in commandstats (#7491)
+* Fix bug in v6.2, wrong value passed to the new unlink callback (#8381)
+* Fix bug in v6.2, modules blocked on keys unblock on commands like LPUSH (#8356)
+
 ================================================================================
 Redis 6.2 RC2   Released Tue Jan 12 16:17:20 IST 2021
 ================================================================================
@@ -255,35 +305,39 @@ and we don't get reports of serious issues for a while.
 A special thank you for the amount of work put into this release by:
 - Oran Agra
 - Yossi Gottlieb
-- Itamar Haber
-- Guy Benoish
 - Filipe Oliveira
+- Viktor Söderqvist
+- Guy Benoish
+- Itamar Haber
+- Yang Bodong
 - Madelyn Olson
 - Wang Yuan
 - Felipe Machado
-- Yang Bodong
+- Wen Hui
 - Tatsuya Arisawa
 - Jonah H. Harris
+- Raghav Muddur
 - Jim Brunner
 - Yaacov Hazan
-- Wen Hui
+- Allen Farris
 - Chen Yang
 - Nitai Caro
 - Meir Shpilraien
 - maohuazhu
 - Valentino Geron
-- Qu Chen
-- sundb
-- George Prekas
 - Zhao Zhao
+- sundb
+- Qu Chen
+- George Prekas
 - Tyson Andre
 - Michael Grunder
 - alexronke-channeladvisor
+- Andy Pan
 - Wu Yunlong
 - Wei Kukey
 - Yoav Steinberg
-- Uri Shachar
 - Greg Femec
+- Uri Shachar
 - Nykolas Laurentino de Lima
 - xhe
 - zhenwei pi
diff --git a/src/version.h b/src/version.h
index d408dd3e8..1c8c1f2a6 100644
--- a/src/version.h
+++ b/src/version.h
@@ -1,2 +1,2 @@
-#define REDIS_VERSION "6.1.241"
-#define REDIS_VERSION_NUM 0x000601f1
+#define REDIS_VERSION "6.1.242"
+#define REDIS_VERSION_NUM 0x000601f2

From f2a5fe36789b50dab76851233966aaac44f9048c Mon Sep 17 00:00:00 2001
From: "zhaozhao.zz" <276441700@qq.com>
Date: Tue, 2 Feb 2021 16:51:19 +0800
Subject: [PATCH 1023/1098] XINFO should use lookupKeyReadOrReply (#8436)

This bug would have let users observe logically expired keys on replicas and
during CLIENT PAUSE WRITE.
---
 src/t_stream.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 197b7d4f7..32bfa1f7d 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -3471,7 +3471,7 @@ NULL
     key = c->argv[2];
 
     /* Lookup the key now, this is common for all the subcommands but HELP. */
-    robj *o = lookupKeyWriteOrReply(c,key,shared.nokeyerr);
+    robj *o = lookupKeyReadOrReply(c,key,shared.nokeyerr);
     if (o == NULL || checkType(c,o,OBJ_STREAM)) return;
     s = o->ptr;
 

From 9760475a3984b88b9fcec07392e3ebd998461841 Mon Sep 17 00:00:00 2001
From: Huang Zw <huang_zhw@126.com>
Date: Tue, 2 Feb 2021 16:54:19 +0800
Subject: [PATCH 1024/1098] Cleanup: addReplyAggregateLen and addReplyBulkLen
 remove redundant check (#8431)

addReplyLongLongWithPrefix, has a check against negative length, and the code
flow removed in this commit bypasses the check.
addReplyAggregateLen has an assertion for negative length, but addReplyBulkLen
does not, so this commit fixes theoretical case of access violation (probably
unreachable though)
---
 src/networking.c | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index da611675c..f9a3ff86a 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -717,10 +717,7 @@ void addReplyLongLong(client *c, long long ll) {
 
 void addReplyAggregateLen(client *c, long length, int prefix) {
     serverAssert(length >= 0);
-    if (prefix == '*' && length < OBJ_SHARED_BULKHDR_LEN)
-        addReply(c,shared.mbulkhdr[length]);
-    else
-        addReplyLongLongWithPrefix(c,length,prefix);
+    addReplyLongLongWithPrefix(c,length,prefix);
 }
 
 void addReplyArrayLen(client *c, long length) {
@@ -781,10 +778,7 @@ void addReplyNullArray(client *c) {
 void addReplyBulkLen(client *c, robj *obj) {
     size_t len = stringObjectLen(obj);
 
-    if (len < OBJ_SHARED_BULKHDR_LEN)
-        addReply(c,shared.bulkhdr[len]);
-    else
-        addReplyLongLongWithPrefix(c,len,'$');
+    addReplyLongLongWithPrefix(c,len,'$');
 }
 
 /* Add a Redis Object as a bulk reply */

From a3718cde0662d110cb989d358924ce0f5919ae25 Mon Sep 17 00:00:00 2001
From: "Jonah H. Harris" <jonah.harris@gmail.com>
Date: Tue, 2 Feb 2021 03:57:12 -0500
Subject: [PATCH 1025/1098] Optimizing sorted GEORADIUS COUNT with partial
 sorting. (#8326)

This commit provides an optimization, in terms of time, for all GEORADIUS*
and GEOSEARCH* searches which utilize the default, sorted, COUNT clause.
This is commonly used for nearest-neighbor (top-K points closest to a given lat/lon)
searches. While the previous implementation appends all matching points to the
geoPoint array and performs pruning after-the-fact via a full sort and [0, count)-based
for-loop, this PR sorts only the required number of elements.

This optimization provides a 5-20% improvement in runtime depending on the
density of points of interest (POI) as well as the radius searched.
No performance degradation has been observed.
---
 src/geo.c | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/geo.c b/src/geo.c
index 9c668288b..b2009b516 100644
--- a/src/geo.c
+++ b/src/geo.c
@@ -31,6 +31,7 @@
 #include "geo.h"
 #include "geohash_helper.h"
 #include "debugmacro.h"
+#include "pqsort.h"
 
 /* Things exported from t_zset.c only for geo.c, since it is the only other
  * part of Redis that requires close zset introspection. */
@@ -699,10 +700,20 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
     long option_length = 0;
 
     /* Process [optional] requested sorting */
-    if (sort == SORT_ASC) {
-        qsort(ga->array, result_length, sizeof(geoPoint), sort_gp_asc);
-    } else if (sort == SORT_DESC) {
-        qsort(ga->array, result_length, sizeof(geoPoint), sort_gp_desc);
+    if (sort != SORT_NONE) {
+        int (*sort_gp_callback)(const void *a, const void *b) = NULL;
+        if (sort == SORT_ASC) {
+            sort_gp_callback = sort_gp_asc;
+        } else if (sort == SORT_DESC) {
+            sort_gp_callback = sort_gp_desc;
+        }
+
+        if (returned_items == result_length) {
+            qsort(ga->array, result_length, sizeof(geoPoint), sort_gp_callback);
+        } else {
+            pqsort(ga->array, result_length, sizeof(geoPoint), sort_gp_callback,
+                0, (returned_items - 1));
+        }
     }
 
     if (storekey == NULL) {

From de6f3ad0170fd69d23530960f406d19beb119300 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 3 Feb 2021 17:35:28 +0200
Subject: [PATCH 1026/1098] Fix FreeBSD tests and CI Daily issues. (#8438)

* Add bash temporarily to allow sentinel fd leaks test to run.
* Use vmactions-freebsd rdist sync to work around bind permission denied
  and slow execution issues.
* Upgrade to tcl8.6 to be aligned with latest Ubuntu envs.
* Concat all command executions to avoid ignoring failures.
* Skip intensive fuzzer on FreeBSD. For some yet unknown reason, generate_fuzzy_traffic_on_key causes TCL to significantly bloat on FreeBSD resulting with out of memory.
---
 .github/workflows/ci.yml                  | 11 -----------
 .github/workflows/daily.yml               | 15 ++++++++-------
 tests/integration/corrupt-dump-fuzzer.tcl |  7 +++++++
 tests/modules/Makefile                    |  2 +-
 4 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2e1e7865c..2582c53a4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -58,14 +58,3 @@ jobs:
       run: |
         yum -y install gcc make
         make REDIS_CFLAGS='-Werror'
-        
-  build-freebsd:
-    runs-on: macos-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make
-      uses: vmactions/freebsd-vm@v0.1.0
-      with:
-        usesh: true
-        prepare: pkg install -y gmake
-        run: gmake
diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 8fb23bac4..589694f38 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -210,13 +210,14 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: test
-      uses: vmactions/freebsd-vm@v0.1.0
+      uses: vmactions/freebsd-vm@v0.1.2
       with:
         usesh: true
-        prepare: pkg install -y gmake lang/tcl85
-        run: |
-          gmake
-          ./runtest --accurate --verbose --no-latency
-          MAKE=gmake ./runtest-moduleapi --verbose
-          ./runtest-sentinel
+        sync: rsync
+        prepare: pkg install -y bash gmake lang/tcl86
+        run: >
+          gmake &&
+          ./runtest --accurate --verbose --no-latency &&
+          MAKE=gmake ./runtest-moduleapi --verbose &&
+          ./runtest-sentinel &&
           ./runtest-cluster
diff --git a/tests/integration/corrupt-dump-fuzzer.tcl b/tests/integration/corrupt-dump-fuzzer.tcl
index d41ed2941..2506205a5 100644
--- a/tests/integration/corrupt-dump-fuzzer.tcl
+++ b/tests/integration/corrupt-dump-fuzzer.tcl
@@ -71,6 +71,13 @@ foreach sanitize_dump {no yes} {
         set min_cycles 10 ; # run at least 10 cycles
     }
 
+    # Don't execute this on FreeBSD due to a yet-undiscovered memory issue
+    # which causes tclsh to bloat.
+    if {[exec uname] == "FreeBSD"} {
+        set min_cycles 1
+        set min_duration 1
+    }
+
     test "Fuzzer corrupt restore payloads - sanitize_dump: $sanitize_dump" {
         if {$min_duration * 2 > $::timeout} {
             fail "insufficient timeout"
diff --git a/tests/modules/Makefile b/tests/modules/Makefile
index 93b4b022f..e1629a8ac 100644
--- a/tests/modules/Makefile
+++ b/tests/modules/Makefile
@@ -43,7 +43,7 @@ all: $(TEST_MODULES)
 	$(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@
 
 %.so: %.xo
-	$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LDFLAGS) $(LIBS) -lc
+	$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LDFLAGS) $(LIBS)
 
 .PHONY: clean
 

From 52fb3065357afcf14f38a12760eb4edd75158ad9 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 4 Feb 2021 11:37:28 +0200
Subject: [PATCH 1027/1098] Fix 32-bit test modules build. (#8448)

---
 tests/modules/Makefile | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/modules/Makefile b/tests/modules/Makefile
index e1629a8ac..73a9be3bc 100644
--- a/tests/modules/Makefile
+++ b/tests/modules/Makefile
@@ -10,6 +10,12 @@ else	# Linux, others
 	SHOBJ_LDFLAGS ?= -shared
 endif
 
+# Needed to satisfy __stack_chk_fail_local on Linux with -m32, due to gcc
+# -fstack-protector by default. Breaks on FreeBSD so we exclude it.
+ifneq ($(uname_S),FreeBSD)
+    LIBS = -lc
+endif
+
 TEST_MODULES = \
     commandfilter.so \
     testrdb.so \

From ded1655d49950afd02bb3c5199bd33caac995e7c Mon Sep 17 00:00:00 2001
From: Yang Bodong <bodong.ybd@alibaba-inc.com>
Date: Fri, 5 Feb 2021 00:08:35 +0800
Subject: [PATCH 1028/1098] GEOSEARCH bybox bug fixes and new fuzzy tester
 (#8445)

Fix errors of GEOSEARCH bybox search due to:
1. projection of the box to a trapezoid (when the meter box is converted to long / lat it's no longer a box).
2. width and height mismatch

Changes:
- New GEOSEARCH point in rectangle algorithm
- Fix GEOSEARCH bybox width and height mismatch bug
- Add GEOSEARCH bybox testing to the existing "GEOADD + GEORANGE randomized test"
- Add new fuzzy test to stress test the bybox corners and edges
- Add some tests for edge cases of the bybox algorithm

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/geo.c            |  16 ++--
 src/geohash_helper.c |  55 ++++++-----
 src/geohash_helper.h |   2 +-
 tests/unit/geo.tcl   | 212 +++++++++++++++++++++++++++++++++++++------
 4 files changed, 225 insertions(+), 60 deletions(-)

diff --git a/src/geo.c b/src/geo.c
index b2009b516..7c75738a2 100644
--- a/src/geo.c
+++ b/src/geo.c
@@ -175,10 +175,10 @@ int extractDistanceOrReply(client *c, robj **argv,
  * that should be in the form: <number> <number> <unit>, and return C_OK or C_ERR means success or failure
  * *conversions is populated with the coefficient to use in order to convert meters to the unit.*/
 int extractBoxOrReply(client *c, robj **argv, double *conversion,
-                         double *height, double *width) {
+                         double *width, double *height) {
     double h, w;
-    if ((getDoubleFromObjectOrReply(c, argv[0], &h, "need numeric height") != C_OK) ||
-        (getDoubleFromObjectOrReply(c, argv[1], &w, "need numeric width") != C_OK)) {
+    if ((getDoubleFromObjectOrReply(c, argv[0], &w, "need numeric width") != C_OK) ||
+        (getDoubleFromObjectOrReply(c, argv[1], &h, "need numeric height") != C_OK)) {
         return C_ERR;
     }
 
@@ -224,8 +224,10 @@ int geoAppendIfWithinShape(geoArray *ga, GeoShape *shape, double score, sds memb
         if (!geohashGetDistanceIfInRadiusWGS84(shape->xy[0], shape->xy[1], xy[0], xy[1],
                                                shape->t.radius*shape->conversion, &distance)) return C_ERR;
     } else if (shape->type == RECTANGLE_TYPE) {
-        if (!geohashGetDistanceIfInRectangle(shape->bounds, shape->xy[0], shape->xy[1],
-                                             xy[0], xy[1], &distance)) return C_ERR;
+        if (!geohashGetDistanceIfInRectangle(shape->t.r.width * shape->conversion,
+                                             shape->t.r.height * shape->conversion,
+                                             shape->xy[0], shape->xy[1], xy[0], xy[1], &distance))
+            return C_ERR;
     }
 
     /* Append the new element. */
@@ -635,8 +637,8 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
                        flags & GEOSEARCH &&
                        !byradius)
             {
-                if (extractBoxOrReply(c, c->argv+base_args+i+1, &shape.conversion, &shape.t.r.height,
-                        &shape.t.r.width) != C_OK) return;
+                if (extractBoxOrReply(c, c->argv+base_args+i+1, &shape.conversion, &shape.t.r.width,
+                        &shape.t.r.height) != C_OK) return;
                 shape.type = RECTANGLE_TYPE;
                 bybox = 1;
                 i += 3;
diff --git a/src/geohash_helper.c b/src/geohash_helper.c
index a1ddf5d31..fec193e8b 100644
--- a/src/geohash_helper.c
+++ b/src/geohash_helper.c
@@ -85,20 +85,16 @@ uint8_t geohashEstimateStepsByRadius(double range_meters, double lat) {
 /* Return the bounding box of the search area by shape (see geohash.h GeoShape)
  * bounds[0] - bounds[2] is the minimum and maximum longitude
  * while bounds[1] - bounds[3] is the minimum and maximum latitude.
+ * since the higher the latitude, the shorter the arc length, the box shape is as follows
+ * (left and right edges are actually bent), as shown in the following diagram:
  *
- * This function does not behave correctly with very large radius values, for
- * instance for the coordinates 81.634948934258375 30.561509253718668 and a
- * radius of 7083 kilometers, it reports as bounding boxes:
- *
- * min_lon 7.680495, min_lat -33.119473, max_lon 155.589402, max_lat 94.242491
- *
- * However, for instance, a min_lon of 7.680495 is not correct, because the
- * point -1.27579540014266968 61.33421815228281559 is at less than 7000
- * kilometers away.
- *
- * Since this function is currently only used as an optimization, the
- * optimization is not used for very big radiuses, however the function
- * should be fixed. */
+ *    \-----------------/          --------               \-----------------/
+ *     \               /         /          \              \               /
+ *      \  (long,lat) /         / (long,lat) \              \  (long,lat) /
+ *       \           /         /              \              /            \
+ *         ---------          /----------------\            /--------------\
+ *  Northern Hemisphere       Southern Hemisphere         Around the equator
+ */
 int geohashBoundingBox(GeoShape *shape, double *bounds) {
     if (!bounds) return 0;
     double longitude = shape->xy[0];
@@ -106,10 +102,14 @@ int geohashBoundingBox(GeoShape *shape, double *bounds) {
     double height = shape->conversion * (shape->type == CIRCULAR_TYPE ? shape->t.radius : shape->t.r.height/2);
     double width = shape->conversion * (shape->type == CIRCULAR_TYPE ? shape->t.radius : shape->t.r.width/2);
 
-    const double long_delta = rad_deg(width/EARTH_RADIUS_IN_METERS/cos(deg_rad(latitude)));
     const double lat_delta = rad_deg(height/EARTH_RADIUS_IN_METERS);
-    bounds[0] = longitude - long_delta;
-    bounds[2] = longitude + long_delta;
+    const double long_delta_top = rad_deg(width/EARTH_RADIUS_IN_METERS/cos(deg_rad(latitude+lat_delta)));
+    const double long_delta_bottom = rad_deg(width/EARTH_RADIUS_IN_METERS/cos(deg_rad(latitude-lat_delta)));
+    /* The directions of the northern and southern hemispheres
+     * are opposite, so we choice different points as min/max long/lat */
+    int southern_hemisphere = latitude < 0 ? 1 : 0;
+    bounds[0] = southern_hemisphere ? longitude-long_delta_bottom : longitude-long_delta_top;
+    bounds[2] = southern_hemisphere ? longitude+long_delta_bottom : longitude+long_delta_top;
     bounds[1] = latitude - lat_delta;
     bounds[3] = latitude + lat_delta;
     return 1;
@@ -137,12 +137,10 @@ GeoHashRadius geohashCalculateAreasByShapeWGS84(GeoShape *shape) {
     double latitude = shape->xy[1];
     /* radius_meters is calculated differently in different search types:
      * 1) CIRCULAR_TYPE, just use radius.
-     * 2) RECTANGLE_TYPE, in order to calculate accurately, we should use
-     * sqrt((width/2)^2 + (height/2)^2), so that the box is bound by a circle,
-     * But the current code a simpler approach resulting in a smaller circle,
-     * which is safe because we search the 8 nearby boxes anyway. */
+     * 2) RECTANGLE_TYPE, we use sqrt((width/2)^2 + (height/2)^2) to
+     * calculate the distance from the center point to the corner */
     double radius_meters = shape->type == CIRCULAR_TYPE ? shape->t.radius :
-                            shape->t.r.width > shape->t.r.height ? shape->t.r.width/2 : shape->t.r.height/2;
+            sqrt((shape->t.r.width/2)*(shape->t.r.width/2) + (shape->t.r.height/2)*(shape->t.r.height/2));
     radius_meters *= shape->conversion;
 
     steps = geohashEstimateStepsByRadius(radius_meters,latitude);
@@ -245,14 +243,21 @@ int geohashGetDistanceIfInRadiusWGS84(double x1, double y1, double x2,
     return geohashGetDistanceIfInRadius(x1, y1, x2, y2, radius, distance);
 }
 
-/* Judge whether a point is in the axis-aligned rectangle.
- * bounds : see geohash.h GeoShape::bounds
+/* Judge whether a point is in the axis-aligned rectangle, when the distance
+ * between a searched point and the center point is less than or equal to
+ * height/2 or width/2 in height and width, the point is in the rectangle.
+ *
+ * width_m, height_m: the rectangle
  * x1, y1 : the center of the box
  * x2, y2 : the point to be searched
  */
-int geohashGetDistanceIfInRectangle(double *bounds, double x1, double y1,
+int geohashGetDistanceIfInRectangle(double width_m, double height_m, double x1, double y1,
                                     double x2, double y2, double *distance) {
-    if (x2 < bounds[0] || x2 > bounds[2] || y2 < bounds[1] || y2 > bounds[3]) return 0;
+    double lon_distance = geohashGetDistance(x2, y2, x1, y2);
+    double lat_distance = geohashGetDistance(x2, y2, x2, y1);
+    if (lon_distance > width_m/2 || lat_distance > height_m/2) {
+        return 0;
+    }
     *distance = geohashGetDistance(x1, y1, x2, y2);
     return 1;
 }
diff --git a/src/geohash_helper.h b/src/geohash_helper.h
index f28896a8d..630a89e30 100644
--- a/src/geohash_helper.h
+++ b/src/geohash_helper.h
@@ -60,7 +60,7 @@ int geohashGetDistanceIfInRadius(double x1, double y1,
 int geohashGetDistanceIfInRadiusWGS84(double x1, double y1, double x2,
                                       double y2, double radius,
                                       double *distance);
-int geohashGetDistanceIfInRectangle(double *bounds, double x1, double y1,
+int geohashGetDistanceIfInRectangle(double width_m, double height_m, double x1, double y1,
                                     double x2, double y2, double *distance);
 
 #endif /* GEOHASH_HELPER_HPP_ */
diff --git a/tests/unit/geo.tcl b/tests/unit/geo.tcl
index 0ac95f7c5..47100dfa8 100644
--- a/tests/unit/geo.tcl
+++ b/tests/unit/geo.tcl
@@ -1,6 +1,7 @@
 # Helper functions to simulate search-in-radius in the Tcl side in order to
 # verify the Redis implementation with a fuzzy test.
-proc geo_degrad deg {expr {$deg*atan(1)*8/360}}
+proc geo_degrad deg {expr {$deg*(atan(1)*8/360)}}
+proc geo_raddeg rad {expr {$rad/(atan(1)*8/360)}}
 
 proc geo_distance {lon1d lat1d lon2d lat2d} {
     set lon1r [geo_degrad $lon1d]
@@ -42,6 +43,34 @@ proc compare_lists {List1 List2} {
    return $DiffList
 }
 
+# return true If a point in circle.
+# search_lon and search_lat define the center of the circle,
+# and lon, lat define the point being searched.
+proc pointInCircle {radius_km lon lat search_lon search_lat} {
+    set radius_m [expr {$radius_km*1000}]
+    set distance [geo_distance $lon $lat $search_lon $search_lat]
+    if {$distance < $radius_m} {
+        return true
+    }
+    return false
+}
+
+# return true If a point in rectangle.
+# search_lon and search_lat define the center of the rectangle,
+# and lon, lat define the point being searched.
+# error: can adjust the width and height of the rectangle according to the error
+proc pointInRectangle {width_km height_km lon lat search_lon search_lat error} {
+    set width_m [expr {$width_km*1000*$error/2}]
+    set height_m [expr {$height_km*1000*$error/2}]
+    set lon_distance [geo_distance $lon $lat $search_lon $lat]
+    set lat_distance [geo_distance $lon $lat $lon $search_lat]
+
+    if {$lon_distance > $width_m || $lat_distance > $height_m} {
+        return false
+    }
+    return true
+}
+
 # The following list represents sets of random seed, search position
 # and radius that caused bugs in the past. It is used by the randomized
 # test later as a starting point. When the regression vectors are scanned
@@ -225,19 +254,26 @@ start_server {tags {"geo"}} {
 
     test {GEOSEARCH non square, long and narrow} {
         r del Sicily
-        r geoadd Sicily 12.75 37.00 "test1"
+        r geoadd Sicily 12.75 36.995 "test1"
         r geoadd Sicily 12.75 36.50 "test2"
         r geoadd Sicily 13.00 36.50 "test3"
         # box height=2km width=400km
-        set ret1 [r geosearch Sicily fromlonlat 15 37 bybox 2 400 km]
+        set ret1 [r geosearch Sicily fromlonlat 15 37 bybox 400 2 km]
         assert_equal $ret1 {test1}
 
         # Add a western Hemisphere point
         r geoadd Sicily -1 37.00 "test3"
-        set ret2 [r geosearch Sicily fromlonlat 15 37 bybox 2 3000 km asc]
+        set ret2 [r geosearch Sicily fromlonlat 15 37 bybox 3000 2 km asc]
         assert_equal $ret2 {test1 test3}
     }
 
+    test {GEOSEARCH corner point test} {
+        r del Sicily
+        r geoadd Sicily 12.758489 38.788135 edge1 17.241510 38.788135 edge2 17.250000 35.202000 edge3 12.750000 35.202000 edge4 12.748489955781654 37 edge5 15 38.798135872540925 edge6 17.251510044218346 37 edge7 15 35.201864127459075 edge8 12.692799634687903 38.798135872540925 corner1 12.692799634687903 38.798135872540925 corner2 17.200560937451133 35.201864127459075 corner3 12.799439062548865 35.201864127459075 corner4
+        set ret [lsort [r geosearch Sicily fromlonlat 15 37 bybox 400 400 km asc]]
+        assert_equal $ret {edge1 edge2 edge5 edge7}
+    }
+
     test {GEORADIUSBYMEMBER withdist (sorted)} {
         r georadiusbymember nyc "wtc one" 7 km withdist
     } {{{wtc one} 0.0000} {{union square} 3.2544} {{central park n/q/r} 6.7000} {4545 6.1975} {{lic market} 6.8969}}
@@ -360,12 +396,22 @@ start_server {tags {"geo"}} {
         assert {[lindex $res 0] eq "Catania"}
     }
 
-    test {GEOADD + GEORANGE randomized test} {
-        set attempt 30
+    test {GEOSEARCH the box spans -180° or 180°} {
+        r del points
+        r geoadd points 179.5 36 point1
+        r geoadd points -179.5 36 point2
+        assert_equal {point1 point2} [r geosearch points fromlonlat 179 37 bybox 400 400 km asc]
+        assert_equal {point2 point1} [r geosearch points fromlonlat -179 37 bybox 400 400 km asc]
+    }
+
+    foreach {type} {byradius bybox} {
+    test "GEOSEARCH fuzzy test - $type" {
+        if {$::accurate} { set attempt 300 } else { set attempt 30 }
         while {[incr attempt -1]} {
             set rv [lindex $regression_vectors $rv_idx]
             incr rv_idx
 
+            set radius_km 0; set width_km 0; set height_km 0
             unset -nocomplain debuginfo
             set srand_seed [clock milliseconds]
             if {$rv ne {}} {set srand_seed [lindex $rv 0]}
@@ -375,33 +421,55 @@ start_server {tags {"geo"}} {
 
             if {[randomInt 10] == 0} {
                 # From time to time use very big radiuses
-                set radius_km [expr {[randomInt 50000]+10}]
+                if {$type == "byradius"} {
+                    set radius_km [expr {[randomInt 5000]+10}]
+                } elseif {$type == "bybox"} {
+                    set width_km [expr {[randomInt 5000]+10}]
+                    set height_km [expr {[randomInt 5000]+10}]
+                }
             } else {
                 # Normally use a few - ~200km radiuses to stress
                 # test the code the most in edge cases.
-                set radius_km [expr {[randomInt 200]+10}]
+                if {$type == "byradius"} {
+                    set radius_km [expr {[randomInt 200]+10}]
+                } elseif {$type == "bybox"} {
+                    set width_km [expr {[randomInt 200]+10}]
+                    set height_km [expr {[randomInt 200]+10}]
+                }
+            }
+            if {$rv ne {}} {
+                set radius_km [lindex $rv 1]
+                set width_km [lindex $rv 1]
+                set height_km [lindex $rv 1]
             }
-            if {$rv ne {}} {set radius_km [lindex $rv 1]}
-            set radius_m [expr {$radius_km*1000}]
             geo_random_point search_lon search_lat
             if {$rv ne {}} {
                 set search_lon [lindex $rv 2]
                 set search_lat [lindex $rv 3]
             }
-            lappend debuginfo "Search area: $search_lon,$search_lat $radius_km km"
+            lappend debuginfo "Search area: $search_lon,$search_lat $radius_km $width_km $height_km km"
             set tcl_result {}
             set argv {}
             for {set j 0} {$j < 20000} {incr j} {
                 geo_random_point lon lat
                 lappend argv $lon $lat "place:$j"
-                set distance [geo_distance $lon $lat $search_lon $search_lat]
-                if {$distance < $radius_m} {
-                    lappend tcl_result "place:$j"
+                if {$type == "byradius"} {
+                    if {[pointInCircle $radius_km $lon $lat $search_lon $search_lat]} {
+                        lappend tcl_result "place:$j"
+                    }
+                } elseif {$type == "bybox"} {
+                    if {[pointInRectangle $width_km $height_km $lon $lat $search_lon $search_lat 1]} {
+                        lappend tcl_result "place:$j"
+                    }
                 }
-                lappend debuginfo "place:$j $lon $lat [expr {$distance/1000}] km"
+                lappend debuginfo "place:$j $lon $lat"
             }
             r geoadd mypoints {*}$argv
-            set res [lsort [r georadius mypoints $search_lon $search_lat $radius_km km]]
+            if {$type == "byradius"} {
+                set res [lsort [r geosearch mypoints fromlonlat $search_lon $search_lat byradius $radius_km km]]
+            } elseif {$type == "bybox"} {
+                set res [lsort [r geosearch mypoints fromlonlat $search_lon $search_lat bybox $width_km $height_km km]]
+            }
             set res2 [lsort $tcl_result]
             set test_result OK
 
@@ -409,18 +477,27 @@ start_server {tags {"geo"}} {
                 set rounding_errors 0
                 set diff [compare_lists $res $res2]
                 foreach place $diff {
+                    lassign [lindex [r geopos mypoints $place] 0] lon lat
                     set mydist [geo_distance $lon $lat $search_lon $search_lat]
                     set mydist [expr $mydist/1000]
-                    if {($mydist / $radius_km) > 0.999} {
-                        incr rounding_errors
-                        continue
-                    }
-                    if {$mydist < $radius_m} {
-                        # This is a false positive for redis since given the 
-                        # same points the higher precision calculation provided 
-                        # by TCL shows the point within range
-                        incr rounding_errors
-                        continue
+                    if {$type == "byradius"} {
+                        if {($mydist / $radius_km) > 0.999} {
+                            incr rounding_errors
+                            continue
+                        }
+                        if {$mydist < [expr {$radius_km*1000}]} {
+                            # This is a false positive for redis since given the
+                            # same points the higher precision calculation provided
+                            # by TCL shows the point within range
+                            incr rounding_errors
+                            continue
+                        }
+                    } elseif {$type == "bybox"} {
+                        # we add 0.1% error for floating point calculation error
+                        if {[pointInRectangle $width_km $height_km $lon $lat $search_lon $search_lat 1.001]} {
+                            incr rounding_errors
+                            continue
+                        }
                     }
                 }
 
@@ -447,7 +524,6 @@ start_server {tags {"geo"}} {
                     set mydist [geo_distance $lon $lat $search_lon $search_lat]
                     set mydist [expr $mydist/1000]
                     puts "$place -> [r geopos mypoints $place] $mydist $where"
-                    if {($mydist / $radius_km) > 0.999} {incr rounding_errors}
                 }
                 set test_result FAIL
             }
@@ -456,4 +532,86 @@ start_server {tags {"geo"}} {
         }
         set test_result
     } {OK}
+    }
+
+    test {GEOSEARCH box edges fuzzy test} {
+        if {$::accurate} { set attempt 300 } else { set attempt 30 }
+        while {[incr attempt -1]} {
+            unset -nocomplain debuginfo
+            set srand_seed [clock milliseconds]
+            lappend debuginfo "srand_seed is $srand_seed"
+            expr {srand($srand_seed)} ; # If you need a reproducible run
+            r del mypoints
+
+            geo_random_point search_lon search_lat
+            set width_m [expr {[randomInt 10000]+10}]
+            set height_m [expr {[randomInt 10000]+10}]
+            set lat_delta [geo_raddeg [expr {$height_m/2/6372797.560856}]]
+            set long_delta_top [geo_raddeg [expr {$width_m/2/6372797.560856/cos([geo_degrad [expr {$search_lat+$lat_delta}]])}]]
+            set long_delta_middle [geo_raddeg [expr {$width_m/2/6372797.560856/cos([geo_degrad $search_lat])}]]
+            set long_delta_bottom [geo_raddeg [expr {$width_m/2/6372797.560856/cos([geo_degrad [expr {$search_lat-$lat_delta}]])}]]
+
+            # Total of 8 points are generated, which are located at each vertex and the center of each side
+            set points(north) [list $search_lon [expr {$search_lat+$lat_delta}]]
+            set points(south) [list $search_lon [expr {$search_lat-$lat_delta}]]
+            set points(east) [list [expr {$search_lon+$long_delta_middle}] $search_lat]
+            set points(west) [list [expr {$search_lon-$long_delta_middle}] $search_lat]
+            set points(north_east) [list [expr {$search_lon+$long_delta_top}] [expr {$search_lat+$lat_delta}]]
+            set points(north_west) [list [expr {$search_lon-$long_delta_top}] [expr {$search_lat+$lat_delta}]]
+            set points(south_east) [list [expr {$search_lon+$long_delta_bottom}] [expr {$search_lat-$lat_delta}]]
+            set points(south_west) [list [expr {$search_lon-$long_delta_bottom}] [expr {$search_lat-$lat_delta}]]
+
+            lappend debuginfo "Search area: geosearch mypoints fromlonlat $search_lon $search_lat bybox $width_m $height_m m"
+            set tcl_result {}
+            foreach name [array names points] {
+                set x [lindex $points($name) 0]
+                set y [lindex $points($name) 1]
+                r geoadd mypoints $x $y place:$name
+                lappend tcl_result "place:$name"
+                lappend debuginfo "geoadd mypoints $x $y place:$name"
+            }
+
+            set res2 [lsort $tcl_result]
+
+            # make the box larger by two meter in each direction to put the coordinate slightly inside the box.
+            set height_new [expr {$height_m+4}]
+            set width_new [expr {$width_m+4}]
+            set res [lsort [r geosearch mypoints fromlonlat $search_lon $search_lat bybox $width_new $height_new m]]
+            if {$res != $res2} {
+                set diff [compare_lists $res $res2]
+                lappend debuginfo "diff: $diff"
+                fail "place should be found, debuginfo: $debuginfo, height_new: $height_new width_new: $width_new"
+            }
+
+            # The width decreases and the height increases. Only north and south are found
+            set width_new [expr {$width_m-4}]
+            set height_new [expr {$height_m+4}]
+            set res [lsort [r geosearch mypoints fromlonlat $search_lon $search_lat bybox $width_new $height_new m]]
+            if {$res != {place:north place:south}} {
+                set diff [compare_lists $res $res2]
+                lappend debuginfo "diff: $diff"
+                fail "place should not be found, debuginfo: $debuginfo, height_new: $height_new width_new: $width_new"
+            }
+
+            # The width increases and the height decreases. Only ease and west are found
+            set width_new [expr {$width_m+4}]
+            set height_new [expr {$height_m-4}]
+            set res [lsort [r geosearch mypoints fromlonlat $search_lon $search_lat bybox $width_new $height_new m]]
+            if {$res != {place:east place:west}} {
+                set diff [compare_lists $res $res2]
+                lappend debuginfo "diff: $diff"
+                fail "place should not be found, debuginfo: $debuginfo, height_new: $height_new width_new: $width_new"
+            }
+
+            # make the box smaller by two meter in each direction to put the coordinate slightly outside the box.
+            set height_new [expr {$height_m-4}]
+            set width_new [expr {$width_m-4}]
+            set res [r geosearch mypoints fromlonlat $search_lon $search_lat bybox $width_new $height_new m]
+            if {$res != ""} {
+                lappend debuginfo "res: $res"
+                fail "place should not be found, debuginfo: $debuginfo, height_new: $height_new width_new: $width_new"
+            }
+            unset -nocomplain debuginfo
+        }
+    }
 }

From b7b23a0ff50b6d921528289173d6edb6cc1f643a Mon Sep 17 00:00:00 2001
From: Yang Bodong <bodong.ybd@alibaba-inc.com>
Date: Fri, 5 Feb 2021 01:39:07 +0800
Subject: [PATCH 1029/1098] Fix GEOSEARCH tcl test error (#8451)

Issue with new test due to longitude wraparound.
---
 tests/unit/geo.tcl | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/tests/unit/geo.tcl b/tests/unit/geo.tcl
index 47100dfa8..a51d1dc5c 100644
--- a/tests/unit/geo.tcl
+++ b/tests/unit/geo.tcl
@@ -566,6 +566,13 @@ start_server {tags {"geo"}} {
             foreach name [array names points] {
                 set x [lindex $points($name) 0]
                 set y [lindex $points($name) 1]
+                # If longitude crosses -180° or 180°, we need to convert it.
+                # latitude doesn't have this problem, because it's scope is -70~70, see geo_random_point
+                if {$x > 180} {
+                    set x [expr {$x-360}]
+                } elseif {$x < -180} {
+                    set x [expr {$x+360}]
+                }
                 r geoadd mypoints $x $y place:$name
                 lappend tcl_result "place:$name"
                 lappend debuginfo "geoadd mypoints $x $y place:$name"
@@ -579,7 +586,7 @@ start_server {tags {"geo"}} {
             set res [lsort [r geosearch mypoints fromlonlat $search_lon $search_lat bybox $width_new $height_new m]]
             if {$res != $res2} {
                 set diff [compare_lists $res $res2]
-                lappend debuginfo "diff: $diff"
+                lappend debuginfo "res: $res, res2: $res2, diff: $diff"
                 fail "place should be found, debuginfo: $debuginfo, height_new: $height_new width_new: $width_new"
             }
 
@@ -588,8 +595,7 @@ start_server {tags {"geo"}} {
             set height_new [expr {$height_m+4}]
             set res [lsort [r geosearch mypoints fromlonlat $search_lon $search_lat bybox $width_new $height_new m]]
             if {$res != {place:north place:south}} {
-                set diff [compare_lists $res $res2]
-                lappend debuginfo "diff: $diff"
+                lappend debuginfo "res: $res"
                 fail "place should not be found, debuginfo: $debuginfo, height_new: $height_new width_new: $width_new"
             }
 
@@ -598,8 +604,7 @@ start_server {tags {"geo"}} {
             set height_new [expr {$height_m-4}]
             set res [lsort [r geosearch mypoints fromlonlat $search_lon $search_lat bybox $width_new $height_new m]]
             if {$res != {place:east place:west}} {
-                set diff [compare_lists $res $res2]
-                lappend debuginfo "diff: $diff"
+                lappend debuginfo "res: $res"
                 fail "place should not be found, debuginfo: $debuginfo, height_new: $height_new width_new: $width_new"
             }
 

From 18ac41973b87c444ed07ff6c043f38e59414b008 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Fri, 5 Feb 2021 21:56:20 +0800
Subject: [PATCH 1030/1098] RAND* commands: fix risk of OOM panic in hash and
 zset, use fair random in hash, and add tests for even distribution to all
 (#8429)

Changes to HRANDFIELD and ZRANDMEMBER:
* Fix risk of OOM panic when client query a very big negative count (avoid allocating huge temporary buffer).
* Fix uneven random distribution in HRANDFIELD with negative count (wasn't using dictGetFairRandomKey).
* Add tests to check an even random distribution (HRANDFIELD, SRANDMEMBER, ZRANDMEMBER).

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/t_hash.c             | 41 +++++++++++++++++++++++++---------------
 src/t_zset.c             | 41 +++++++++++++++++++++++++---------------
 src/ziplist.c            |  2 ++
 tests/support/util.tcl   | 18 ++++++++++++++++++
 tests/unit/type/hash.tcl | 19 ++++++++++++++++---
 tests/unit/type/set.tcl  | 37 ++++++++++++++++++++++++++++++++++++
 tests/unit/type/zset.tcl | 19 ++++++++++++++++---
 7 files changed, 141 insertions(+), 36 deletions(-)

diff --git a/src/t_hash.c b/src/t_hash.c
index 9f7540a72..98ba69df7 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -964,6 +964,11 @@ void hscanCommand(client *c) {
  * implementation for more info. */
 #define HRANDFIELD_SUB_STRATEGY_MUL 3
 
+/* If client is trying to ask for a very large number of random elements,
+ * queuing may consume an unlimited amount of memory, so we want to limit
+ * the number of randoms per time. */
+#define HRANDFIELD_RANDOM_SAMPLE_LIMIT 1000
+
 void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
     unsigned long count, size;
     int uniq = 1;
@@ -999,7 +1004,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
         if (hash->encoding == OBJ_ENCODING_HT) {
             sds key, value;
             while (count--) {
-                dictEntry *de = dictGetRandomKey(hash->ptr);
+                dictEntry *de = dictGetFairRandomKey(hash->ptr);
                 key = dictGetKey(de);
                 value = dictGetVal(de);
                 if (withvalues && c->resp > 2)
@@ -1010,22 +1015,28 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
             }
         } else if (hash->encoding == OBJ_ENCODING_ZIPLIST) {
             ziplistEntry *keys, *vals = NULL;
-            keys = zmalloc(sizeof(ziplistEntry)*count);
+            unsigned long limit, sample_count;
+            limit = count > HRANDFIELD_RANDOM_SAMPLE_LIMIT ? HRANDFIELD_RANDOM_SAMPLE_LIMIT : count;
+            keys = zmalloc(sizeof(ziplistEntry)*limit);
             if (withvalues)
-                vals = zmalloc(sizeof(ziplistEntry)*count);
-            ziplistRandomPairs(hash->ptr, count, keys, vals);
-            for (unsigned long i = 0; i < count; i++) {
-                if (withvalues && c->resp > 2)
-                    addReplyArrayLen(c,2);
-                if (keys[i].sval)
-                    addReplyBulkCBuffer(c, keys[i].sval, keys[i].slen);
-                else
-                    addReplyBulkLongLong(c, keys[i].lval);
-                if (withvalues) {
-                    if (vals[i].sval)
-                        addReplyBulkCBuffer(c, vals[i].sval, vals[i].slen);
+                vals = zmalloc(sizeof(ziplistEntry)*limit);
+            while (count) {
+                sample_count = count > limit ? limit : count;
+                count -= sample_count;
+                ziplistRandomPairs(hash->ptr, sample_count, keys, vals);
+                for (unsigned long i = 0; i < sample_count; i++) {
+                    if (withvalues && c->resp > 2)
+                        addReplyArrayLen(c,2);
+                    if (keys[i].sval)
+                        addReplyBulkCBuffer(c, keys[i].sval, keys[i].slen);
                     else
-                        addReplyBulkLongLong(c, vals[i].lval);
+                        addReplyBulkLongLong(c, keys[i].lval);
+                    if (withvalues) {
+                        if (vals[i].sval)
+                            addReplyBulkCBuffer(c, vals[i].sval, vals[i].slen);
+                        else
+                            addReplyBulkLongLong(c, vals[i].lval);
+                    }
                 }
             }
             zfree(keys);
diff --git a/src/t_zset.c b/src/t_zset.c
index b55fc169e..c14a37263 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -3961,6 +3961,11 @@ void bzpopmaxCommand(client *c) {
  * implementation for more info. */
 #define ZRANDMEMBER_SUB_STRATEGY_MUL 3
 
+/* If client is trying to ask for a very large number of random elements,
+ * queuing may consume an unlimited amount of memory, so we want to limit
+ * the number of randoms per time. */
+#define ZRANDMEMBER_RANDOM_SAMPLE_LIMIT 1000
+
 void zrandmemberWithCountCommand(client *c, long l, int withscores) {
     unsigned long count, size;
     int uniq = 1;
@@ -4006,22 +4011,28 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) {
             }
         } else if (zsetobj->encoding == OBJ_ENCODING_ZIPLIST) {
             ziplistEntry *keys, *vals = NULL;
-            keys = zmalloc(sizeof(ziplistEntry)*count);
+            unsigned long limit, sample_count;
+            limit = count > ZRANDMEMBER_RANDOM_SAMPLE_LIMIT ? ZRANDMEMBER_RANDOM_SAMPLE_LIMIT : count;
+            keys = zmalloc(sizeof(ziplistEntry)*limit);
             if (withscores)
-                vals = zmalloc(sizeof(ziplistEntry)*count);
-            ziplistRandomPairs(zsetobj->ptr, count, keys, vals);
-            for (unsigned long i = 0; i < count; i++) {
-                if (withscores && c->resp > 2)
-                    addReplyArrayLen(c,2);
-                if (keys[i].sval)
-                    addReplyBulkCBuffer(c, keys[i].sval, keys[i].slen);
-                else
-                    addReplyBulkLongLong(c, keys[i].lval);
-                if (withscores) {
-                    if (vals[i].sval) {
-                        addReplyDouble(c, zzlStrtod(vals[i].sval,vals[i].slen));
-                    } else
-                        addReplyDouble(c, vals[i].lval);
+                vals = zmalloc(sizeof(ziplistEntry)*limit);
+            while (count) {
+                sample_count = count > limit ? limit : count;
+                count -= sample_count;
+                ziplistRandomPairs(zsetobj->ptr, sample_count, keys, vals);
+                for (unsigned long i = 0; i < sample_count; i++) {
+                    if (withscores && c->resp > 2)
+                        addReplyArrayLen(c,2);
+                    if (keys[i].sval)
+                        addReplyBulkCBuffer(c, keys[i].sval, keys[i].slen);
+                    else
+                        addReplyBulkLongLong(c, keys[i].lval);
+                    if (withscores) {
+                        if (vals[i].sval) {
+                            addReplyDouble(c, zzlStrtod(vals[i].sval,vals[i].slen));
+                        } else
+                            addReplyDouble(c, vals[i].lval);
+                    }
                 }
             }
             zfree(keys);
diff --git a/src/ziplist.c b/src/ziplist.c
index 0cd20630a..248166ac2 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -1541,6 +1541,8 @@ void ziplistRandomPairs(unsigned char *zl, int count, ziplistEntry *keys, ziplis
     unsigned char *p, *key, *value;
     unsigned int klen, vlen;
     long long klval, vlval;
+
+    /* Notice: the index member must be first due to the use in intCompare */
     typedef struct {
         int index;
         int order;
diff --git a/tests/support/util.tcl b/tests/support/util.tcl
index 80f8598ce..886ef5020 100644
--- a/tests/support/util.tcl
+++ b/tests/support/util.tcl
@@ -681,3 +681,21 @@ proc string2printable s {
     set res "\"$res\""
     return $res
 }
+
+# Check that probability of each element are between {min_prop} and {max_prop}.
+proc check_histogram_distribution {res min_prop max_prop} {
+    unset -nocomplain mydict
+    foreach key $res {
+        dict incr mydict $key 1
+    }
+
+    foreach key [dict keys $mydict] {
+        set value [dict get $mydict $key]
+        set probability [expr {double($value) / [llength $res]}]
+        if {$probability < $min_prop || $probability > $max_prop} {
+            return false
+        }
+    }
+
+    return true
+}
diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl
index 2f3ea37c2..2eea98890 100644
--- a/tests/unit/type/hash.tcl
+++ b/tests/unit/type/hash.tcl
@@ -96,9 +96,17 @@ start_server {tags {"hash"}} {
             # 1) Check that it returns repeated elements with and without values.
             set res [r hrandfield myhash -20]
             assert_equal [llength $res] 20
+            set res [r hrandfield myhash -1001]
+            assert_equal [llength $res] 1001
             # again with WITHVALUES
             set res [r hrandfield myhash -20 withvalues]
             assert_equal [llength $res] 40
+            set res [r hrandfield myhash -1001 withvalues]
+            assert_equal [llength $res] 2002
+
+            # Test random uniform distribution
+            set res [r hrandfield myhash -1000]
+            assert_equal [check_histogram_distribution $res 0.05 0.15] true
 
             # 2) Check that all the elements actually belong to the original hash.
             foreach {key val} $res {
@@ -167,26 +175,31 @@ start_server {tags {"hash"}} {
                 # 2) Check that eventually all the elements are returned.
                 #    Use both WITHVALUES and without
                 unset -nocomplain auxset
-                set iterations 1000
+                unset -nocomplain allkey
+                set iterations [expr {1000 / $size}]
+                set all_ele_return false
                 while {$iterations != 0} {
                     incr iterations -1
                     if {[expr {$iterations % 2}] == 0} {
                         set res [r hrandfield myhash $size withvalues]
                         foreach {key value} $res {
                             dict append auxset $key $value
+                            lappend allkey $key
                         }
                     } else {
                         set res [r hrandfield myhash $size]
                         foreach key $res {
                             dict append auxset $key
+                            lappend allkey $key
                         }
                     }
                     if {[lsort [dict keys $mydict]] eq
                         [lsort [dict keys $auxset]]} {
-                        break;
+                        set all_ele_return true
                     }
                 }
-                assert {$iterations != 0}
+                assert_equal $all_ele_return true
+                assert_equal [check_histogram_distribution $allkey 0.05 0.15] true
             }
         }
         r config set hash-max-ziplist-value $original_max_value
diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl
index 091ef7f0f..4eb93a21e 100644
--- a/tests/unit/type/set.tcl
+++ b/tests/unit/type/set.tcl
@@ -515,6 +515,43 @@ start_server {
         }
     }
 
+    foreach {type contents} {
+        hashtable {
+            1 5 10 50 125
+            MARY PATRICIA LINDA BARBARA ELIZABETH
+        }
+        intset {
+            0 1 2 3 4 5 6 7 8 9
+        }
+    } {
+        test "SRANDMEMBER histogram distribution - $type" {
+            create_set myset $contents
+            unset -nocomplain myset
+            array set myset {}
+            foreach ele [r smembers myset] {
+                set myset($ele) 1
+            }
+
+            # Use negative count (PATH 1).
+            set res [r srandmember myset -1000]
+            assert_equal [check_histogram_distribution $res 0.05 0.15] true
+
+            # Use positive count (both PATH 3 and PATH 4).
+            foreach size {8 2} {
+                unset -nocomplain allkey
+                set iterations [expr {1000 / $size}]
+                while {$iterations != 0} {
+                    incr iterations -1
+                    set res [r srandmember myset $size]
+                    foreach ele $res {
+                        lappend allkey $ele
+                    }
+                }
+                assert_equal [check_histogram_distribution $allkey 0.05 0.15] true
+            }
+        }
+    }
+
     proc setup_move {} {
         r del myset3 myset4
         create_set myset1 {1 a b}
diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl
index c657c1e4e..2456815f2 100644
--- a/tests/unit/type/zset.tcl
+++ b/tests/unit/type/zset.tcl
@@ -1646,9 +1646,17 @@ start_server {tags {"zset"}} {
             # 1) Check that it returns repeated elements with and without values.
             set res [r zrandmember myzset -20]
             assert_equal [llength $res] 20
+            set res [r zrandmember myzset -1001]
+            assert_equal [llength $res] 1001
             # again with WITHSCORES
             set res [r zrandmember myzset -20 withscores]
             assert_equal [llength $res] 40
+            set res [r zrandmember myzset -1001 withscores]
+            assert_equal [llength $res] 2002
+
+            # Test random uniform distribution
+            set res [r zrandmember myzset -1000]
+            assert_equal [check_histogram_distribution $res 0.05 0.15] true
 
             # 2) Check that all the elements actually belong to the original zset.
             foreach {key val} $res {
@@ -1717,26 +1725,31 @@ start_server {tags {"zset"}} {
                 # 2) Check that eventually all the elements are returned.
                 #    Use both WITHSCORES and without
                 unset -nocomplain auxset
-                set iterations 1000
+                unset -nocomplain allkey
+                set iterations [expr {1000 / $size}]
+                set all_ele_return false
                 while {$iterations != 0} {
                     incr iterations -1
                     if {[expr {$iterations % 2}] == 0} {
                         set res [r zrandmember myzset $size withscores]
                         foreach {key value} $res {
                             dict append auxset $key $value
+                            lappend allkey $key
                         }
                     } else {
                         set res [r zrandmember myzset $size]
                         foreach key $res {
                             dict append auxset $key
+                            lappend allkey $key
                         }
                     }
                     if {[lsort [dict keys $mydict]] eq
                         [lsort [dict keys $auxset]]} {
-                        break;
+                        set all_ele_return true
                     }
                 }
-                assert {$iterations != 0}
+                assert_equal $all_ele_return true
+                assert_equal [check_histogram_distribution $allkey 0.05 0.15] true
             }
         }
         r config set zset-max-ziplist-value $original_max_value

From b3bdcd2278d1822d861d6e4c6b9688e6b0a33d37 Mon Sep 17 00:00:00 2001
From: filipe oliveira <filipecosta.90@gmail.com>
Date: Fri, 5 Feb 2021 17:51:31 +0000
Subject: [PATCH 1031/1098] =?UTF-8?q?Fix=20compiler=20warning=20on=20impli?=
 =?UTF-8?q?cit=20declaration=20of=20=E2=80=98nanosleep=E2=80=99=20.=20Remo?=
 =?UTF-8?q?ved=20unused=20variable=20(#8454)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tests/modules/blockonbackground.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/modules/blockonbackground.c b/tests/modules/blockonbackground.c
index cf7e9c7c1..fcabfa4d2 100644
--- a/tests/modules/blockonbackground.c
+++ b/tests/modules/blockonbackground.c
@@ -1,4 +1,5 @@
 #define REDISMODULE_EXPERIMENTAL_API
+#define _XOPEN_SOURCE 700
 #include "redismodule.h"
 #include <stdio.h>
 #include <stdlib.h>
@@ -55,7 +56,7 @@ void *BlockDebug_ThreadMain(void *arg) {
 }
 
 /* The thread entry point that actually executes the blocking part
- * of the command BLOCK.DEBUG. */
+ * of the command BLOCK.DOUBLE_DEBUG. */
 void *DoubleBlock_ThreadMain(void *arg) {
     void **targ = arg;
     RedisModuleBlockedClient *bc = targ[0];
@@ -173,14 +174,13 @@ int HelloBlockNoTracking_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **a
 int HelloDoubleBlock_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
     if (argc != 2) return RedisModule_WrongArity(ctx);
     long long delay;
-    long long timeout;
 
     if (RedisModule_StringToLongLong(argv[1],&delay) != REDISMODULE_OK) {
         return RedisModule_ReplyWithError(ctx,"ERR invalid count");
     }
 
     pthread_t tid;
-    RedisModuleBlockedClient *bc = RedisModule_BlockClient(ctx,HelloBlock_Reply,HelloBlock_Timeout,HelloBlock_FreeData,timeout);
+    RedisModuleBlockedClient *bc = RedisModule_BlockClient(ctx,HelloBlock_Reply,HelloBlock_Timeout,HelloBlock_FreeData,0);
 
     /* Now that we setup a blocking client, we need to pass the control
      * to the thread. However we need to pass arguments to the thread:

From aea6e71ef82701e07177744e600e1ef20d60b7d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= <viktor.soderqvist@est.tech>
Date: Fri, 5 Feb 2021 18:54:01 +0100
Subject: [PATCH 1032/1098] RM_ZsetRem: Delete key if empty (#8453)

Without this fix, RM_ZsetRem can leave empty sorted sets which are
not allowed to exist.

Removing from a sorted set while iterating seems to work (while
inserting causes failed assetions). RM_ZsetRangeEndReached is
modified to return 1 if the key doesn't exist, to terminate
iteration when the last element has been removed.
---
 runtest-moduleapi             |  1 +
 src/module.c                  |  2 ++
 tests/modules/Makefile        |  1 +
 tests/modules/zset.c          | 30 ++++++++++++++++++++++++++++++
 tests/unit/moduleapi/zset.tcl | 16 ++++++++++++++++
 5 files changed, 50 insertions(+)
 create mode 100644 tests/modules/zset.c
 create mode 100644 tests/unit/moduleapi/zset.tcl

diff --git a/runtest-moduleapi b/runtest-moduleapi
index e554226c1..53656fad7 100755
--- a/runtest-moduleapi
+++ b/runtest-moduleapi
@@ -32,5 +32,6 @@ $TCLSH tests/test_helper.tcl \
 --single unit/moduleapi/getkeys \
 --single unit/moduleapi/test_lazyfree \
 --single unit/moduleapi/defrag \
+--single unit/moduleapi/zset \
 --single unit/moduleapi/stream \
 "${@}"
diff --git a/src/module.c b/src/module.c
index b04595801..039465e1a 100644
--- a/src/module.c
+++ b/src/module.c
@@ -2612,6 +2612,7 @@ int RM_ZsetRem(RedisModuleKey *key, RedisModuleString *ele, int *deleted) {
     if (key->value && key->value->type != OBJ_ZSET) return REDISMODULE_ERR;
     if (key->value != NULL && zsetDel(key->value,ele->ptr)) {
         if (deleted) *deleted = 1;
+        moduleDelKeyIfEmpty(key);
     } else {
         if (deleted) *deleted = 0;
     }
@@ -2657,6 +2658,7 @@ void RM_ZsetRangeStop(RedisModuleKey *key) {
 
 /* Return the "End of range" flag value to signal the end of the iteration. */
 int RM_ZsetRangeEndReached(RedisModuleKey *key) {
+    if (!key->value || key->value->type != OBJ_ZSET) return 1;
     return key->u.zset.er;
 }
 
diff --git a/tests/modules/Makefile b/tests/modules/Makefile
index 73a9be3bc..8ea1d91a2 100644
--- a/tests/modules/Makefile
+++ b/tests/modules/Makefile
@@ -35,6 +35,7 @@ TEST_MODULES = \
     test_lazyfree.so \
     timer.so \
     defragtest.so \
+    zset.so \
     stream.so
 
 
diff --git a/tests/modules/zset.c b/tests/modules/zset.c
new file mode 100644
index 000000000..4806f6549
--- /dev/null
+++ b/tests/modules/zset.c
@@ -0,0 +1,30 @@
+#include "redismodule.h"
+
+/* ZSET.REM key element
+ *
+ * Removes an occurrence of an element from a sorted set. Replies with the
+ * number of removed elements (0 or 1).
+ */
+int zset_rem(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc != 3) return RedisModule_WrongArity(ctx);
+    RedisModule_AutoMemory(ctx);
+    int keymode = REDISMODULE_READ | REDISMODULE_WRITE;
+    RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], keymode);
+    int deleted;
+    if (RedisModule_ZsetRem(key, argv[2], &deleted) == REDISMODULE_OK)
+        return RedisModule_ReplyWithLongLong(ctx, deleted);
+    else
+        return RedisModule_ReplyWithError(ctx, "ERR ZsetRem failed");
+}
+
+int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+    if (RedisModule_Init(ctx, "zset", 1, REDISMODULE_APIVER_1) ==
+        REDISMODULE_OK &&
+        RedisModule_CreateCommand(ctx, "zset.rem", zset_rem, "",
+                                  1, 1, 1) == REDISMODULE_OK)
+        return REDISMODULE_OK;
+    else
+        return REDISMODULE_ERR;
+}
diff --git a/tests/unit/moduleapi/zset.tcl b/tests/unit/moduleapi/zset.tcl
new file mode 100644
index 000000000..998d20765
--- /dev/null
+++ b/tests/unit/moduleapi/zset.tcl
@@ -0,0 +1,16 @@
+set testmodule [file normalize tests/modules/zset.so]
+
+start_server {tags {"modules"}} {
+    r module load $testmodule
+
+    test {Module zset rem} {
+        r del k
+        r zadd k 100 hello 200 world
+        assert_equal 1 [r zset.rem k hello]
+        assert_equal 0 [r zset.rem k hello]
+        assert_equal 1 [r exists k]
+        # Check that removing the last element deletes the key
+        assert_equal 1 [r zset.rem k world]
+        assert_equal 0 [r exists k]
+    }
+}

From be83bb13a8eaad68b7580b95c696f2554cf7100e Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 7 Feb 2021 12:36:56 +0200
Subject: [PATCH 1033/1098] Add --insecure option to command line tools.
 (#8416)

Disable certificate validation, making it possible to connect to servers
without configuring full trust chain.

The use of this option is insecure and makes the connection vulnerable
to man in the middle attacks.
---
 src/cli_common.c      | 2 +-
 src/cli_common.h      | 2 ++
 src/redis-benchmark.c | 4 ++++
 src/redis-cli.c       | 4 ++++
 4 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/cli_common.c b/src/cli_common.c
index c2db9fffc..e88327ace 100644
--- a/src/cli_common.c
+++ b/src/cli_common.c
@@ -54,7 +54,7 @@ int cliSecureConnection(redisContext *c, cliSSLconfig config, const char **err)
             goto error;
         }
         SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3);
-        SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL);
+        SSL_CTX_set_verify(ssl_ctx, config.skip_cert_verify ? SSL_VERIFY_NONE : SSL_VERIFY_PEER, NULL);
 
         if (config.cacert || config.cacertdir) {
             if (!SSL_CTX_load_verify_locations(ssl_ctx, config.cacert, config.cacertdir)) {
diff --git a/src/cli_common.h b/src/cli_common.h
index f3a91e9db..16d6ec2a9 100644
--- a/src/cli_common.h
+++ b/src/cli_common.h
@@ -10,6 +10,8 @@ typedef struct cliSSLconfig {
     char *cacert;
     /* Directory where trusted CA certificates are stored, or NULL */
     char *cacertdir;
+    /* Skip server certificate verification. */
+    int skip_cert_verify;
     /* Client certificate to authenticate with, or NULL */
     char *cert;
     /* Private key file to authenticate with, or NULL */
diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index a955c0d4c..164f5e3ee 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -1516,6 +1516,8 @@ int parseOptions(int argc, const char **argv) {
         } else if (!strcmp(argv[i],"--cacert")) {
             if (lastarg) goto invalid;
             config.sslconfig.cacert = strdup(argv[++i]);
+        } else if (!strcmp(argv[i],"--insecure")) {
+            config.sslconfig.skip_cert_verify = 1;
         } else if (!strcmp(argv[i],"--cert")) {
             if (lastarg) goto invalid;
             config.sslconfig.cert = strdup(argv[++i]);
@@ -1585,6 +1587,7 @@ usage:
 " --cacertdir <dir>  Directory where trusted CA certificates are stored.\n"
 "                    If neither cacert nor cacertdir are specified, the default\n"
 "                    system-wide trusted root certs configuration will apply.\n"
+" --insecure         Allow insecure TLS connection by skipping cert validation.\n"
 " --cert <file>      Client certificate to authenticate with.\n"
 " --key <file>       Private key file to authenticate with.\n"
 " --tls-ciphers <list> Sets the list of prefered ciphers (TLSv1.2 and below)\n"
@@ -1682,6 +1685,7 @@ int main(int argc, const char **argv) {
     signal(SIGHUP, SIG_IGN);
     signal(SIGPIPE, SIG_IGN);
 
+    memset(&config.sslconfig, 0, sizeof(config.sslconfig));
     config.numclients = 50;
     config.requests = 100000;
     config.liveclients = 0;
diff --git a/src/redis-cli.c b/src/redis-cli.c
index ed3075317..ab30edc75 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -1695,6 +1695,8 @@ static int parseOptions(int argc, char **argv) {
             config.sslconfig.key = argv[++i];
         } else if (!strcmp(argv[i],"--tls-ciphers") && !lastarg) {
             config.sslconfig.ciphers = argv[++i];
+        } else if (!strcmp(argv[i],"--insecure")) {
+            config.sslconfig.skip_cert_verify = 1;
         #ifdef TLS1_3_VERSION
         } else if (!strcmp(argv[i],"--tls-ciphersuites") && !lastarg) {
             config.sslconfig.ciphersuites = argv[++i];
@@ -1820,6 +1822,7 @@ static void usage(void) {
 "  --cacertdir <dir>  Directory where trusted CA certificates are stored.\n"
 "                     If neither cacert nor cacertdir are specified, the default\n"
 "                     system-wide trusted root certs configuration will apply.\n"
+"  --insecure         Allow insecure TLS connection by skipping cert validation.\n"
 "  --cert <file>      Client certificate to authenticate with.\n"
 "  --key <file>       Private key file to authenticate with.\n"
 "  --tls-ciphers <list> Sets the list of prefered ciphers (TLSv1.2 and below)\n"
@@ -8131,6 +8134,7 @@ int main(int argc, char **argv) {
     int firstarg;
     struct timeval tv;
 
+    memset(&config.sslconfig, 0, sizeof(config.sslconfig));
     config.hostip = sdsnew("127.0.0.1");
     config.hostport = 6379;
     config.hostsocket = NULL;

From 5b8350aaaa1b10efc49ee6b3c15a0559c69c9b20 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 7 Feb 2021 12:37:24 +0200
Subject: [PATCH 1034/1098] Add --dump-logs tests option. (#8459)

Dump the entire server log if a test failed, to easy troubleshooting
with no access to log files.
---
 .github/workflows/daily.yml | 24 ++++++++++++------------
 tests/support/server.tcl    | 14 ++++++++++++++
 tests/test_helper.tcl       |  4 ++++
 3 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 589694f38..e3cb1a344 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -21,7 +21,7 @@ jobs:
     - name: test
       run: |
         sudo apt-get install tcl8.5
-        ./runtest --accurate --verbose
+        ./runtest --accurate --verbose --dump-logs
     - name: module api test
       run: ./runtest-moduleapi --verbose
     - name: sentinel tests
@@ -40,7 +40,7 @@ jobs:
     - name: test
       run: |
         sudo apt-get install tcl8.5
-        ./runtest --accurate --verbose
+        ./runtest --accurate --verbose --dump-logs
     - name: module api test
       run: ./runtest-moduleapi --verbose
     - name: sentinel tests
@@ -61,7 +61,7 @@ jobs:
     - name: test
       run: |
         sudo apt-get install tcl8.5
-        ./runtest --accurate --verbose
+        ./runtest --accurate --verbose --dump-logs
     - name: module api test
       run: |
         make -C tests/modules 32bit # the script below doesn't have an argument, we must build manually ahead of time
@@ -84,8 +84,8 @@ jobs:
       run: |
         sudo apt-get install tcl8.5 tcl-tls
         ./utils/gen-test-certs.sh
-        ./runtest --accurate --verbose --tls
-        ./runtest --accurate --verbose
+        ./runtest --accurate --verbose --tls --dump-logs
+        ./runtest --accurate --verbose --dump-logs
     - name: module api test
       run: |
         ./runtest-moduleapi --verbose --tls
@@ -111,7 +111,7 @@ jobs:
     - name: test
       run: |
         sudo apt-get install tcl8.5 tcl-tls
-        ./runtest --config io-threads 4 --config io-threads-do-reads yes --accurate --verbose --tags network
+        ./runtest --config io-threads 4 --config io-threads-do-reads yes --accurate --verbose --tags network --dump-logs
     - name: cluster tests
       run: |
         ./runtest-cluster --config io-threads 4 --config io-threads-do-reads yes
@@ -128,7 +128,7 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install tcl8.5 valgrind -y
-        ./runtest --valgrind --verbose --clients 1
+        ./runtest --valgrind --verbose --clients 1 --dump-logs
     - name: module api test
       run: ./runtest-moduleapi --valgrind --verbose --clients 1
 
@@ -146,7 +146,7 @@ jobs:
     - name: test
       run: |
         yum -y install which tcl
-        ./runtest --accurate --verbose
+        ./runtest --accurate --verbose --dump-logs
     - name: module api test
       run: ./runtest-moduleapi --verbose
     - name: sentinel tests
@@ -170,8 +170,8 @@ jobs:
       run: |
         yum -y install tcl tcltls
         ./utils/gen-test-certs.sh
-        ./runtest --accurate --verbose --tls
-        ./runtest --accurate --verbose
+        ./runtest --accurate --verbose --tls --dump-logs
+        ./runtest --accurate --verbose --dump-logs
     - name: module api test
       run: |
         ./runtest-moduleapi --verbose --tls
@@ -195,7 +195,7 @@ jobs:
       run: make
     - name: test
       run: |
-        ./runtest --accurate --verbose --no-latency
+        ./runtest --accurate --verbose --no-latency --dump-logs
     - name: module api test
       run: ./runtest-moduleapi --verbose
     - name: sentinel tests
@@ -217,7 +217,7 @@ jobs:
         prepare: pkg install -y bash gmake lang/tcl86
         run: >
           gmake &&
-          ./runtest --accurate --verbose --no-latency &&
+          ./runtest --accurate --verbose --no-latency --dump-logs &&
           MAKE=gmake ./runtest-moduleapi --verbose &&
           ./runtest-sentinel &&
           ./runtest-cluster
diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index 0d36d46be..4fbb99920 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -259,6 +259,13 @@ proc wait_server_started {config_file stdout pid} {
     return $port_busy
 }
 
+proc dump_server_log {srv} {
+    set pid [dict get $srv "pid"]
+    puts "\n===== Start of server log (pid $pid) =====\n"
+    puts [exec cat [dict get $srv "stdout"]]
+    puts "===== End of server log (pid $pid) =====\n"
+}
+
 proc start_server {options {code undefined}} {
     # setup defaults
     set baseconfig "default.conf"
@@ -492,6 +499,9 @@ proc start_server {options {code undefined}} {
         # connect client (after server dict is put on the stack)
         reconnect
 
+        # remember previous num_failed to catch new errors
+        set prev_num_failed $::num_failed
+
         # execute provided block
         set num_tests $::num_tests
         if {[catch { uplevel 1 $code } error]} {
@@ -529,6 +539,10 @@ proc start_server {options {code undefined}} {
                 # Re-raise, let handler up the stack take care of this.
                 error $error $backtrace
             }
+        } else {
+            if {$::dump_logs && $prev_num_failed != $::num_failed} {
+                dump_server_log $srv
+            }
         }
 
         # fetch srv back from the server list, in case it was restarted by restart_server (new PID)
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 2b7854780..eb14b5ee9 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -110,6 +110,7 @@ set ::active_servers {} ; # Pids of active Redis instances.
 set ::dont_clean 0
 set ::wait_server 0
 set ::stop_on_failure 0
+set ::dump_logs 0
 set ::loop 0
 set ::tlsdir "tests/tls"
 
@@ -555,6 +556,7 @@ proc print_help_screen {} {
         "--stop             Blocks once the first test fails."
         "--loop             Execute the specified set of tests forever."
         "--wait-server      Wait after server is started (so that you can attach a debugger)."
+        "--dump-logs        Dump server log on test failure."
         "--tls              Run tests in TLS mode."
         "--host <addr>      Run tests against an external host."
         "--port <port>      TCP port to use against external host."
@@ -657,6 +659,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
         set ::no_latency 1
     } elseif {$opt eq {--wait-server}} {
         set ::wait_server 1
+    } elseif {$opt eq {--dump-logs}} {
+        set ::dump_logs 1
     } elseif {$opt eq {--stop}} {
         set ::stop_on_failure 1
     } elseif {$opt eq {--loop}} {

From 018f7b7378d576ef80f2af11293713cc1a26da53 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 7 Feb 2021 15:41:49 +0200
Subject: [PATCH 1035/1098] Update CI on Ubuntu to tcl8.6 (since 20.04 is now
 used) (#8460)

Github started shifting some repositoreis to use ubuntu 20.04 by default
tcl8.5 is missing in these, but 8.6 exists in both 20.04 and 18.04
---
 .github/workflows/ci.yml    |  2 +-
 .github/workflows/daily.yml | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2582c53a4..39b332a17 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -14,7 +14,7 @@ jobs:
       run: make REDIS_CFLAGS='-Werror' BUILD_TLS=yes
     - name: test
       run: |
-        sudo apt-get install tcl8.5
+        sudo apt-get install tcl8.6
         ./runtest --verbose
     - name: module api test
       run: ./runtest-moduleapi --verbose
diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index e3cb1a344..0a899d174 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -20,7 +20,7 @@ jobs:
       run: make
     - name: test
       run: |
-        sudo apt-get install tcl8.5
+        sudo apt-get install tcl8.6
         ./runtest --accurate --verbose --dump-logs
     - name: module api test
       run: ./runtest-moduleapi --verbose
@@ -39,7 +39,7 @@ jobs:
       run: make MALLOC=libc
     - name: test
       run: |
-        sudo apt-get install tcl8.5
+        sudo apt-get install tcl8.6
         ./runtest --accurate --verbose --dump-logs
     - name: module api test
       run: ./runtest-moduleapi --verbose
@@ -60,7 +60,7 @@ jobs:
         make 32bit
     - name: test
       run: |
-        sudo apt-get install tcl8.5
+        sudo apt-get install tcl8.6
         ./runtest --accurate --verbose --dump-logs
     - name: module api test
       run: |
@@ -82,7 +82,7 @@ jobs:
         make BUILD_TLS=yes
     - name: test
       run: |
-        sudo apt-get install tcl8.5 tcl-tls
+        sudo apt-get install tcl8.6 tcl-tls
         ./utils/gen-test-certs.sh
         ./runtest --accurate --verbose --tls --dump-logs
         ./runtest --accurate --verbose --dump-logs
@@ -110,7 +110,7 @@ jobs:
         make
     - name: test
       run: |
-        sudo apt-get install tcl8.5 tcl-tls
+        sudo apt-get install tcl8.6 tcl-tls
         ./runtest --config io-threads 4 --config io-threads-do-reads yes --accurate --verbose --tags network --dump-logs
     - name: cluster tests
       run: |
@@ -127,7 +127,7 @@ jobs:
     - name: test
       run: |
         sudo apt-get update
-        sudo apt-get install tcl8.5 valgrind -y
+        sudo apt-get install tcl8.6 valgrind -y
         ./runtest --valgrind --verbose --clients 1 --dump-logs
     - name: module api test
       run: ./runtest-moduleapi --valgrind --verbose --clients 1

From 02ab14cc2e4f4981001b90fe3bb84cab1964d3ea Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 7 Feb 2021 16:22:30 +0200
Subject: [PATCH 1036/1098] solve race in replication-2 test (#8461)

use SIGSTOP instead of DEBUG SLEEP, reduces the test
time by some 2 seconds and avoids failures on slow machines
---
 tests/integration/replication-2.tcl | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tests/integration/replication-2.tcl b/tests/integration/replication-2.tcl
index 08905f11e..fc67e78c1 100644
--- a/tests/integration/replication-2.tcl
+++ b/tests/integration/replication-2.tcl
@@ -39,17 +39,15 @@ start_server {tags {"repl"}} {
         }
 
         test {No write if min-slaves-max-lag is > of the slave lag} {
-            r -1 deferred 1
             r config set min-slaves-to-write 1
             r config set min-slaves-max-lag 2
-            r -1 debug sleep 6
+            exec kill -SIGSTOP [srv -1 pid]
             assert {[r set foo 12345] eq {OK}}
             after 4000
             catch {r set foo 12345} err
-            assert {[r -1 read] eq {OK}}
-            r -1 deferred 0
             set err
         } {NOREPLICAS*}
+        exec kill -SIGCONT [srv -1 pid]
 
         test {min-slaves-to-write is ignored by slaves} {
             r config set min-slaves-to-write 1

From 62b1f32062b8f688179a8262959a5b80d0ad4de7 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Sun, 7 Feb 2021 16:55:11 +0200
Subject: [PATCH 1037/1098] Optimize HRANDFIELD and ZRANDMEMBER case 4 when
 ziplist encoded (#8444)

It is inefficient to repeatedly pick a single random element from a
ziplist.
For CASE4, which is when the user requested a low number of unique
random picks from the collectoin, we used thta pattern.

Now we use a different algorithm that picks unique elements from a
ziplist, and guarentee no duplicate but doesn't provide random order
(which is only needed in the non-unique random picks case)

Unrelated changes:
* change ziplist count and indexes variables to unsigned
* solve compilation warnings about uninitialized vars in gcc 10.2

Co-authored-by: xinluton <xinluton@qq.com>
---
 src/t_hash.c  | 47 +++++++++++++++++++++----------
 src/t_zset.c  | 47 +++++++++++++++++++++----------
 src/ziplist.c | 76 +++++++++++++++++++++++++++++++++++++++++----------
 src/ziplist.h |  3 +-
 4 files changed, 129 insertions(+), 44 deletions(-)

diff --git a/src/t_hash.c b/src/t_hash.c
index 98ba69df7..082e0f129 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -959,6 +959,23 @@ void hscanCommand(client *c) {
     scanGenericCommand(c,o,cursor);
 }
 
+static void harndfieldReplyWithZiplist(client *c, unsigned int count, ziplistEntry *keys, ziplistEntry *vals) {
+    for (unsigned long i = 0; i < count; i++) {
+        if (vals && c->resp > 2)
+            addReplyArrayLen(c,2);
+        if (keys[i].sval)
+            addReplyBulkCBuffer(c, keys[i].sval, keys[i].slen);
+        else
+            addReplyBulkLongLong(c, keys[i].lval);
+        if (vals) {
+            if (vals[i].sval)
+                addReplyBulkCBuffer(c, vals[i].sval, vals[i].slen);
+            else
+                addReplyBulkLongLong(c, vals[i].lval);
+        }
+    }
+}
+
 /* How many times bigger should be the hash compared to the requested size
  * for us to not use the "remove elements" strategy? Read later in the
  * implementation for more info. */
@@ -1024,20 +1041,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
                 sample_count = count > limit ? limit : count;
                 count -= sample_count;
                 ziplistRandomPairs(hash->ptr, sample_count, keys, vals);
-                for (unsigned long i = 0; i < sample_count; i++) {
-                    if (withvalues && c->resp > 2)
-                        addReplyArrayLen(c,2);
-                    if (keys[i].sval)
-                        addReplyBulkCBuffer(c, keys[i].sval, keys[i].slen);
-                    else
-                        addReplyBulkLongLong(c, keys[i].lval);
-                    if (withvalues) {
-                        if (vals[i].sval)
-                            addReplyBulkCBuffer(c, vals[i].sval, vals[i].slen);
-                        else
-                            addReplyBulkLongLong(c, vals[i].lval);
-                    }
-                }
+                harndfieldReplyWithZiplist(c, sample_count, keys, vals);
             }
             zfree(keys);
             zfree(vals);
@@ -1130,6 +1134,21 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
      * to the temporary hash, trying to eventually get enough unique elements
      * to reach the specified count. */
     else {
+        if (hash->encoding == OBJ_ENCODING_ZIPLIST) {
+            /* it is inefficient to repeatedly pick one random element from a
+             * ziplist. so we use this instead: */
+            ziplistEntry *keys, *vals = NULL;
+            keys = zmalloc(sizeof(ziplistEntry)*count);
+            if (withvalues)
+                vals = zmalloc(sizeof(ziplistEntry)*count);
+            serverAssert(ziplistRandomPairsUnique(hash->ptr, count, keys, vals) == count);
+            harndfieldReplyWithZiplist(c, count, keys, vals);
+            zfree(keys);
+            zfree(vals);
+            return;
+        }
+
+        /* Hashtable encoding (generic implementation) */
         unsigned long added = 0;
         ziplistEntry key, value;
         dict *d = dictCreate(&hashDictType, NULL);
diff --git a/src/t_zset.c b/src/t_zset.c
index c14a37263..869cf6c9a 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -3956,6 +3956,23 @@ void bzpopmaxCommand(client *c) {
     blockingGenericZpopCommand(c,ZSET_MAX);
 }
 
+static void zarndmemberReplyWithZiplist(client *c, unsigned int count, ziplistEntry *keys, ziplistEntry *vals) {
+    for (unsigned long i = 0; i < count; i++) {
+        if (vals && c->resp > 2)
+            addReplyArrayLen(c,2);
+        if (keys[i].sval)
+            addReplyBulkCBuffer(c, keys[i].sval, keys[i].slen);
+        else
+            addReplyBulkLongLong(c, keys[i].lval);
+        if (vals) {
+            if (vals[i].sval) {
+                addReplyDouble(c, zzlStrtod(vals[i].sval,vals[i].slen));
+            } else
+                addReplyDouble(c, vals[i].lval);
+        }
+    }
+}
+
 /* How many times bigger should be the zset compared to the requested size
  * for us to not use the "remove elements" strategy? Read later in the
  * implementation for more info. */
@@ -4020,20 +4037,7 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) {
                 sample_count = count > limit ? limit : count;
                 count -= sample_count;
                 ziplistRandomPairs(zsetobj->ptr, sample_count, keys, vals);
-                for (unsigned long i = 0; i < sample_count; i++) {
-                    if (withscores && c->resp > 2)
-                        addReplyArrayLen(c,2);
-                    if (keys[i].sval)
-                        addReplyBulkCBuffer(c, keys[i].sval, keys[i].slen);
-                    else
-                        addReplyBulkLongLong(c, keys[i].lval);
-                    if (withscores) {
-                        if (vals[i].sval) {
-                            addReplyDouble(c, zzlStrtod(vals[i].sval,vals[i].slen));
-                        } else
-                            addReplyDouble(c, vals[i].lval);
-                    }
-                }
+                zarndmemberReplyWithZiplist(c, sample_count, keys, vals);
             }
             zfree(keys);
             zfree(vals);
@@ -4122,6 +4126,21 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) {
      * to the temporary set, trying to eventually get enough unique elements
      * to reach the specified count. */
     else {
+        if (zsetobj->encoding == OBJ_ENCODING_ZIPLIST) {
+            /* it is inefficient to repeatedly pick one random element from a
+             * ziplist. so we use this instead: */
+            ziplistEntry *keys, *vals = NULL;
+            keys = zmalloc(sizeof(ziplistEntry)*count);
+            if (withscores)
+                vals = zmalloc(sizeof(ziplistEntry)*count);
+            serverAssert(ziplistRandomPairsUnique(zsetobj->ptr, count, keys, vals) == count);
+            zarndmemberReplyWithZiplist(c, count, keys, vals);
+            zfree(keys);
+            zfree(vals);
+            return;
+        }
+
+        /* Hashtable encoding (generic implementation) */
         unsigned long added = 0;
         dict *d = dictCreate(&hashDictType, NULL);
 
diff --git a/src/ziplist.c b/src/ziplist.c
index 248166ac2..2c0074d5d 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -1523,8 +1523,8 @@ void ziplistRandomPair(unsigned char *zl, unsigned long total_count, ziplistEntr
 }
 
 /* int compare for qsort */
-int intCompare(const void *a, const void *b) {
-    return (*(int *) a - *(int *) b);
+int uintCompare(const void *a, const void *b) {
+    return (*(unsigned int *) a - *(unsigned int *) b);
 }
 
 /* Helper method to store a string into from val or lval into dest */
@@ -1534,41 +1534,42 @@ static inline void ziplistSaveValue(unsigned char *val, unsigned int len, long l
     dest->lval = lval;
 }
 
-/* Randomly select unique count of key value pairs and store into 'keys' and
- * 'vals' args. The order of the picked entries is random.
+/* Randomly select count of key value pairs and store into 'keys' and
+ * 'vals' args. The order of the picked entries is random, and the selections
+ * are non-unique (repetitions are possible).
  * The 'vals' arg can be NULL in which case we skip these. */
-void ziplistRandomPairs(unsigned char *zl, int count, ziplistEntry *keys, ziplistEntry *vals) {
+void ziplistRandomPairs(unsigned char *zl, unsigned int count, ziplistEntry *keys, ziplistEntry *vals) {
     unsigned char *p, *key, *value;
-    unsigned int klen, vlen;
-    long long klval, vlval;
+    unsigned int klen = 0, vlen = 0;
+    long long klval = 0, vlval = 0;
 
-    /* Notice: the index member must be first due to the use in intCompare */
+    /* Notice: the index member must be first due to the use in uintCompare */
     typedef struct {
-        int index;
-        int order;
+        unsigned int index;
+        unsigned int order;
     } rand_pick;
     rand_pick *picks = zmalloc(sizeof(rand_pick)*count);
-    unsigned long total_size = ziplistLen(zl)/2;
+    unsigned int total_size = ziplistLen(zl)/2;
 
     /* Avoid div by zero on corrupt ziplist */
     assert(total_size);
 
     /* create a pool of random indexes (some may be duplicate). */
-    for (int i = 0; i < count; i++) {
+    for (unsigned int i = 0; i < count; i++) {
         picks[i].index = (rand() % total_size) * 2; /* Generate even indexes */
         /* keep track of the order we picked them */
         picks[i].order = i;
     }
 
     /* sort by indexes. */
-    qsort(picks, count, sizeof(rand_pick), intCompare);
+    qsort(picks, count, sizeof(rand_pick), uintCompare);
 
     /* fetch the elements form the ziplist into a output array respecting the original order. */
-    int zipindex = 0, pickindex = 0;
+    unsigned int zipindex = 0, pickindex = 0;
     p = ziplistIndex(zl, 0);
     while (ziplistGet(p, &key, &klen, &klval) && pickindex < count) {
         p = ziplistNext(zl, p);
-        ziplistGet(p, &value, &vlen, &vlval);
+        assert(ziplistGet(p, &value, &vlen, &vlval));
         while (pickindex < count && zipindex == picks[pickindex].index) {
             int storeorder = picks[pickindex].order;
             ziplistSaveValue(key, klen, klval, &keys[storeorder]);
@@ -1583,6 +1584,51 @@ void ziplistRandomPairs(unsigned char *zl, int count, ziplistEntry *keys, ziplis
     zfree(picks);
 }
 
+/* Randomly select count of key value pairs and store into 'keys' and
+ * 'vals' args. The selections are unique (no repetitions), and the order of
+ * the picked entries is NOT-random.
+ * The 'vals' arg can be NULL in which case we skip these.
+ * The return value is the number of items picked which can be lower than the
+ * requested count if the ziplist doesn't hold enough pairs. */
+unsigned int ziplistRandomPairsUnique(unsigned char *zl, unsigned int count, ziplistEntry *keys, ziplistEntry *vals) {
+    unsigned char *p, *key;
+    unsigned int klen = 0;
+    long long klval = 0;
+    unsigned int total_size = ziplistLen(zl)/2;
+    unsigned int index = 0;
+    if (count > total_size)
+        count = total_size;
+
+    /* To only iterate once, every time we try to pick a member, the probability
+     * we pick it is the quotient of the count left we want to pick and the
+     * count still we haven't visited in the dict, this way, we could make every
+     * member be equally picked.*/
+    p = ziplistIndex(zl, 0);
+    unsigned int picked = 0, remaining = count;
+    while (picked < count && p) {
+        double randomDouble = ((double)rand()) / RAND_MAX;
+        double threshold = ((double)remaining) / (total_size - index);
+        if(randomDouble < threshold){
+            assert(ziplistGet(p, &key, &klen, &klval));
+            ziplistSaveValue(key, klen, klval, &keys[picked]);
+            p = ziplistNext(zl, p);
+            assert(p);
+            if (vals) {
+                assert(ziplistGet(p, &key, &klen, &klval));
+                ziplistSaveValue(key, klen, klval, &vals[picked]);
+            }
+            remaining--;
+            picked++;
+        } else {
+            p = ziplistNext(zl, p);
+            assert(p);
+        }
+        p = ziplistNext(zl, p);
+        index++;
+    }
+    return picked;
+}
+
 #ifdef REDIS_TEST
 #include <sys/time.h>
 #include "adlist.h"
diff --git a/src/ziplist.h b/src/ziplist.h
index 5fb8fd46a..fff334cbd 100644
--- a/src/ziplist.h
+++ b/src/ziplist.h
@@ -62,7 +62,8 @@ typedef int (*ziplistValidateEntryCB)(unsigned char* p, void* userdata);
 int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep,
                              ziplistValidateEntryCB entry_cb, void *cb_userdata);
 void ziplistRandomPair(unsigned char *zl, unsigned long total_count, ziplistEntry *key, ziplistEntry *val);
-void ziplistRandomPairs(unsigned char *zl, int count, ziplistEntry *keys, ziplistEntry *vals);
+void ziplistRandomPairs(unsigned char *zl, unsigned int count, ziplistEntry *keys, ziplistEntry *vals);
+unsigned int ziplistRandomPairsUnique(unsigned char *zl, unsigned int count, ziplistEntry *keys, ziplistEntry *vals);
 
 #ifdef REDIS_TEST
 int ziplistTest(int argc, char *argv[]);

From 88272cf7ac926eb0be74d6ec6f0d0ba0afaa7102 Mon Sep 17 00:00:00 2001
From: Andy Pan <panjf2000@gmail.com>
Date: Mon, 8 Feb 2021 18:09:39 +0800
Subject: [PATCH 1038/1098] Fix typos in comments (#8466)

---
 src/cluster.c    | 2 +-
 src/networking.c | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index 97a25b0b3..e3fab02e9 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -2815,7 +2815,7 @@ void clusterPropagatePublish(robj *channel, robj *message) {
  * SLAVE node specific functions
  * -------------------------------------------------------------------------- */
 
-/* This function sends a FAILOVE_AUTH_REQUEST message to every node in order to
+/* This function sends a FAILOVER_AUTH_REQUEST message to every node in order to
  * see if there is the quorum for this slave instance to failover its failing
  * master.
  *
diff --git a/src/networking.c b/src/networking.c
index f9a3ff86a..5a465c460 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -527,7 +527,7 @@ void trimReplyUnusedTailSpace(client *c) {
     listNode *ln = listLast(c->reply);
     clientReplyBlock *tail = ln? listNodeValue(ln): NULL;
 
-    /* Note that 'tail' may be NULL even if we have a tail node, becuase when
+    /* Note that 'tail' may be NULL even if we have a tail node, because when
      * addReplyDeferredLen() is used */
     if (!tail) return;
 
@@ -1426,7 +1426,7 @@ void freeClientAsync(client *c) {
     pthread_mutex_unlock(&async_free_queue_mutex);
 }
 
-/* Free the clietns marked as CLOSE_ASAP, return the number of clients
+/* Free the clients marked as CLOSE_ASAP, return the number of clients
  * freed. */
 int freeClientsInAsyncFreeQueue(void) {
     int freed = 0;
@@ -3519,7 +3519,7 @@ int handleClientsWithPendingWritesUsingThreads(void) {
     if (processed == 0) return 0; /* Return ASAP if there are no clients. */
 
     /* If I/O threads are disabled or we have few clients to serve, don't
-     * use I/O threads, but thejboring synchronous code. */
+     * use I/O threads, but the boring synchronous code. */
     if (server.io_threads_num == 1 || stopThreadedIOIfNeeded()) {
         return handleClientsWithPendingWrites();
     }

From 1f12be30723abf6487e65557be501e54a0ed8463 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Mon, 8 Feb 2021 18:41:16 +0800
Subject: [PATCH 1039/1098] Fix random probability check in
 ziplistRandomPairsUnique (#8467)

When (remaining == (total_size - index)), element will definitely be random to.
But when rand() == RAND_MAX, the element will miss, this will trigger assert
in serverAssert(ziplistRandomPairsUnique(zsetobj->ptr, count, keys, vals) == count).
---
 src/ziplist.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ziplist.c b/src/ziplist.c
index 2c0074d5d..35a0f283a 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -1608,7 +1608,7 @@ unsigned int ziplistRandomPairsUnique(unsigned char *zl, unsigned int count, zip
     while (picked < count && p) {
         double randomDouble = ((double)rand()) / RAND_MAX;
         double threshold = ((double)remaining) / (total_size - index);
-        if(randomDouble < threshold){
+        if (randomDouble <= threshold) {
             assert(ziplistGet(p, &key, &klen, &klval));
             ziplistSaveValue(key, klen, klval, &keys[picked]);
             p = ziplistNext(zl, p);

From b2351ea0dc9f60e7987dc6db4c1ef09d70003a9e Mon Sep 17 00:00:00 2001
From: filipe oliveira <filipecosta.90@gmail.com>
Date: Mon, 8 Feb 2021 14:24:00 +0000
Subject: [PATCH 1040/1098] [fix] Increasing block on background timeout time
 to avoid test failure (#8470)

The test failed from time to time on Github actions.
We think it's possible that on the module's blocking timeout
time tracking test, the timeout is happening prior we issue the
RedisModule_BlockedClientMeasureTimeStart(bc) on the
background thread. If that is the case one possible solution
is to increase the timeout.
Increasing to 200ms to 500ms to see if nightly stops failing.
---
 tests/unit/moduleapi/blockonbackground.tcl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/unit/moduleapi/blockonbackground.tcl b/tests/unit/moduleapi/blockonbackground.tcl
index 23111ab73..46a72bfd2 100644
--- a/tests/unit/moduleapi/blockonbackground.tcl
+++ b/tests/unit/moduleapi/blockonbackground.tcl
@@ -29,14 +29,14 @@ start_server {tags {"modules"}} {
         r block.debug 0 20000
         assert_equal [r slowlog len] 0
         r config resetstat
-        r block.debug 20000 200
+        r block.debug 20000 500
         assert_equal [r slowlog len] 1
 
         set cmdstatline [cmdrstat block.debug r]
 
         regexp "calls=1,usec=(.*?),usec_per_call=(.*?),rejected_calls=0,failed_calls=0" $cmdstatline usec usec_per_call
-        assert {$usec >= 100000}
-        assert {$usec_per_call >= 100000}
+        assert {$usec >= 250000}
+        assert {$usec_per_call >= 250000}
     }
 
     test { blocked clients time tracking - check blocked command with multiple calls RedisModule_BlockedClientMeasureTimeStart()  is tracking the total background time } {

From dbcc0a85d070f94f5c524a4bbdd3e492c32e845b Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 8 Feb 2021 17:02:46 +0200
Subject: [PATCH 1041/1098] Fix and cleanup Sentinel leaked fds test. (#8469)

* For consistency, use tclsh for the script as well
* Ignore leaked fds that originate from grandparent process, since we
  only care about fds redis-sentinel itself is responsible for
* Check every test iteration to catch problems early
* Some cleanups, e.g. parameterization of file name, etc.
---
 tests/instances.tcl                           | 27 +++----
 tests/sentinel/run.tcl                        |  3 +
 .../tests/helpers/check_leaked_fds.tcl        | 70 +++++++++++++++++++
 tests/sentinel/tests/includes/init-tests.tcl  |  6 +-
 tests/sentinel/tests/includes/notify.sh       | 21 ------
 5 files changed, 91 insertions(+), 36 deletions(-)
 create mode 100755 tests/sentinel/tests/helpers/check_leaked_fds.tcl
 delete mode 100755 tests/sentinel/tests/includes/notify.sh

diff --git a/tests/instances.tcl b/tests/instances.tcl
index 8cb616ae8..793bce80d 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -29,6 +29,7 @@ set ::sentinel_base_port 20000
 set ::redis_base_port 30000
 set ::redis_port_count 1024
 set ::host "127.0.0.1"
+set ::leaked_fds_file [file normalize "tmp/leaked_fds.txt"]
 set ::pids {} ; # We kill everything at exit
 set ::dirs {} ; # We remove all the temp dirs at exit
 set ::run_matching {} ; # If non empty, only tests matching pattern are run.
@@ -410,13 +411,13 @@ proc check_leaks instance_types {
 
 # Execute all the units inside the 'tests' directory.
 proc run_tests {} {
-    set sentinel_fd_leaks_file "sentinel_fd_leaks"
-    if { [file exists $sentinel_fd_leaks_file] } {
-        file delete $sentinel_fd_leaks_file
-    }
-
     set tests [lsort [glob ../tests/*]]
     foreach test $tests {
+        # Remove leaked_fds file before starting
+        if {$::leaked_fds_file != "" && [file exists $::leaked_fds_file]} {
+            file delete $::leaked_fds_file
+        }
+
         if {$::run_matching ne {} && [string match $::run_matching $test] == 0} {
             continue
         }
@@ -424,19 +425,19 @@ proc run_tests {} {
         puts [colorstr yellow "Testing unit: [lindex [file split $test] end]"]
         source $test
         check_leaks {redis sentinel}
+
+        # Check if a leaked fds file was created and abort the test.
+        if {$::leaked_fds_file != "" && [file exists $::leaked_fds_file]} {
+            puts [colorstr red "ERROR: Sentinel has leaked fds to scripts:"]
+            puts [exec cat $::leaked_fds_file]
+            puts "----"
+            incr ::failed
+        }
     }
 }
 
 # Print a message and exists with 0 / 1 according to zero or more failures.
 proc end_tests {} {
-    set sentinel_fd_leaks_file "sentinel_fd_leaks"
-    if { [file exists $sentinel_fd_leaks_file] } {
-        # temporarily disabling this error from failing the tests until leaks are fixed.
-        #puts [colorstr red "WARNING: sentinel test(s) failed, there are leaked fds in sentinel:"] 
-        #puts [exec cat $sentinel_fd_leaks_file]
-        #exit 1
-    }
-
     if {$::failed == 0 } {
         puts "GOOD! No errors."
         exit 0
diff --git a/tests/sentinel/run.tcl b/tests/sentinel/run.tcl
index c275aa762..64c99c103 100644
--- a/tests/sentinel/run.tcl
+++ b/tests/sentinel/run.tcl
@@ -10,6 +10,9 @@ set ::tlsdir "../../tls"
 
 proc main {} {
     parse_options
+    if {$::leaked_fds_file != ""} {
+        set ::env(LEAKED_FDS_FILE) $::leaked_fds_file
+    }
     spawn_instance sentinel $::sentinel_base_port $::instances_count [list "sentinel deny-scripts-reconfig no"] "../tests/includes/sentinel.conf"
     spawn_instance redis $::redis_base_port $::instances_count
     run_tests
diff --git a/tests/sentinel/tests/helpers/check_leaked_fds.tcl b/tests/sentinel/tests/helpers/check_leaked_fds.tcl
new file mode 100755
index 000000000..cb7b5d3d9
--- /dev/null
+++ b/tests/sentinel/tests/helpers/check_leaked_fds.tcl
@@ -0,0 +1,70 @@
+#!/usr/bin/env tclsh
+#
+# This script detects file descriptors that have leaked from a parent process.
+#
+# Our goal is to detect file descriptors that were opened by the parent and
+# not cleaned up prior to exec(), but not file descriptors that were inherited
+# from the grandparent which the parent knows nothing about. To do that, we
+# look up every potential leak and try to match it against open files by the
+# grandparent process.
+
+# Get PID of parent process
+proc get_parent_pid {_pid} {
+    set fd [open "/proc/$_pid/status" "r"]
+    set content [read $fd]
+    close $fd
+
+    if {[regexp {\nPPid:\s+(\d+)} $content _ ppid]} {
+        return $ppid
+    }
+
+    error "failed to get parent pid"
+}
+
+# Linux only
+set os [exec uname]
+if {$os != "Linux"} {
+    puts "Only Linux is supported."
+    exit 0
+}
+
+if {![info exists env(LEAKED_FDS_FILE)]} {
+    puts "Missing LEAKED_FDS_FILE environment variable."
+    exit 0
+}
+
+set outfile $::env(LEAKED_FDS_FILE)
+set parent_pid [get_parent_pid [pid]]
+set grandparent_pid [get_parent_pid $parent_pid]
+set leaked_fds {}
+
+# Look for fds that were directly inherited from our parent but not from
+# our grandparent (tcl)
+foreach fd [glob -tails -directory "/proc/self/fd" *] {
+    # Ignore stdin/stdout/stderr
+    if {$fd == 0 || $fd == 1 || $fd == 2} {
+        continue
+    }
+
+    # Read symlink to get info about the file descriptor. This can be a real
+    # file name or an arbitrary string that identifies the fd.
+    if { [catch {set fdlink [file readlink "/proc/self/fd/$fd"]} err] } {
+        continue
+    }
+
+    # See if grandparent process has the same fd and info and skip if it does.
+    if { ! [catch {set gp_fdlink [file readlink "/proc/$grandparent_pid/fd/$fd"]} err] } {
+        if {$gp_fdlink == $fdlink} {
+            continue
+        }
+    }
+
+    lappend leaked_fds [list $fd $fdlink]
+}
+
+# Produce report only if we found leaks
+if {[llength $leaked_fds] > 0} {
+    set fd [open $outfile "w"]
+    puts $fd [join $leaked_fds "\n"]
+    close $fd
+}
diff --git a/tests/sentinel/tests/includes/init-tests.tcl b/tests/sentinel/tests/includes/init-tests.tcl
index b4626caed..be6a8f502 100644
--- a/tests/sentinel/tests/includes/init-tests.tcl
+++ b/tests/sentinel/tests/includes/init-tests.tcl
@@ -41,8 +41,10 @@ test "(init) Sentinels can start monitoring a master" {
         S $id SENTINEL SET mymaster down-after-milliseconds 2000
         S $id SENTINEL SET mymaster failover-timeout 20000
         S $id SENTINEL SET mymaster parallel-syncs 10
-        S $id SENTINEL SET mymaster notification-script ../../tests/includes/notify.sh
-        S $id SENTINEL SET mymaster client-reconfig-script ../../tests/includes/notify.sh
+        if {$::leaked_fds_file != ""} {
+            S $id SENTINEL SET mymaster notification-script ../../tests/helpers/check_leaked_fds.tcl
+            S $id SENTINEL SET mymaster client-reconfig-script ../../tests/helpers/check_leaked_fds.tcl
+        }
     }
 }
 
diff --git a/tests/sentinel/tests/includes/notify.sh b/tests/sentinel/tests/includes/notify.sh
deleted file mode 100755
index 5de0eaf76..000000000
--- a/tests/sentinel/tests/includes/notify.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-OS=`uname -s`
-if [ ${OS} != "Linux" ]
-then
-    exit 0
-fi
-
-# fd 3 is meant to catch the actual access to /proc/pid/fd, 
-# in case there's an fd leak by the sentinel,
-# it can take 3, but then the access to /proc will take another fd, and we'll catch that.
-leaked_fd_count=`ls /proc/self/fd | grep -vE '^[0|1|2|3]$' | wc -l`
-if [ $leaked_fd_count -gt 0 ]
-then
-    sentinel_fd_leaks_file="../sentinel_fd_leaks"
-    if [ ! -f $sentinel_fd_leaks_file ]
-    then
-        ls -l /proc/self/fd | cat >> $sentinel_fd_leaks_file
-        lsof -p $$ | cat >> $sentinel_fd_leaks_file
-    fi
-fi

From 8f9958dc24fa5992d3d10f6b9caf999e1beee4e5 Mon Sep 17 00:00:00 2001
From: Huang Zw <huang_zhw@126.com>
Date: Tue, 9 Feb 2021 01:29:32 +0800
Subject: [PATCH 1042/1098] Fix typo and some out of date comments (#8449)

Fix typo and some out of date comments
---
 src/db.c        | 5 ++---
 src/defrag.c    | 4 ++--
 src/evict.c     | 2 +-
 src/expire.c    | 7 +++----
 src/quicklist.c | 2 +-
 src/rdb.h       | 2 +-
 src/t_string.c  | 2 +-
 src/tls.c       | 2 +-
 8 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/src/db.c b/src/db.c
index 3871753dd..ed522401f 100644
--- a/src/db.c
+++ b/src/db.c
@@ -106,9 +106,8 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) {
     robj *val;
 
     if (expireIfNeeded(db,key) == 1) {
-        /* Key expired. If we are in the context of a master, expireIfNeeded()
-         * returns 0 only when the key does not exist at all, so it's safe
-         * to return NULL ASAP. */
+        /* If we are in the context of a master, expireIfNeeded() returns 1
+         * when the key is no longer valid, so we can return NULL ASAP. */
         if (server.masterhost == NULL)
             goto keymiss;
 
diff --git a/src/defrag.c b/src/defrag.c
index db797711e..2b28d523d 100644
--- a/src/defrag.c
+++ b/src/defrag.c
@@ -638,7 +638,7 @@ int defragRaxNode(raxNode **noderef) {
 }
 
 /* returns 0 if no more work needs to be been done, and 1 if time is up and more work is needed. */
-int scanLaterStraemListpacks(robj *ob, unsigned long *cursor, long long endtime, long long *defragged) {
+int scanLaterStreamListpacks(robj *ob, unsigned long *cursor, long long endtime, long long *defragged) {
     static unsigned char last[sizeof(streamID)];
     raxIterator ri;
     long iterations = 0;
@@ -958,7 +958,7 @@ int defragLaterItem(dictEntry *de, unsigned long *cursor, long long endtime) {
         } else if (ob->type == OBJ_HASH) {
             server.stat_active_defrag_hits += scanLaterHash(ob, cursor);
         } else if (ob->type == OBJ_STREAM) {
-            return scanLaterStraemListpacks(ob, cursor, endtime, &server.stat_active_defrag_hits);
+            return scanLaterStreamListpacks(ob, cursor, endtime, &server.stat_active_defrag_hits);
         } else if (ob->type == OBJ_MODULE) {
             return moduleLateDefrag(dictGetKey(de), ob, cursor, endtime, &server.stat_active_defrag_hits);
         } else {
diff --git a/src/evict.c b/src/evict.c
index 04513cd1a..d3cdaaa80 100644
--- a/src/evict.c
+++ b/src/evict.c
@@ -135,7 +135,7 @@ void evictionPoolAlloc(void) {
 
 /* This is an helper function for performEvictions(), it is used in order
  * to populate the evictionPool with a few entries every time we want to
- * expire a key. Keys with idle time smaller than one of the current
+ * expire a key. Keys with idle time bigger than one of the current
  * keys are added. Keys are always added if there are free entries.
  *
  * We insert keys on place in ascending order, so keys with the smaller
diff --git a/src/expire.c b/src/expire.c
index f79510817..0ce34aea8 100644
--- a/src/expire.c
+++ b/src/expire.c
@@ -83,9 +83,8 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) {
  * keys that can be removed from the keyspace.
  *
  * Every expire cycle tests multiple databases: the next call will start
- * again from the next db, with the exception of exists for time limit: in that
- * case we restart again from the last database we were processing. Anyway
- * no more than CRON_DBS_PER_CALL databases are tested at every iteration.
+ * again from the next db. No more than CRON_DBS_PER_CALL databases are
+ * tested at every iteration.
  *
  * The function can perform more or less work, depending on the "type"
  * argument. It can execute a "fast cycle" or a "slow cycle". The slow
@@ -141,7 +140,7 @@ void activeExpireCycle(int type) {
 
     /* This function has some global state in order to continue the work
      * incrementally across calls. */
-    static unsigned int current_db = 0; /* Last DB tested. */
+    static unsigned int current_db = 0; /* Next DB to test. */
     static int timelimit_exit = 0;      /* Time limit hit in previous call? */
     static long long last_fast_cycle = 0; /* When last fast cycle ran. */
 
diff --git a/src/quicklist.c b/src/quicklist.c
index 32871ac12..b2a406004 100644
--- a/src/quicklist.c
+++ b/src/quicklist.c
@@ -1427,7 +1427,7 @@ void quicklistPush(quicklist *quicklist, void *value, const size_t sz,
  * Returns 1 on success (creation of new bookmark or override of an existing one).
  * Returns 0 on failure (reached the maximum supported number of bookmarks).
  * NOTE: use short simple names, so that string compare on find is quick.
- * NOTE: bookmakrk creation may re-allocate the quicklist, so the input pointer
+ * NOTE: bookmark creation may re-allocate the quicklist, so the input pointer
          may change and it's the caller responsibilty to update the reference.
  */
 int quicklistBookmarkCreate(quicklist **ql_ref, const char *name, quicklistNode *node) {
diff --git a/src/rdb.h b/src/rdb.h
index f22fbecd1..00ed5297c 100644
--- a/src/rdb.h
+++ b/src/rdb.h
@@ -45,7 +45,7 @@
  * the first byte to interpreter the length:
  *
  * 00|XXXXXX => if the two MSB are 00 the len is the 6 bits of this byte
- * 01|XXXXXX XXXXXXXX =>  01, the len is 14 byes, 6 bits + 8 bits of next byte
+ * 01|XXXXXX XXXXXXXX =>  01, the len is 14 bits, 6 bits + 8 bits of next byte
  * 10|000000 [32 bit integer] => A full 32 bit len in net byte order will follow
  * 10|000001 [64 bit integer] => A full 64 bit len in net byte order will follow
  * 11|OBKIND this means: specially encoded object will follow. The six bits
diff --git a/src/t_string.c b/src/t_string.c
index de67484fc..d05366b7c 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -764,7 +764,7 @@ void stralgoLCS(client *c) {
                 addReplyError(c,
                     "The specified keys must contain string values");
                 /* Don't cleanup the objects, we need to do that
-                 * only after callign getDecodedObject(). */
+                 * only after calling getDecodedObject(). */
                 obja = NULL;
                 objb = NULL;
                 goto cleanup;
diff --git a/src/tls.c b/src/tls.c
index 43c2f5551..bcfe53a35 100644
--- a/src/tls.c
+++ b/src/tls.c
@@ -350,7 +350,7 @@ ConnectionType CT_TLS;
  * socket operation.
  *
  * When this happens, we need to do two things:
- * 1. Make sure we register for the even.
+ * 1. Make sure we register for the event.
  * 2. Make sure we know which handler needs to execute when the
  *    event fires.  That is, if we notify the caller of a write operation
  *    that it blocks, and SSL asks for a read, we need to trigger the

From 203f357c32f82b0e10c3e55721fccd07bc2bed39 Mon Sep 17 00:00:00 2001
From: WuYunlong <xzsyeb@126.com>
Date: Tue, 9 Feb 2021 18:36:09 +0800
Subject: [PATCH 1043/1098] Cleanup in redis-cli and tests: release memory on
 exit, change dup test name (#8475)

1. Rename 18-cluster-nodes-slots.tcl to 19-cluster-nodes-slots.tcl.
  it was conflicting with another test prefixed by 18
2. Release memory on exit in redis-cli.c.
3. Fix freeConvertedSds indentation.
---
 src/redis-cli.c                               | 23 ++++++++++++++-----
 ...s-slots.tcl => 19-cluster-nodes-slots.tcl} |  0
 2 files changed, 17 insertions(+), 6 deletions(-)
 rename tests/cluster/tests/{18-cluster-nodes-slots.tcl => 19-cluster-nodes-slots.tcl} (100%)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index ab30edc75..ef1d49e3e 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -1929,13 +1929,20 @@ static int confirmWithYes(char *msg, int ignore_force) {
 
 /* Turn the plain C strings into Sds strings */
 static char **convertToSds(int count, char** args) {
-  int j;
-  char **sds = zmalloc(sizeof(char*)*count);
+    int j;
+    char **sds = zmalloc(sizeof(char*)*count);
 
-  for(j = 0; j < count; j++)
-    sds[j] = sdsnew(args[j]);
+    for(j = 0; j < count; j++)
+        sds[j] = sdsnew(args[j]);
 
-  return sds;
+    return sds;
+}
+
+static void freeConvertedSds(int count, char **sds) {
+    int j;
+    for (j = 0; j < count; j++)
+        sdsfree(sds[j]);
+    zfree(sds);
 }
 
 static int issueCommandRepeat(int argc, char **argv, long repeat) {
@@ -2168,13 +2175,17 @@ static void repl(void) {
 
 static int noninteractive(int argc, char **argv) {
     int retval = 0;
+
+    argv = convertToSds(argc, argv);
     if (config.stdinarg) {
         argv = zrealloc(argv, (argc+1)*sizeof(char*));
         argv[argc] = readArgFromStdin();
         retval = issueCommand(argc+1, argv);
+        sdsfree(argv[argc]);
     } else {
         retval = issueCommand(argc, argv);
     }
+    freeConvertedSds(argc, argv);
     return retval;
 }
 
@@ -8331,6 +8342,6 @@ int main(int argc, char **argv) {
     if (config.eval) {
         return evalMode(argc,argv);
     } else {
-        return noninteractive(argc,convertToSds(argc,argv));
+        return noninteractive(argc,argv);
     }
 }
diff --git a/tests/cluster/tests/18-cluster-nodes-slots.tcl b/tests/cluster/tests/19-cluster-nodes-slots.tcl
similarity index 100%
rename from tests/cluster/tests/18-cluster-nodes-slots.tcl
rename to tests/cluster/tests/19-cluster-nodes-slots.tcl

From 4554a49d32c5c0921f251262f2fbe02d7ac64b59 Mon Sep 17 00:00:00 2001
From: Itamar Haber <itamar@redislabs.com>
Date: Tue, 9 Feb 2021 14:38:09 +0200
Subject: [PATCH 1044/1098] Adds code of conduct (#8471)

---
 CONDUCT | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 96 insertions(+)
 create mode 100644 CONDUCT

diff --git a/CONDUCT b/CONDUCT
new file mode 100644
index 000000000..e0e15e268
--- /dev/null
+++ b/CONDUCT
@@ -0,0 +1,96 @@
+Contributor Covenant Code of Conduct
+Our Pledge
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+Our Standards
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others’ private information, such as a physical or email
+address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+professional setting
+
+Enforcement Responsibilities
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+Scope
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+Enforcement
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+this email address: redis@redis.io.
+All complaints will be reviewed and investigated promptly and fairly.
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+Enforcement Guidelines
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+1. Correction
+Community Impact: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+Consequence: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+2. Warning
+Community Impact: A violation through a single incident or series
+of actions.
+Consequence: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+3. Temporary Ban
+Community Impact: A serious violation of community standards, including
+sustained inappropriate behavior.
+Consequence: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+4. Permanent Ban
+Community Impact: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+Consequence: A permanent ban from any sort of public interaction within
+the community.
+Attribution
+This Code of Conduct is adapted from the Contributor Covenant,
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+Community Impact Guidelines were inspired by Mozilla’s code of conduct
+enforcement ladder.
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.
\ No newline at end of file

From 899c85ae67631f4f9f866a254ac5e149b86e5529 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Tue, 9 Feb 2021 11:52:28 -0800
Subject: [PATCH 1045/1098] Moved most static strings into the shared structure
 (#8411)

Moved most static strings into the shared structure
---
 src/acl.c         |  6 +-----
 src/aof.c         |  3 +--
 src/object.c      | 17 -----------------
 src/replication.c |  3 +--
 src/scripting.c   |  7 +++----
 src/server.c      | 40 +++++++++++++++++++++++++++++++---------
 src/server.h      |  7 +++++--
 src/t_hash.c      |  6 ++----
 src/t_set.c       |  9 +++------
 src/t_stream.c    | 37 ++++++++++++-------------------------
 src/t_string.c    |  6 ++----
 11 files changed, 61 insertions(+), 80 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 1f07c292f..74210ebce 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -2253,7 +2253,7 @@ void authCommand(client *c) {
             return;
         }
 
-        username = createStringObject("default",7);
+        username = shared.default_username; 
         password = c->argv[1];
     } else {
         username = c->argv[1];
@@ -2265,9 +2265,5 @@ void authCommand(client *c) {
     } else {
         addReplyError(c,"-WRONGPASS invalid username-password pair or user is disabled.");
     }
-
-    /* Free the "default" string object we created for the two
-     * arguments form. */
-    if (c->argc == 2) decrRefCount(username);
 }
 
diff --git a/src/aof.c b/src/aof.c
index 6753e8bcc..518b98c84 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -588,11 +588,10 @@ sds catAppendOnlyExpireAtCommand(sds buf, struct redisCommand *cmd, robj *key, r
     }
     decrRefCount(seconds);
 
-    argv[0] = createStringObject("PEXPIREAT",9);
+    argv[0] = shared.pexpireat;
     argv[1] = key;
     argv[2] = createStringObjectFromLongLong(when);
     buf = catAppendOnlyGenericCommand(buf, 3, argv);
-    decrRefCount(argv[0]);
     decrRefCount(argv[2]);
     return buf;
 }
diff --git a/src/object.c b/src/object.c
index 8573ef0b5..ce2f3dc14 100644
--- a/src/object.c
+++ b/src/object.c
@@ -384,23 +384,6 @@ void decrRefCountVoid(void *o) {
     decrRefCount(o);
 }
 
-/* This function set the ref count to zero without freeing the object.
- * It is useful in order to pass a new object to functions incrementing
- * the ref count of the received object. Example:
- *
- *    functionThatWillIncrementRefCount(resetRefCount(CreateObject(...)));
- *
- * Otherwise you need to resort to the less elegant pattern:
- *
- *    *obj = createObject(...);
- *    functionThatWillIncrementRefCount(obj);
- *    decrRefCount(obj);
- */
-robj *resetRefCount(robj *obj) {
-    obj->refcount = 0;
-    return obj;
-}
-
 int checkType(client *c, robj *o, int type) {
     /* A NULL is considered an empty key */
     if (o && o->type != type) {
diff --git a/src/replication.c b/src/replication.c
index f23fcb6de..90fa6c782 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -3333,10 +3333,9 @@ void replicationCron(void) {
             checkClientPauseTimeoutAndReturnIfPaused();
 
         if (!manual_failover_in_progress) {
-            ping_argv[0] = createStringObject("PING",4);
+            ping_argv[0] = shared.ping;
             replicationFeedSlaves(server.slaves, server.slaveseldb,
                 ping_argv, 1);
-            decrRefCount(ping_argv[0]);
         }
     }
 
diff --git a/src/scripting.c b/src/scripting.c
index 41469ee2e..e23d49e8b 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -1668,12 +1668,11 @@ void evalGenericCommand(client *c, int evalsha) {
              * or just running a CPU costly read-only script on the slaves. */
             if (server.dirty == initial_server_dirty) {
                 rewriteClientCommandVector(c,3,
-                    resetRefCount(createStringObject("SCRIPT",6)),
-                    resetRefCount(createStringObject("LOAD",4)),
+                    shared.script,
+                    shared.load,
                     script);
             } else {
-                rewriteClientCommandArgument(c,0,
-                    resetRefCount(createStringObject("EVAL",4)));
+                rewriteClientCommandArgument(c,0,shared.eval);
                 rewriteClientCommandArgument(c,1,script);
             }
             forceCommandPropagation(c,PROPAGATE_REPL|PROPAGATE_AOF);
diff --git a/src/server.c b/src/server.c
index faaca7215..bc0eb8255 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2418,13 +2418,10 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     if (server.get_ack_from_slaves && !checkClientPauseTimeoutAndReturnIfPaused()) {
         robj *argv[3];
 
-        argv[0] = createStringObject("REPLCONF",8);
-        argv[1] = createStringObject("GETACK",6);
-        argv[2] = createStringObject("*",1); /* Not used argument. */
+        argv[0] = shared.replconf;
+        argv[1] = shared.getack;
+        argv[2] = shared.special_asterick; /* Not used argument. */
         replicationFeedSlaves(server.slaves, server.slaveseldb, argv, 3);
-        decrRefCount(argv[0]);
-        decrRefCount(argv[1]);
-        decrRefCount(argv[2]);
         server.get_ack_from_slaves = 0;
     }
 
@@ -2565,6 +2562,8 @@ void createSharedObjects(void) {
     shared.unsubscribebulk = createStringObject("$11\r\nunsubscribe\r\n",18);
     shared.psubscribebulk = createStringObject("$10\r\npsubscribe\r\n",17);
     shared.punsubscribebulk = createStringObject("$12\r\npunsubscribe\r\n",19);
+
+    /* Shared command names */
     shared.del = createStringObject("DEL",3);
     shared.unlink = createStringObject("UNLINK",6);
     shared.rpop = createStringObject("RPOP",4);
@@ -2577,15 +2576,38 @@ void createSharedObjects(void) {
     shared.zpopmax = createStringObject("ZPOPMAX",7);
     shared.multi = createStringObject("MULTI",5);
     shared.exec = createStringObject("EXEC",4);
-    /* Used in the LMOVE/BLMOVE commands */
-    shared.left = createStringObject("left",4);
-    shared.right = createStringObject("right",5);
+    shared.hset = createStringObject("HSET",4);
+    shared.srem = createStringObject("SREM",4);
+    shared.xgroup = createStringObject("XGROUP",6);
+    shared.xclaim = createStringObject("XCLAIM",6);
+    shared.script = createStringObject("SCRIPT",6);
+    shared.replconf = createStringObject("REPLCONF",8);
     shared.pexpireat = createStringObject("PEXPIREAT",9);
     shared.pexpire = createStringObject("PEXPIRE",7);
     shared.persist = createStringObject("PERSIST",7);
     shared.set = createStringObject("SET",3);
+    shared.eval = createStringObject("EVAL",4);
+
+    /* Shared command argument */
+    shared.left = createStringObject("left",4);
+    shared.right = createStringObject("right",5);
     shared.pxat = createStringObject("PXAT", 4);
     shared.px = createStringObject("PX",2);
+    shared.time = createStringObject("TIME",4);
+    shared.retrycount = createStringObject("RETRYCOUNT",10);
+    shared.force = createStringObject("FORCE",5);
+    shared.justid = createStringObject("JUSTID",6);
+    shared.lastid = createStringObject("LASTID",6);
+    shared.default_username = createStringObject("default",7);
+    shared.ping = createStringObject("ping",7);
+    shared.setid = createStringObject("SETID",5);
+    shared.keepttl = createStringObject("KEEPTTL",7);
+    shared.load = createStringObject("LOAD",4);
+    shared.createconsumer = createStringObject("CREATECONSUMER",14);
+    shared.getack = createStringObject("GETACK",6);
+    shared.special_asterick = createStringObject("*",1);
+    shared.special_equals = createStringObject("=",1);
+
     for (j = 0; j < OBJ_SHARED_INTEGERS; j++) {
         shared.integers[j] =
             makeObjectShared(createObject(OBJ_STRING,(void*)(long)j));
diff --git a/src/server.h b/src/server.h
index b293afcee..7140c4571 100644
--- a/src/server.h
+++ b/src/server.h
@@ -976,8 +976,11 @@ struct sharedObjectsStruct {
     *busykeyerr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk,
     *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *unlink,
     *rpop, *lpop, *lpush, *rpoplpush, *lmove, *blmove, *zpopmin, *zpopmax,
-    *emptyscan, *multi, *exec, *left, *right, *persist, *set, *pexpireat,
-    *pexpire, *pxat, *px,
+    *emptyscan, *multi, *exec, *left, *right, *hset, *srem, *xgroup, *xclaim,  
+    *script, *replconf, *eval, *persist, *set, *pexpireat, *pexpire, 
+    *time, *pxat, *px, *retrycount, *force, *justid, 
+    *lastid, *ping, *setid, *keepttl, *load, *createconsumer,
+    *getack, *special_asterick, *special_equals, *default_username,
     *select[PROTO_SHARED_SELECT_CMDS],
     *integers[OBJ_SHARED_INTEGERS],
     *mbulkhdr[OBJ_SHARED_BULKHDR_LEN], /* "*<value>\r\n" */
diff --git a/src/t_hash.c b/src/t_hash.c
index 082e0f129..cae350566 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -759,11 +759,9 @@ void hincrbyfloatCommand(client *c) {
     /* Always replicate HINCRBYFLOAT as an HSET command with the final value
      * in order to make sure that differences in float precision or formatting
      * will not create differences in replicas or after an AOF restart. */
-    robj *aux, *newobj;
-    aux = createStringObject("HSET",4);
+    robj *newobj;
     newobj = createRawStringObject(buf,len);
-    rewriteClientCommandArgument(c,0,aux);
-    decrRefCount(aux);
+    rewriteClientCommandArgument(c,0,shared.hset);
     rewriteClientCommandArgument(c,3,newobj);
     decrRefCount(newobj);
 }
diff --git a/src/t_set.c b/src/t_set.c
index de0a9f954..f7a05206d 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -499,7 +499,7 @@ void spopWithCountCommand(client *c) {
      * Prepare our replication argument vector. Also send the array length
      * which is common to both the code paths. */
     robj *propargv[3];
-    propargv[0] = createStringObject("SREM",4);
+    propargv[0] = shared.srem;
     propargv[1] = c->argv[1];
     addReplySetLen(c,count);
 
@@ -590,13 +590,12 @@ void spopWithCountCommand(client *c) {
      * dirty counter. We don't want to propagate an SPOP command since
      * we propagated the command as a set of SREMs operations using
      * the alsoPropagate() API. */
-    decrRefCount(propargv[0]);
     preventCommandPropagation(c);
     signalModifiedKey(c,c->db,c->argv[1]);
 }
 
 void spopCommand(client *c) {
-    robj *set, *ele, *aux;
+    robj *set, *ele;
     sds sdsele;
     int64_t llele;
     int encoding;
@@ -629,9 +628,7 @@ void spopCommand(client *c) {
     notifyKeyspaceEvent(NOTIFY_SET,"spop",c->argv[1],c->db->id);
 
     /* Replicate/AOF this command as an SREM operation */
-    aux = createStringObject("SREM",4);
-    rewriteClientCommandVector(c,3,aux,c->argv[1],ele);
-    decrRefCount(aux);
+    rewriteClientCommandVector(c,3,shared.srem,c->argv[1],ele);
 
     /* Add the element to the reply */
     addReplyBulk(c,ele);
diff --git a/src/t_stream.c b/src/t_stream.c
index 32bfa1f7d..7b4ffe3c4 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -1328,19 +1328,19 @@ void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupnam
      * Note that JUSTID is useful in order to avoid that XCLAIM will do
      * useless work in the slave side, trying to fetch the stream item. */
     robj *argv[14];
-    argv[0] = createStringObject("XCLAIM",6);
+    argv[0] = shared.xclaim;
     argv[1] = key;
     argv[2] = groupname;
     argv[3] = createStringObject(nack->consumer->name,sdslen(nack->consumer->name));
-    argv[4] = createStringObjectFromLongLong(0);
+    argv[4] = shared.integers[0];
     argv[5] = id;
-    argv[6] = createStringObject("TIME",4);
+    argv[6] = shared.time;
     argv[7] = createStringObjectFromLongLong(nack->delivery_time);
-    argv[8] = createStringObject("RETRYCOUNT",10);
+    argv[8] = shared.retrycount;
     argv[9] = createStringObjectFromLongLong(nack->delivery_count);
-    argv[10] = createStringObject("FORCE",5);
-    argv[11] = createStringObject("JUSTID",6);
-    argv[12] = createStringObject("LASTID",6);
+    argv[10] = shared.force;
+    argv[11] = shared.justid;
+    argv[12] = shared.lastid;
     argv[13] = createObjectFromStreamID(&group->last_id);
 
     /* We use progagate() because this code path is not always called from
@@ -1348,16 +1348,9 @@ void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupnam
      * consumer group state, and we don't need MULTI/EXEC wrapping because
      * there is no message state cross-message atomicity required. */
     propagate(server.xclaimCommand,c->db->id,argv,14,PROPAGATE_AOF|PROPAGATE_REPL);
-    decrRefCount(argv[0]);
     decrRefCount(argv[3]);
-    decrRefCount(argv[4]);
-    decrRefCount(argv[6]);
     decrRefCount(argv[7]);
-    decrRefCount(argv[8]);
     decrRefCount(argv[9]);
-    decrRefCount(argv[10]);
-    decrRefCount(argv[11]);
-    decrRefCount(argv[12]);
     decrRefCount(argv[13]);
 }
 
@@ -1369,8 +1362,8 @@ void streamPropagateXCLAIM(client *c, robj *key, streamCG *group, robj *groupnam
  */
 void streamPropagateGroupID(client *c, robj *key, streamCG *group, robj *groupname) {
     robj *argv[5];
-    argv[0] = createStringObject("XGROUP",6);
-    argv[1] = createStringObject("SETID",5);
+    argv[0] = shared.xgroup;
+    argv[1] = shared.setid;
     argv[2] = key;
     argv[3] = groupname;
     argv[4] = createObjectFromStreamID(&group->last_id);
@@ -1380,8 +1373,6 @@ void streamPropagateGroupID(client *c, robj *key, streamCG *group, robj *groupna
      * consumer group state, and we don't need MULTI/EXEC wrapping because
      * there is no message state cross-message atomicity required. */
     propagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL);
-    decrRefCount(argv[0]);
-    decrRefCount(argv[1]);
     decrRefCount(argv[4]);
 }
 
@@ -1393,8 +1384,8 @@ void streamPropagateGroupID(client *c, robj *key, streamCG *group, robj *groupna
  */
 void streamPropagateConsumerCreation(client *c, robj *key, robj *groupname, sds consumername) {
     robj *argv[5];
-    argv[0] = createStringObject("XGROUP",6);
-    argv[1] = createStringObject("CREATECONSUMER",14);
+    argv[0] = shared.xgroup;
+    argv[1] = shared.createconsumer;
     argv[2] = key;
     argv[3] = groupname;
     argv[4] = createObject(OBJ_STRING,sdsdup(consumername));
@@ -1404,8 +1395,6 @@ void streamPropagateConsumerCreation(client *c, robj *key, robj *groupname, sds
      * consumer group state, and we don't need MULTI/EXEC wrapping because
      * there is no message state cross-message atomicity required. */
     propagate(server.xgroupCommand,c->db->id,argv,5,PROPAGATE_AOF|PROPAGATE_REPL);
-    decrRefCount(argv[0]);
-    decrRefCount(argv[1]);
     decrRefCount(argv[4]);
 }
 
@@ -1725,9 +1714,7 @@ int streamParseIntervalIDOrReply(client *c, robj *o, streamID *id, int *exclude,
 }
 
 void streamRewriteApproxSpecifier(client *c, int idx) {
-    robj *equal_obj = createStringObject("=",1);
-    rewriteClientCommandArgument(c,idx,equal_obj);
-    decrRefCount(equal_obj);
+    rewriteClientCommandArgument(c,idx,shared.special_equals);
 }
 
 /* We propagate MAXLEN/MINID ~ <count> as MAXLEN/MINID = <resulting-len-of-stream>
diff --git a/src/t_string.c b/src/t_string.c
index d05366b7c..6a3b256b8 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -631,7 +631,7 @@ void decrbyCommand(client *c) {
 
 void incrbyfloatCommand(client *c) {
     long double incr, value;
-    robj *o, *new, *aux;
+    robj *o, *new;
 
     o = lookupKeyWrite(c->db,c->argv[1]);
     if (checkType(c,o,OBJ_STRING)) return;
@@ -659,9 +659,7 @@ void incrbyfloatCommand(client *c) {
      * will not create differences in replicas or after an AOF restart. */
     rewriteClientCommandArgument(c,0,shared.set);
     rewriteClientCommandArgument(c,2,new);
-    aux = createStringObject("KEEPTTL",7);
-    rewriteClientCommandArgument(c,3,aux);
-    decrRefCount(aux);
+    rewriteClientCommandArgument(c,3,shared.keepttl);
 }
 
 void appendCommand(client *c) {

From b5ca1e9e53de93bb84e5b03de39cbaf8093fcd41 Mon Sep 17 00:00:00 2001
From: filipe oliveira <filipecosta.90@gmail.com>
Date: Wed, 10 Feb 2021 06:59:07 +0000
Subject: [PATCH 1046/1098] Removed time sensitive checks from block on
 background tests. Fixed uninitialized variable (#8479)

- removes time sensitive checks from block on background tests during leak checks.
- fix uninitialized variable on RedisModuleBlockedClient() when calling
  RM_BlockedClientMeasureTimeEnd() without RM_BlockedClientMeasureTimeStart()
---
 .github/workflows/daily.yml                |  2 +-
 src/module.c                               |  1 +
 tests/modules/blockonbackground.c          | 11 +++---
 tests/unit/moduleapi/blockonbackground.tcl | 45 ++++++++++++++++------
 4 files changed, 40 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 0a899d174..9d7eb65c9 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -130,7 +130,7 @@ jobs:
         sudo apt-get install tcl8.6 valgrind -y
         ./runtest --valgrind --verbose --clients 1 --dump-logs
     - name: module api test
-      run: ./runtest-moduleapi --valgrind --verbose --clients 1
+      run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1
 
   test-centos7-jemalloc:
     runs-on: ubuntu-latest
diff --git a/src/module.c b/src/module.c
index 039465e1a..e7f0cde26 100644
--- a/src/module.c
+++ b/src/module.c
@@ -5122,6 +5122,7 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF
     bc->dbid = c->db->id;
     bc->blocked_on_keys = keys != NULL;
     bc->unblocked = 0;
+    bc->background_timer = 0;
     bc->background_duration = 0;
     c->bpop.timeout = timeout;
 
diff --git a/tests/modules/blockonbackground.c b/tests/modules/blockonbackground.c
index fcabfa4d2..855fef9dc 100644
--- a/tests/modules/blockonbackground.c
+++ b/tests/modules/blockonbackground.c
@@ -5,7 +5,6 @@
 #include <stdlib.h>
 #include <pthread.h>
 #include <time.h>
-#include "assert.h"
 
 #define UNUSED(x) (void)(x)
 
@@ -22,7 +21,7 @@ int HelloBlock_Timeout(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     UNUSED(argv);
     UNUSED(argc);
     RedisModuleBlockedClient *bc = RedisModule_GetBlockedClientHandle(ctx);
-    assert(RedisModule_BlockedClientMeasureTimeEnd(bc)==REDISMODULE_OK);
+    RedisModule_BlockedClientMeasureTimeEnd(bc);
     return RedisModule_ReplyWithSimpleString(ctx,"Request timedout");
 }
 
@@ -40,7 +39,7 @@ void *BlockDebug_ThreadMain(void *arg) {
     long long delay = (unsigned long)targ[1];
     long long enable_time_track = (unsigned long)targ[2];
     if (enable_time_track)
-        assert(RedisModule_BlockedClientMeasureTimeStart(bc)==REDISMODULE_OK);
+        RedisModule_BlockedClientMeasureTimeStart(bc);
     RedisModule_Free(targ);
 
     struct timespec ts;
@@ -50,7 +49,7 @@ void *BlockDebug_ThreadMain(void *arg) {
     int *r = RedisModule_Alloc(sizeof(int));
     *r = rand();
     if (enable_time_track)
-        assert(RedisModule_BlockedClientMeasureTimeEnd(bc)==REDISMODULE_OK);
+        RedisModule_BlockedClientMeasureTimeEnd(bc);
     RedisModule_UnblockClient(bc,r);
     return NULL;
 }
@@ -61,7 +60,7 @@ void *DoubleBlock_ThreadMain(void *arg) {
     void **targ = arg;
     RedisModuleBlockedClient *bc = targ[0];
     long long delay = (unsigned long)targ[1];
-    assert(RedisModule_BlockedClientMeasureTimeStart(bc)==REDISMODULE_OK);
+    RedisModule_BlockedClientMeasureTimeStart(bc);
     RedisModule_Free(targ);
     struct timespec ts;
     ts.tv_sec = delay / 1000;
@@ -73,7 +72,7 @@ void *DoubleBlock_ThreadMain(void *arg) {
     /* call again RedisModule_BlockedClientMeasureTimeStart() and
      * RedisModule_BlockedClientMeasureTimeEnd and ensure that the
      * total execution time is 2x the delay. */
-    assert(RedisModule_BlockedClientMeasureTimeStart(bc)==REDISMODULE_OK);
+    RedisModule_BlockedClientMeasureTimeStart(bc);
     nanosleep(&ts, NULL);
     RedisModule_BlockedClientMeasureTimeEnd(bc);
 
diff --git a/tests/unit/moduleapi/blockonbackground.tcl b/tests/unit/moduleapi/blockonbackground.tcl
index 46a72bfd2..66a232fab 100644
--- a/tests/unit/moduleapi/blockonbackground.tcl
+++ b/tests/unit/moduleapi/blockonbackground.tcl
@@ -8,12 +8,18 @@ start_server {tags {"modules"}} {
     test { blocked clients time tracking - check blocked command that uses RedisModule_BlockedClientMeasureTimeStart() is tracking background time} {
         r slowlog reset
         r config set slowlog-log-slower-than 200000
-        assert_equal [r slowlog len] 0
+        if {!$::no_latency} {
+            assert_equal [r slowlog len] 0
+        }
         r block.debug 0 10000
-        assert_equal [r slowlog len] 0
+        if {!$::no_latency} {
+            assert_equal [r slowlog len] 0
+        }
         r config resetstat
         r block.debug 200 10000
-        assert_equal [r slowlog len] 1
+        if {!$::no_latency} {
+            assert_equal [r slowlog len] 1
+        }
 
         set cmdstatline [cmdrstat block.debug r]
 
@@ -25,12 +31,18 @@ start_server {tags {"modules"}} {
     test { blocked clients time tracking - check blocked command that uses RedisModule_BlockedClientMeasureTimeStart() is tracking background time even in timeout } {
         r slowlog reset
         r config set slowlog-log-slower-than 200000
-        assert_equal [r slowlog len] 0
+        if {!$::no_latency} {
+            assert_equal [r slowlog len] 0
+        }
         r block.debug 0 20000
-        assert_equal [r slowlog len] 0
+        if {!$::no_latency} {
+            assert_equal [r slowlog len] 0
+        }
         r config resetstat
         r block.debug 20000 500
-        assert_equal [r slowlog len] 1
+        if {!$::no_latency} {
+            assert_equal [r slowlog len] 1
+        }
 
         set cmdstatline [cmdrstat block.debug r]
 
@@ -42,13 +54,18 @@ start_server {tags {"modules"}} {
     test { blocked clients time tracking - check blocked command with multiple calls RedisModule_BlockedClientMeasureTimeStart()  is tracking the total background time } {
         r slowlog reset
         r config set slowlog-log-slower-than 200000
-        assert_equal [r slowlog len] 0
+        if {!$::no_latency} {
+            assert_equal [r slowlog len] 0
+        }
         r block.double_debug 0
-        assert_equal [r slowlog len] 0
+        if {!$::no_latency} {
+            assert_equal [r slowlog len] 0
+        }
         r config resetstat
         r block.double_debug 100
-        assert_equal [r slowlog len] 1
-
+        if {!$::no_latency} {
+            assert_equal [r slowlog len] 1
+        }
         set cmdstatline [cmdrstat block.double_debug r]
 
         regexp "calls=1,usec=(.*?),usec_per_call=(.*?),rejected_calls=0,failed_calls=0" $cmdstatline usec usec_per_call
@@ -59,9 +76,13 @@ start_server {tags {"modules"}} {
     test { blocked clients time tracking - check blocked command without calling RedisModule_BlockedClientMeasureTimeStart() is not reporting background time } {
         r slowlog reset
         r config set slowlog-log-slower-than 200000
-        assert_equal [r slowlog len] 0
+        if {!$::no_latency} {
+            assert_equal [r slowlog len] 0
+        }
         r block.debug_no_track 200 1000
         # ensure slowlog is still empty
-        assert_equal [r slowlog len] 0
+        if {!$::no_latency} {
+            assert_equal [r slowlog len] 0
+        }
     }
 }

From 29ac9aea5de2f395960834b262b3d94f9efedbd8 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 10 Feb 2021 18:38:10 +0200
Subject: [PATCH 1047/1098] Fix Sentinel configuration rewrite. (#8480)

The `resolve-hostnames` and `announce-hostnames` parameters were not
specified correctly according to the new convention introduced by
1aad55b66.
---
 src/sentinel.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/sentinel.c b/src/sentinel.c
index a87766ebe..fb68fd0ea 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -2077,13 +2077,13 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) {
      */
     line = sdscatprintf(sdsempty(), "sentinel resolve-hostnames %s",
                         sentinel.resolve_hostnames ? "yes" : "no");
-    rewriteConfigRewriteLine(state,"sentinel",line,
+    rewriteConfigRewriteLine(state,"sentinel resolve-hostnames",line,
                              sentinel.resolve_hostnames != SENTINEL_DEFAULT_RESOLVE_HOSTNAMES);
 
     /* sentinel announce-hostnames. */
     line = sdscatprintf(sdsempty(), "sentinel announce-hostnames %s",
                         sentinel.announce_hostnames ? "yes" : "no");
-    rewriteConfigRewriteLine(state,"sentinel",line,
+    rewriteConfigRewriteLine(state,"sentinel announce-hostnames",line,
                              sentinel.announce_hostnames != SENTINEL_DEFAULT_ANNOUNCE_HOSTNAMES);
 
     /* For every master emit a "sentinel monitor" config entry. */

From 94bc26e65277a327cefa464e812c7efc5c78d0a2 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 11 Feb 2021 11:50:47 +0200
Subject: [PATCH 1048/1098] Fix duplicate replicas issue. (#8481)

We need to store replicas referenced by their announced address (IP or
address). Before that, if hostnames were used and the IP address
changed, duplicate entries would have been created.
---
 src/sentinel.c | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/src/sentinel.c b/src/sentinel.c
index fb68fd0ea..8597a10df 100644
--- a/src/sentinel.c
+++ b/src/sentinel.c
@@ -649,6 +649,17 @@ const char *announceSentinelAddr(const sentinelAddr *a) {
     return sentinel.announce_hostnames ? a->hostname : a->ip;
 }
 
+/* Return an allocated sds with hostname/address:port. IPv6
+ * addresses are bracketed the same way anetFormatAddr() does.
+ */
+sds announceSentinelAddrAndPort(const sentinelAddr *a) {
+    const char *addr = announceSentinelAddr(a);
+    if (strchr(addr, ':') != NULL)
+        return sdscatprintf(sdsempty(), "[%s]:%d", addr, a->port);
+    else
+        return sdscatprintf(sdsempty(), "%s:%d", addr, a->port);
+}
+
 /* =========================== Events notification ========================== */
 
 /* Send an event to log, pub/sub, user notification script.
@@ -1273,7 +1284,7 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char *
     sentinelRedisInstance *ri;
     sentinelAddr *addr;
     dict *table = NULL;
-    char slavename[NET_ADDR_STR_LEN], *sdsname;
+    sds sdsname;
 
     serverAssert(flags & (SRI_MASTER|SRI_SLAVE|SRI_SENTINEL));
     serverAssert((flags & SRI_MASTER) || master != NULL);
@@ -1282,11 +1293,11 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char *
     addr = createSentinelAddr(hostname,port);
     if (addr == NULL) return NULL;
 
-    /* For slaves use ip:port as name. */
-    if (flags & SRI_SLAVE) {
-        anetFormatAddr(slavename, sizeof(slavename), addr->ip, port);
-        name = slavename;
-    }
+    /* For slaves use ip/host:port as name. */
+    if (flags & SRI_SLAVE)
+        sdsname = announceSentinelAddrAndPort(addr);
+    else
+        sdsname = sdsnew(name);
 
     /* Make sure the entry is not duplicated. This may happen when the same
      * name for a master is used multiple times inside the configuration or
@@ -1295,7 +1306,6 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char *
     if (flags & SRI_MASTER) table = sentinel.masters;
     else if (flags & SRI_SLAVE) table = master->slaves;
     else if (flags & SRI_SENTINEL) table = master->sentinels;
-    sdsname = sdsnew(name);
     if (dictFind(table,sdsname)) {
         releaseSentinelAddr(addr);
         sdsfree(sdsname);
@@ -1399,7 +1409,6 @@ sentinelRedisInstance *sentinelRedisInstanceLookupSlave(
 {
     sds key;
     sentinelRedisInstance *slave;
-    char buf[NET_ADDR_STR_LEN];
     sentinelAddr *addr;
 
     serverAssert(ri->flags & SRI_MASTER);
@@ -1410,11 +1419,9 @@ sentinelRedisInstance *sentinelRedisInstanceLookupSlave(
      */
     addr = createSentinelAddr(slave_addr, port);
     if (!addr) return NULL;
-
-    anetFormatAddr(buf,sizeof(buf),addr->ip,addr->port);
+    key = announceSentinelAddrAndPort(addr);
     releaseSentinelAddr(addr);
 
-    key = sdsnew(buf);
     slave = dictFetchValue(ri->slaves,key);
     sdsfree(key);
     return slave;

From 8c42d1257f8b6d7a1a8f30e37eef9b55785b2426 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 11 Feb 2021 15:25:01 +0200
Subject: [PATCH 1049/1098] Fix errors with sentinel leaked fds test. (#8482)

* Don't run test script on non-Linux.
* Verify that reported fds do indeed exist also in parent, to avoid
  false negatives on some systems (namely CentOS).

Co-authored-by: Andy Pan <panjf2000@gmail.com>
---
 .../tests/helpers/check_leaked_fds.tcl        | 25 +++++++++++++------
 tests/sentinel/tests/includes/init-tests.tcl  |  2 +-
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/tests/sentinel/tests/helpers/check_leaked_fds.tcl b/tests/sentinel/tests/helpers/check_leaked_fds.tcl
index cb7b5d3d9..482b3e0d5 100755
--- a/tests/sentinel/tests/helpers/check_leaked_fds.tcl
+++ b/tests/sentinel/tests/helpers/check_leaked_fds.tcl
@@ -21,6 +21,16 @@ proc get_parent_pid {_pid} {
     error "failed to get parent pid"
 }
 
+# Read symlink to get info about the specified fd of the specified process.
+# The result can be the file name or an arbitrary string that identifies it.
+# When not able to read, an empty string is returned.
+proc get_fdlink {_pid fd} {
+    if { [catch {set fdlink [file readlink "/proc/$_pid/fd/$fd"]} err] } {
+        return ""
+    }
+    return $fdlink
+}
+
 # Linux only
 set os [exec uname]
 if {$os != "Linux"} {
@@ -46,17 +56,16 @@ foreach fd [glob -tails -directory "/proc/self/fd" *] {
         continue
     }
 
-    # Read symlink to get info about the file descriptor. This can be a real
-    # file name or an arbitrary string that identifies the fd.
-    if { [catch {set fdlink [file readlink "/proc/self/fd/$fd"]} err] } {
+    set fdlink [get_fdlink "self" $fd]
+    if {$fdlink == ""} {
         continue
     }
 
-    # See if grandparent process has the same fd and info and skip if it does.
-    if { ! [catch {set gp_fdlink [file readlink "/proc/$grandparent_pid/fd/$fd"]} err] } {
-        if {$gp_fdlink == $fdlink} {
-            continue
-        }
+    # We ignore fds that existed in the grandparent, or fds that don't exist
+    # in our parent (Sentinel process).
+    if {[get_fdlink $grandparent_pid $fd] == $fdlink ||
+	[get_fdlink $parent_pid $fd] != $fdlink} {
+        continue
     }
 
     lappend leaked_fds [list $fd $fdlink]
diff --git a/tests/sentinel/tests/includes/init-tests.tcl b/tests/sentinel/tests/includes/init-tests.tcl
index be6a8f502..b5baa256f 100644
--- a/tests/sentinel/tests/includes/init-tests.tcl
+++ b/tests/sentinel/tests/includes/init-tests.tcl
@@ -41,7 +41,7 @@ test "(init) Sentinels can start monitoring a master" {
         S $id SENTINEL SET mymaster down-after-milliseconds 2000
         S $id SENTINEL SET mymaster failover-timeout 20000
         S $id SENTINEL SET mymaster parallel-syncs 10
-        if {$::leaked_fds_file != ""} {
+        if {$::leaked_fds_file != "" && [exec uname] == "Linux"} {
             S $id SENTINEL SET mymaster notification-script ../../tests/helpers/check_leaked_fds.tcl
             S $id SENTINEL SET mymaster client-reconfig-script ../../tests/helpers/check_leaked_fds.tcl
         }

From efccd6353bacec9addc7278292fd8c3108d116e4 Mon Sep 17 00:00:00 2001
From: filipe oliveira <filipecosta.90@gmail.com>
Date: Sun, 14 Feb 2021 12:42:41 +0000
Subject: [PATCH 1050/1098] redis-benchmark: Fix broken protocol when used with
 -a or --dbnum (#8486)

Fix the pointers to the slot hash tags in the case of prefixed commands usage
i.e. AUTH / SELECT

It adjusts the pointers to the slot hash tags in the case of prefixed commands
usage as soon as we get the 1st reply (same like we already did for the random
strings within the command )
---
 src/redis-benchmark.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c
index 164f5e3ee..186739766 100644
--- a/src/redis-benchmark.c
+++ b/src/redis-benchmark.c
@@ -567,6 +567,9 @@ static void readHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
                         * we need to randomize. */
                         for (j = 0; j < c->randlen; j++)
                             c->randptr[j] -= c->prefixlen;
+                        /* Fix the pointers to the slot hash tags */
+                        for (j = 0; j < c->staglen; j++)
+                            c->stagptr[j] -= c->prefixlen;
                         c->prefixlen = 0;
                     }
                     continue;

From 0bc8c9c8f975f326c2ce87a0759b8b65b94a4120 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= <viktor.soderqvist@est.tech>
Date: Mon, 15 Feb 2021 10:40:05 +0100
Subject: [PATCH 1051/1098] Modules: In RM_HashSet, add COUNT_ALL flag and set
 errno (#8446)

The added flag affects the return value of RM_HashSet() to include
the number of inserted fields, in addition to updated and deleted
fields.

errno is set on errors, tests are added and documentation updated.
---
 runtest-moduleapi             |  1 +
 src/module.c                  | 50 ++++++++++++++-----
 src/redismodule.h             |  1 +
 tests/modules/Makefile        |  1 +
 tests/modules/hash.c          | 90 +++++++++++++++++++++++++++++++++++
 tests/unit/moduleapi/hash.tcl | 23 +++++++++
 6 files changed, 155 insertions(+), 11 deletions(-)
 create mode 100644 tests/modules/hash.c
 create mode 100644 tests/unit/moduleapi/hash.tcl

diff --git a/runtest-moduleapi b/runtest-moduleapi
index 53656fad7..7c17501e0 100755
--- a/runtest-moduleapi
+++ b/runtest-moduleapi
@@ -32,6 +32,7 @@ $TCLSH tests/test_helper.tcl \
 --single unit/moduleapi/getkeys \
 --single unit/moduleapi/test_lazyfree \
 --single unit/moduleapi/defrag \
+--single unit/moduleapi/hash \
 --single unit/moduleapi/zset \
 --single unit/moduleapi/stream \
 "${@}"
diff --git a/src/module.c b/src/module.c
index e7f0cde26..f8d3e3170 100644
--- a/src/module.c
+++ b/src/module.c
@@ -2976,6 +2976,10 @@ int RM_ZsetRangePrev(RedisModuleKey *key) {
  *                          are created.
  *     REDISMODULE_HASH_CFIELDS: The field names passed are null terminated C
  *                               strings instead of RedisModuleString objects.
+ *     REDISMODULE_HASH_COUNT_ALL: Include the number of inserted fields in the
+ *                                 returned number, in addition to the number of
+ *                                 updated and deleted fields. (Added in Redis
+ *                                 6.2.)
  *
  * Unless NX is specified, the command overwrites the old field value with
  * the new one.
@@ -2989,21 +2993,43 @@ int RM_ZsetRangePrev(RedisModuleKey *key) {
  *
  * Return value:
  *
- * The number of fields updated (that may be less than the number of fields
- * specified because of the XX or NX options).
+ * The number of fields existing in the hash prior to the call, which have been
+ * updated (its old value has been replaced by a new value) or deleted. If the
+ * flag REDISMODULE_HASH_COUNT_ALL is set, insterted fields not previously
+ * existing in the hash are also counted.
  *
- * In the following case the return value is always zero:
+ * If the return value is zero, `errno` is set (since Redis 6.2) as follows:
  *
- * * The key was not open for writing.
- * * The key was associated with a non Hash value.
+ * - EINVAL if any unknown flags are set or if key is NULL.
+ * - ENOTSUP if the key is associated with a non Hash value.
+ * - EBADF if the key was not opened for writing.
+ * - ENOENT if no fields were counted as described under Return value above.
+ *   This is not actually an error. The return value can be zero if all fields
+ *   were just created and the COUNT_ALL flag was unset, or if changes were held
+ *   back due to the NX and XX flags.
+ *
+ * NOTICE: The return value semantics of this function are very different
+ * between Redis 6.2 and older versions. Modules that use it should determine
+ * the Redis version and handle it accordingly.
  */
 int RM_HashSet(RedisModuleKey *key, int flags, ...) {
     va_list ap;
-    if (!(key->mode & REDISMODULE_WRITE)) return 0;
-    if (key->value && key->value->type != OBJ_HASH) return 0;
+    if (!key || (flags & ~(REDISMODULE_HASH_NX |
+                           REDISMODULE_HASH_XX |
+                           REDISMODULE_HASH_CFIELDS |
+                           REDISMODULE_HASH_COUNT_ALL))) {
+        errno = EINVAL;
+        return 0;
+    } else if (key->value && key->value->type != OBJ_HASH) {
+        errno = ENOTSUP;
+        return 0;
+    } else if (!(key->mode & REDISMODULE_WRITE)) {
+        errno = EBADF;
+        return 0;
+    }
     if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_HASH);
 
-    int updated = 0;
+    int count = 0;
     va_start(ap, flags);
     while(1) {
         RedisModuleString *field, *value;
@@ -3031,7 +3057,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) {
 
         /* Handle deletion if value is REDISMODULE_HASH_DELETE. */
         if (value == REDISMODULE_HASH_DELETE) {
-            updated += hashTypeDelete(key->value, field->ptr);
+            count += hashTypeDelete(key->value, field->ptr);
             if (flags & REDISMODULE_HASH_CFIELDS) decrRefCount(field);
             continue;
         }
@@ -3045,7 +3071,8 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) {
 
         robj *argv[2] = {field,value};
         hashTypeTryConversion(key->value,argv,0,1);
-        updated += hashTypeSet(key->value, field->ptr, value->ptr, low_flags);
+        int updated = hashTypeSet(key->value, field->ptr, value->ptr, low_flags);
+        count += (flags & REDISMODULE_HASH_COUNT_ALL) ? 1 : updated;
 
         /* If CFIELDS is active, SDS string ownership is now of hashTypeSet(),
          * however we still have to release the 'field' object shell. */
@@ -3056,7 +3083,8 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) {
     }
     va_end(ap);
     moduleDelKeyIfEmpty(key);
-    return updated;
+    if (count == 0) errno = ENOENT;
+    return count;
 }
 
 /* Get fields from an hash value. This function is called using a variable
diff --git a/src/redismodule.h b/src/redismodule.h
index 9d8c6c5ea..60a152452 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -68,6 +68,7 @@
 #define REDISMODULE_HASH_XX         (1<<1)
 #define REDISMODULE_HASH_CFIELDS    (1<<2)
 #define REDISMODULE_HASH_EXISTS     (1<<3)
+#define REDISMODULE_HASH_COUNT_ALL  (1<<4)
 
 /* StreamID type. */
 typedef struct RedisModuleStreamID {
diff --git a/tests/modules/Makefile b/tests/modules/Makefile
index 8ea1d91a2..f56313964 100644
--- a/tests/modules/Makefile
+++ b/tests/modules/Makefile
@@ -35,6 +35,7 @@ TEST_MODULES = \
     test_lazyfree.so \
     timer.so \
     defragtest.so \
+    hash.so \
     zset.so \
     stream.so
 
diff --git a/tests/modules/hash.c b/tests/modules/hash.c
new file mode 100644
index 000000000..05ab03800
--- /dev/null
+++ b/tests/modules/hash.c
@@ -0,0 +1,90 @@
+#include "redismodule.h"
+#include <strings.h>
+#include <errno.h>
+#include <stdlib.h>
+
+/* If a string is ":deleted:", the special value for deleted hash fields is
+ * returned; otherwise the input string is returned. */
+static RedisModuleString *value_or_delete(RedisModuleString *s) {
+    if (!strcasecmp(RedisModule_StringPtrLen(s, NULL), ":delete:"))
+        return REDISMODULE_HASH_DELETE;
+    else
+        return s;
+}
+
+/* HASH.SET key flags field1 value1 [field2 value2 ..]
+ *
+ * Sets 1-4 fields. Returns the same as RedisModule_HashSet().
+ * Flags is a string of "nxa" where n = NX, x = XX, a = COUNT_ALL.
+ * To delete a field, use the value ":delete:".
+ */
+int hash_set(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    if (argc < 5 || argc % 2 == 0 || argc > 11)
+        return RedisModule_WrongArity(ctx);
+
+    RedisModule_AutoMemory(ctx);
+    RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE);
+
+    size_t flags_len;
+    const char *flags_str = RedisModule_StringPtrLen(argv[2], &flags_len);
+    int flags = REDISMODULE_HASH_NONE;
+    for (size_t i = 0; i < flags_len; i++) {
+        switch (flags_str[i]) {
+        case 'n': flags |= REDISMODULE_HASH_NX; break;
+        case 'x': flags |= REDISMODULE_HASH_XX; break;
+        case 'a': flags |= REDISMODULE_HASH_COUNT_ALL; break;
+        }
+    }
+
+    /* Test some varargs. (In real-world, use a loop and set one at a time.) */
+    int result;
+    errno = 0;
+    if (argc == 5) {
+        result = RedisModule_HashSet(key, flags,
+                                     argv[3], value_or_delete(argv[4]),
+                                     NULL);
+    } else if (argc == 7) {
+        result = RedisModule_HashSet(key, flags,
+                                     argv[3], value_or_delete(argv[4]),
+                                     argv[5], value_or_delete(argv[6]),
+                                     NULL);
+    } else if (argc == 9) {
+        result = RedisModule_HashSet(key, flags,
+                                     argv[3], value_or_delete(argv[4]),
+                                     argv[5], value_or_delete(argv[6]),
+                                     argv[7], value_or_delete(argv[8]),
+                                     NULL);
+    } else if (argc == 11) {
+        result = RedisModule_HashSet(key, flags,
+                                     argv[3], value_or_delete(argv[4]),
+                                     argv[5], value_or_delete(argv[6]),
+                                     argv[7], value_or_delete(argv[8]),
+                                     argv[9], value_or_delete(argv[10]),
+                                     NULL);
+    } else {
+        return RedisModule_ReplyWithError(ctx, "ERR too many fields");
+    }
+
+    /* Check errno */
+    if (result == 0) {
+        if (errno == ENOTSUP)
+            return RedisModule_ReplyWithError(ctx, REDISMODULE_ERRORMSG_WRONGTYPE);
+        else
+            RedisModule_Assert(errno == ENOENT);
+    }
+
+    return RedisModule_ReplyWithLongLong(ctx, result);
+}
+
+int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
+    REDISMODULE_NOT_USED(argv);
+    REDISMODULE_NOT_USED(argc);
+    if (RedisModule_Init(ctx, "hash", 1, REDISMODULE_APIVER_1) ==
+        REDISMODULE_OK &&
+        RedisModule_CreateCommand(ctx, "hash.set", hash_set, "",
+                                  1, 1, 1) == REDISMODULE_OK) {
+        return REDISMODULE_OK;
+    } else {
+        return REDISMODULE_ERR;
+    }
+}
diff --git a/tests/unit/moduleapi/hash.tcl b/tests/unit/moduleapi/hash.tcl
new file mode 100644
index 000000000..89bb6c63a
--- /dev/null
+++ b/tests/unit/moduleapi/hash.tcl
@@ -0,0 +1,23 @@
+set testmodule [file normalize tests/modules/hash.so]
+
+start_server {tags {"modules"}} {
+    r module load $testmodule
+
+    test {Module hash set} {
+        r set k mystring
+        assert_error "WRONGTYPE*" {r hash.set k "" hello world}
+        r del k
+        # "" = count updates and deletes of existing fields only
+        assert_equal 0 [r hash.set k "" squirrel yes]
+        # "a" = COUNT_ALL = count inserted, modified and deleted fields
+        assert_equal 2 [r hash.set k "a" banana no sushi whynot]
+        # "n" = NX = only add fields not already existing in the hash
+        # "x" = XX = only replace the value for existing fields
+        assert_equal 0 [r hash.set k "n" squirrel hoho what nothing]
+        assert_equal 1 [r hash.set k "na" squirrel hoho something nice]
+        assert_equal 0 [r hash.set k "xa" new stuff not inserted]
+        assert_equal 1 [r hash.set k "x" squirrel ofcourse]
+        assert_equal 1 [r hash.set k "" sushi :delete: none :delete:]
+        r hgetall k
+    } {squirrel ofcourse banana no what nothing something nice}
+}

From 30775bc3e3a919e6a168523e772551332738e9f3 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 15 Feb 2021 12:50:23 +0200
Subject: [PATCH 1052/1098] solve race in replication-2 test - again (#8491)

this should make it timing independent and also faster in most cases
---
 tests/integration/replication-2.tcl | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/integration/replication-2.tcl b/tests/integration/replication-2.tcl
index fc67e78c1..9524f3563 100644
--- a/tests/integration/replication-2.tcl
+++ b/tests/integration/replication-2.tcl
@@ -43,10 +43,14 @@ start_server {tags {"repl"}} {
             r config set min-slaves-max-lag 2
             exec kill -SIGSTOP [srv -1 pid]
             assert {[r set foo 12345] eq {OK}}
-            after 4000
+            wait_for_condition 100 100 {
+                [catch {r set foo 12345}] != 0
+            } else {
+                fail "Master didn't become readonly"
+            }
             catch {r set foo 12345} err
-            set err
-        } {NOREPLICAS*}
+            assert_match {NOREPLICAS*} $err
+        }
         exec kill -SIGCONT [srv -1 pid]
 
         test {min-slaves-to-write is ignored by slaves} {

From 141ac8df59c1d77e2f2a10792ff34b26eb13bcf0 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 15 Feb 2021 17:08:53 +0200
Subject: [PATCH 1053/1098] Escape unsafe field name characters in INFO.
 (#8492)

Fixes #8489
---
 src/module.c                      |  7 +++++--
 src/server.c                      | 27 +++++++++++++++++++++++++--
 src/server.h                      |  1 +
 src/util.c                        | 27 +++++++++++++++++++++++++++
 src/util.h                        |  2 ++
 tests/integration/redis-cli.tcl   |  2 +-
 tests/modules/infotest.c          |  5 +++++
 tests/unit/info.tcl               |  8 ++++++++
 tests/unit/moduleapi/infotest.tcl |  5 +++++
 9 files changed, 79 insertions(+), 5 deletions(-)

diff --git a/src/module.c b/src/module.c
index f8d3e3170..5282bd742 100644
--- a/src/module.c
+++ b/src/module.c
@@ -6759,10 +6759,13 @@ int RM_InfoBeginDictField(RedisModuleInfoCtx *ctx, char *name) {
     /* Implicitly end dicts, instead of returning an error which is likely un checked. */
     if (ctx->in_dict_field)
         RM_InfoEndDictField(ctx);
+    char *tmpmodname, *tmpname;
     ctx->info = sdscatfmt(ctx->info,
         "%s_%s:",
-        ctx->module->name,
-        name);
+        getSafeInfoString(ctx->module->name, strlen(ctx->module->name), &tmpmodname),
+        getSafeInfoString(name, strlen(name), &tmpname));
+    if (tmpmodname != NULL) zfree(tmpmodname);
+    if (tmpname != NULL) zfree(tmpname);
     ctx->in_dict_field = 1;
     return REDISMODULE_OK;
 }
diff --git a/src/server.c b/src/server.c
index bc0eb8255..3f3ac3517 100644
--- a/src/server.c
+++ b/src/server.c
@@ -4501,6 +4501,25 @@ void bytesToHuman(char *s, unsigned long long n) {
     }
 }
 
+/* Characters we sanitize on INFO output to maintain expected format. */
+static char unsafe_info_chars[] = "#:\n\r";
+static char unsafe_info_chars_substs[] = "____";   /* Must be same length as above */
+
+/* Returns a sanitized version of s that contains no unsafe info string chars.
+ * If no unsafe characters are found, simply returns s. Caller needs to
+ * free tmp if it is non-null on return.
+ */
+const char *getSafeInfoString(const char *s, size_t len, char **tmp) {
+    *tmp = NULL;
+    if (mempbrk(s, len, unsafe_info_chars,sizeof(unsafe_info_chars)-1)
+        == NULL) return s;
+    char *new = *tmp = zmalloc(len + 1);
+    memcpy(new, s, len);
+    new[len] = '\0';
+    return memmapchars(new, len, unsafe_info_chars, unsafe_info_chars_substs,
+                       sizeof(unsafe_info_chars)-1);
+}
+
 /* Create the string returned by the INFO command. This is decoupled
  * by the INFO command itself as we need to report the same information
  * on memory corruption problems. */
@@ -5126,15 +5145,17 @@ sds genRedisInfoString(const char *section) {
         dictIterator *di;
         di = dictGetSafeIterator(server.commands);
         while((de = dictNext(di)) != NULL) {
+            char *tmpsafe;
             c = (struct redisCommand *) dictGetVal(de);
             if (!c->calls && !c->failed_calls && !c->rejected_calls)
                 continue;
             info = sdscatprintf(info,
                 "cmdstat_%s:calls=%lld,usec=%lld,usec_per_call=%.2f"
                 ",rejected_calls=%lld,failed_calls=%lld\r\n",
-                c->name, c->calls, c->microseconds,
+                getSafeInfoString(c->name, strlen(c->name), &tmpsafe), c->calls, c->microseconds,
                 (c->calls == 0) ? 0 : ((float)c->microseconds/c->calls),
                 c->rejected_calls, c->failed_calls);
+            if (tmpsafe != NULL) zfree(tmpsafe);
         }
         dictReleaseIterator(di);
     }
@@ -5147,10 +5168,12 @@ sds genRedisInfoString(const char *section) {
         raxSeek(&ri,"^",NULL,0);
         struct redisError *e;
         while(raxNext(&ri)) {
+            char *tmpsafe;
             e = (struct redisError *) ri.data;
             info = sdscatprintf(info,
                 "errorstat_%.*s:count=%lld\r\n",
-                (int)ri.key_len, ri.key, e->count);
+                (int)ri.key_len, getSafeInfoString((char *) ri.key, ri.key_len, &tmpsafe), e->count);
+            if (tmpsafe != NULL) zfree(tmpsafe);
         }
         raxStop(&ri);
     }
diff --git a/src/server.h b/src/server.h
index 7140c4571..a9841886d 100644
--- a/src/server.h
+++ b/src/server.h
@@ -2685,6 +2685,7 @@ void _serverPanic(const char *file, int line, const char *msg, ...);
 #endif
 void serverLogObjectDebugInfo(const robj *o);
 void sigsegvHandler(int sig, siginfo_t *info, void *secret);
+const char *getSafeInfoString(const char *s, size_t len, char **tmp);
 sds genRedisInfoString(const char *section);
 sds genModulesInfoString(sds info);
 void enableWatchdog(int period);
diff --git a/src/util.c b/src/util.c
index 3243fa51e..8087c8b7a 100644
--- a/src/util.c
+++ b/src/util.c
@@ -244,6 +244,33 @@ long long memtoll(const char *p, int *err) {
     return val*mul;
 }
 
+/* Search a memory buffer for any set of bytes, like strpbrk().
+ * Returns pointer to first found char or NULL.
+ */
+const char *mempbrk(const char *s, size_t len, const char *chars, size_t charslen) {
+    for (size_t j = 0; j < len; j++) {
+        for (size_t n = 0; n < charslen; n++)
+            if (s[j] == chars[n]) return &s[j];
+    }
+
+    return NULL;
+}
+
+/* Modify the buffer replacing all occurrences of chars from the 'from'
+ * set with the corresponding char in the 'to' set. Always returns s.
+ */
+char *memmapchars(char *s, size_t len, const char *from, const char *to, size_t setlen) {
+    for (size_t j = 0; j < len; j++) {
+        for (size_t i = 0; i < setlen; i++) {
+            if (s[j] == from[i]) {
+                s[j] = to[i];
+                break;
+            }
+        }
+    }
+    return s;
+}
+
 /* Return the number of digits of 'v' when converted to string in radix 10.
  * See ll2string() for more information. */
 uint32_t digits10(uint64_t v) {
diff --git a/src/util.h b/src/util.h
index feaa82924..3a15c793e 100644
--- a/src/util.h
+++ b/src/util.h
@@ -49,6 +49,8 @@ int stringmatchlen(const char *p, int plen, const char *s, int slen, int nocase)
 int stringmatch(const char *p, const char *s, int nocase);
 int stringmatchlen_fuzz_test(void);
 long long memtoll(const char *p, int *err);
+const char *mempbrk(const char *s, size_t len, const char *chars, size_t charslen);
+char *memmapchars(char *s, size_t len, const char *from, const char *to, size_t setlen);
 uint32_t digits10(uint64_t v);
 uint32_t sdigits10(int64_t v);
 int ll2string(char *s, size_t len, long long value);
diff --git a/tests/integration/redis-cli.tcl b/tests/integration/redis-cli.tcl
index 1e346a9a5..d877542ed 100644
--- a/tests/integration/redis-cli.tcl
+++ b/tests/integration/redis-cli.tcl
@@ -109,7 +109,7 @@ start_server {tags {"cli"}} {
     test_interactive_cli "INFO response should be printed raw" {
         set lines [split [run_command $fd info] "\n"]
         foreach line $lines {
-            assert [regexp {^$|^#|^[a-z0-9_]+:.+} $line]
+            assert [regexp {^$|^#|^[^#:]+:} $line]
         }
     }
 
diff --git a/tests/modules/infotest.c b/tests/modules/infotest.c
index 4cb77ee87..87a89dcb1 100644
--- a/tests/modules/infotest.c
+++ b/tests/modules/infotest.c
@@ -21,6 +21,11 @@ void InfoFunc(RedisModuleInfoCtx *ctx, int for_crash_report) {
     RedisModule_InfoAddFieldLongLong(ctx, "expires", 1);
     RedisModule_InfoEndDictField(ctx);
 
+    RedisModule_InfoAddSection(ctx, "unsafe");
+    RedisModule_InfoBeginDictField(ctx, "unsafe:field");
+    RedisModule_InfoAddFieldLongLong(ctx, "value", 1);
+    RedisModule_InfoEndDictField(ctx);
+
     if (for_crash_report) {
         RedisModule_InfoAddSection(ctx, "Klingon");
         RedisModule_InfoAddFieldCString(ctx, "one", "wa’");
diff --git a/tests/unit/info.tcl b/tests/unit/info.tcl
index 5a44c0647..08171fff9 100644
--- a/tests/unit/info.tcl
+++ b/tests/unit/info.tcl
@@ -150,4 +150,12 @@ start_server {tags {"info"}} {
             assert_match {} [errorstat NOPERM]
         }
     }
+
+    start_server {} {
+        test {Unsafe command names are sanitized in INFO output} {
+            catch {r host:} e
+            set info [r info commandstats]
+            assert_match {*cmdstat_host_:calls=1*} $info
+        }
+    }
 }
diff --git a/tests/unit/moduleapi/infotest.tcl b/tests/unit/moduleapi/infotest.tcl
index 80a28656c..1ad2ee6fc 100644
--- a/tests/unit/moduleapi/infotest.tcl
+++ b/tests/unit/moduleapi/infotest.tcl
@@ -85,5 +85,10 @@ start_server {tags {"modules"}} {
         set keys [scan [regexp -inline {keys\=([\d]*)} $keyspace] keys=%d]
     } {3}
 
+    test {module info unsafe fields} {
+        set info [r info infotest_unsafe]
+        assert_match {*infotest_unsafe_field:value=1*} $info
+    }
+
     # TODO: test crash report.
 } 

From b1929bb2b6145e28335ed988589caacf57087171 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 15 Feb 2021 17:09:33 +0200
Subject: [PATCH 1054/1098] Fix incorrect shared ping length. (#8498)

---
 src/server.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index 3f3ac3517..0d8a55000 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2599,7 +2599,7 @@ void createSharedObjects(void) {
     shared.justid = createStringObject("JUSTID",6);
     shared.lastid = createStringObject("LASTID",6);
     shared.default_username = createStringObject("default",7);
-    shared.ping = createStringObject("ping",7);
+    shared.ping = createStringObject("ping",4);
     shared.setid = createStringObject("SETID",5);
     shared.keepttl = createStringObject("KEEPTTL",7);
     shared.load = createStringObject("LOAD",4);

From fb3457d157a1b79665891d2a2bfc997e47978a94 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 15 Feb 2021 17:20:03 +0200
Subject: [PATCH 1055/1098] minor test suite cleanup, revive old test (#8497)

There are two tests in other.tcl that were dependant of the sha1 package
import which meant that they didn't usually run.
The reason it was like that was that prior to the creation of DEBUG
DIGEST, the test suite used to have an equivalent function, but that's
no longer the case and this dependency isn't needed.

The other change is to revert config changes done by the test before the
test suite continues. can be useful if using `--host` to run multiple
units against the same server
---
 tests/unit/other.tcl     | 2 +-
 tests/unit/scripting.tcl | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/unit/other.tcl b/tests/unit/other.tcl
index a35ac1752..16ed092be 100644
--- a/tests/unit/other.tcl
+++ b/tests/unit/other.tcl
@@ -55,7 +55,7 @@ start_server {tags {"other"}} {
     } {*index is out of range*}
 
     tags {consistency} {
-        if {![catch {package require sha1}]} {
+        if {true} {
             if {$::accurate} {set numops 10000} else {set numops 1000}
             test {Check consistency of different data types after a reload} {
                 r flushdb
diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl
index 3aa3c0fba..c44ec74f5 100644
--- a/tests/unit/scripting.tcl
+++ b/tests/unit/scripting.tcl
@@ -468,6 +468,7 @@ start_server {tags {"scripting"}} {
         r slaveof no one
         set res
     } {102}
+    r config set aof-use-rdb-preamble yes
 
     test {EVAL timeout from AOF} {
         # generate a long running script that is propagated to the AOF as script
@@ -505,8 +506,6 @@ start_server {tags {"scripting"}} {
         $rd close
         r get x
     } {y}
-    r config set aof-use-rdb-preamble yes
-    r config set lua-replicate-commands yes
 
     test {We can call scripts rewriting client->argv from Lua} {
         r del myset

From f521498b43a8edcec01c88dc7361eaac460f2f6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= <viktor.soderqvist@est.tech>
Date: Fri, 12 Feb 2021 12:20:54 +0100
Subject: [PATCH 1056/1098] Avoid useless copying in LINDEX command for reply

---
 src/t_list.c | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/t_list.c b/src/t_list.c
index f019a7ec0..7b7cbe01f 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -324,7 +324,6 @@ void lindexCommand(client *c) {
     robj *o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp]);
     if (o == NULL || checkType(c,o,OBJ_LIST)) return;
     long index;
-    robj *value = NULL;
 
     if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != C_OK))
         return;
@@ -333,12 +332,10 @@ void lindexCommand(client *c) {
         quicklistEntry entry;
         if (quicklistIndex(o->ptr, index, &entry)) {
             if (entry.value) {
-                value = createStringObject((char*)entry.value,entry.sz);
+                addReplyBulkCBuffer(c, entry.value, entry.sz);
             } else {
-                value = createStringObjectFromLongLong(entry.longval);
+                addReplyBulkLongLong(c, entry.longval);
             }
-            addReplyBulk(c,value);
-            decrRefCount(value);
         } else {
             addReplyNull(c);
         }

From 683e530cf3c3e71266b8d18073fd026da9de4ddb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= <viktor.soderqvist@est.tech>
Date: Fri, 12 Feb 2021 12:31:41 +0100
Subject: [PATCH 1057/1098] Use stack for decoding integer-encoded values in
 list push

Less heap allocations when commands like LMOVE push integer values.
---
 src/t_list.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/t_list.c b/src/t_list.c
index 7b7cbe01f..961f59ae4 100644
--- a/src/t_list.c
+++ b/src/t_list.c
@@ -41,10 +41,13 @@
 void listTypePush(robj *subject, robj *value, int where) {
     if (subject->encoding == OBJ_ENCODING_QUICKLIST) {
         int pos = (where == LIST_HEAD) ? QUICKLIST_HEAD : QUICKLIST_TAIL;
-        value = getDecodedObject(value);
-        size_t len = sdslen(value->ptr);
-        quicklistPush(subject->ptr, value->ptr, len, pos);
-        decrRefCount(value);
+        if (value->encoding == OBJ_ENCODING_INT) {
+            char buf[32];
+            ll2string(buf, 32, (long)value->ptr);
+            quicklistPush(subject->ptr, buf, strlen(buf), pos);
+        } else {
+            quicklistPush(subject->ptr, value->ptr, sdslen(value->ptr), pos);
+        }
     } else {
         serverPanic("Unknown list encoding");
     }

From acb32d472da72a36dc73eca5fef1df71cffdc328 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= <viktor.soderqvist@est.tech>
Date: Fri, 12 Feb 2021 18:03:10 +0100
Subject: [PATCH 1058/1098] Add ziplistReplace, in-place optimized for elements
 of same size

Avoids memmove and reallocs when replacing a ziplist element of the
same encoded size as the new value.

Affects HSET, HINRBY, HINCRBYFLOAT (via hashTypeSet) and LSET (via
quicklistReplaceAtIndex).
---
 src/quicklist.c |  3 +--
 src/t_hash.c    |  7 ++---
 src/ziplist.c   | 71 +++++++++++++++++++++++++++++++++++++++++++++++++
 src/ziplist.h   |  1 +
 4 files changed, 75 insertions(+), 7 deletions(-)

diff --git a/src/quicklist.c b/src/quicklist.c
index b2a406004..c8517414c 100644
--- a/src/quicklist.c
+++ b/src/quicklist.c
@@ -680,8 +680,7 @@ int quicklistReplaceAtIndex(quicklist *quicklist, long index, void *data,
     quicklistEntry entry;
     if (likely(quicklistIndex(quicklist, index, &entry))) {
         /* quicklistIndex provides an uncompressed node */
-        entry.node->zl = ziplistDelete(entry.node->zl, &entry.zi);
-        entry.node->zl = ziplistInsert(entry.node->zl, entry.zi, data, sz);
+        entry.node->zl = ziplistReplace(entry.node->zl, entry.zi, data, sz);
         quicklistNodeUpdateSz(entry.node);
         quicklistCompress(quicklist, entry.node);
         return 1;
diff --git a/src/t_hash.c b/src/t_hash.c
index cae350566..15b57c1d7 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -215,11 +215,8 @@ int hashTypeSet(robj *o, sds field, sds value, int flags) {
                 serverAssert(vptr != NULL);
                 update = 1;
 
-                /* Delete value */
-                zl = ziplistDelete(zl, &vptr);
-
-                /* Insert new value */
-                zl = ziplistInsert(zl, vptr, (unsigned char*)value,
+                /* Replace value */
+                zl = ziplistReplace(zl, vptr, (unsigned char*)value,
                         sdslen(value));
             }
         }
diff --git a/src/ziplist.c b/src/ziplist.c
index 35a0f283a..b66f97ef8 100644
--- a/src/ziplist.c
+++ b/src/ziplist.c
@@ -1265,6 +1265,42 @@ unsigned char *ziplistDeleteRange(unsigned char *zl, int index, unsigned int num
     return (p == NULL) ? zl : __ziplistDelete(zl,p,num);
 }
 
+/* Replaces the entry at p. This is equivalent to a delete and an insert,
+ * but avoids some overhead when replacing a value of the same size. */
+unsigned char *ziplistReplace(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) {
+
+    /* get metadata of the current entry */
+    zlentry entry;
+    zipEntry(p, &entry);
+
+    /* compute length of entry to store, excluding prevlen */
+    unsigned int reqlen;
+    unsigned char encoding = 0;
+    long long value = 123456789; /* initialized to avoid warning. */
+    if (zipTryEncoding(s,slen,&value,&encoding)) {
+        reqlen = zipIntSize(encoding); /* encoding is set */
+    } else {
+        reqlen = slen; /* encoding == 0 */
+    }
+    reqlen += zipStoreEntryEncoding(NULL,encoding,slen);
+
+    if (reqlen == entry.lensize + entry.len) {
+        /* Simply overwrite the element. */
+        p += entry.prevrawlensize;
+        p += zipStoreEntryEncoding(p,encoding,slen);
+        if (ZIP_IS_STR(encoding)) {
+            memcpy(p,s,slen);
+        } else {
+            zipSaveInteger(p,value,encoding);
+        }
+    } else {
+        /* Fallback. */
+        zl = ziplistDelete(zl,&p);
+        zl = ziplistInsert(zl,p,s,slen);
+    }
+    return zl;
+}
+
 /* Compare entry pointer to by 'p' with 'sstr' of length 'slen'. */
 /* Return 1 if equal. */
 unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int slen) {
@@ -2070,6 +2106,41 @@ int ziplistTest(int argc, char **argv) {
         zfree(zl);
     }
 
+    printf("Replace with same size:\n");
+    {
+        zl = createList(); /* "hello", "foo", "quux", "1024" */
+        unsigned char *orig_zl = zl;
+        p = ziplistIndex(zl, 0);
+        zl = ziplistReplace(zl, p, (unsigned char*)"zoink", 5);
+        p = ziplistIndex(zl, 3);
+        zl = ziplistReplace(zl, p, (unsigned char*)"yy", 2);
+        p = ziplistIndex(zl, 1);
+        zl = ziplistReplace(zl, p, (unsigned char*)"65536", 5);
+        p = ziplistIndex(zl, 0);
+        assert(!memcmp((char*)p,
+                       "\x00\x05zoink"
+                       "\x07\xf0\x00\x00\x01" /* 65536 as int24 */
+                       "\x05\x04quux" "\x06\x02yy" "\xff",
+                       23));
+        assert(zl == orig_zl); /* no reallocations have happened */
+        zfree(zl);
+        printf("SUCCESS\n\n");
+    }
+
+    printf("Replace with different size:\n");
+    {
+        zl = createList(); /* "hello", "foo", "quux", "1024" */
+        p = ziplistIndex(zl, 1);
+        zl = ziplistReplace(zl, p, (unsigned char*)"squirrel", 8);
+        p = ziplistIndex(zl, 0);
+        assert(!strncmp((char*)p,
+                        "\x00\x05hello" "\x07\x08squirrel" "\x0a\x04quux"
+                        "\x06\xc0\x00\x04" "\xff",
+                        28));
+        zfree(zl);
+        printf("SUCCESS\n\n");
+    }
+
     printf("Regression test for >255 byte strings:\n");
     {
         char v1[257] = {0}, v2[257] = {0};
diff --git a/src/ziplist.h b/src/ziplist.h
index fff334cbd..9dc1061b0 100644
--- a/src/ziplist.h
+++ b/src/ziplist.h
@@ -53,6 +53,7 @@ unsigned int ziplistGet(unsigned char *p, unsigned char **sval, unsigned int *sl
 unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen);
 unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p);
 unsigned char *ziplistDeleteRange(unsigned char *zl, int index, unsigned int num);
+unsigned char *ziplistReplace(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen);
 unsigned int ziplistCompare(unsigned char *p, unsigned char *s, unsigned int slen);
 unsigned char *ziplistFind(unsigned char *zl, unsigned char *p, unsigned char *vstr, unsigned int vlen, unsigned int skip);
 unsigned int ziplistLen(unsigned char *zl);

From fd052d2a86b1a9ace29abf2868785f0b4621b715 Mon Sep 17 00:00:00 2001
From: uriyage <78144248+uriyage@users.noreply.github.com>
Date: Tue, 16 Feb 2021 16:06:51 +0200
Subject: [PATCH 1059/1098] Adds INFO fields to track fork child progress
 (#8414)

* Adding current_save_keys_total and current_save_keys_processed info fields.
  Present in replication, BGSAVE and AOFRW.
* Changing RM_SendChildCOWInfo() to RM_SendChildHeartbeat(double progress)
* Adding new info field current_fork_perc. Present in Replication, BGSAVE, AOFRW,
  and module forks.
---
 src/aof.c                 | 20 +++++-----
 src/childinfo.c           | 80 ++++++++++++++++++++++-----------------
 src/module.c              | 12 +++---
 src/rdb.c                 | 23 +++++------
 src/redismodule.h         |  4 +-
 src/server.c              | 36 +++++++++++++-----
 src/server.h              | 15 +++++++-
 tests/integration/rdb.tcl | 24 ++++++++++--
 8 files changed, 134 insertions(+), 80 deletions(-)

diff --git a/src/aof.c b/src/aof.c
index 518b98c84..f1586cf90 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -1440,7 +1440,7 @@ int rewriteAppendOnlyFileRio(rio *aof) {
     size_t processed = 0;
     int j;
     long key_count = 0;
-    long long cow_updated_time = 0;
+    long long updated_time = 0;
 
     for (j = 0; j < server.dbnum; j++) {
         char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n";
@@ -1501,18 +1501,16 @@ int rewriteAppendOnlyFileRio(rio *aof) {
                 aofReadDiffFromParent();
             }
 
-            /* Update COW info every 1 second (approximately).
+            /* Update info every 1 second (approximately).
              * in order to avoid calling mstime() on each iteration, we will
              * check the diff every 1024 keys */
-            if ((key_count & 1023) == 0) {
-                key_count = 0;
+            if ((key_count++ & 1023) == 0) {
                 long long now = mstime();
-                if (now - cow_updated_time >= 1000) {
-                    sendChildCOWInfo(CHILD_TYPE_AOF, 0, "AOF rewrite");
-                    cow_updated_time = now;
+                if (now - updated_time >= 1000) {
+                    sendChildInfo(CHILD_INFO_TYPE_CURRENT_INFO, key_count, "AOF rewrite");
+                    updated_time = now;
                 }
             }
-            key_count++;
         }
         dictReleaseIterator(di);
         di = NULL;
@@ -1613,7 +1611,7 @@ int rewriteAppendOnlyFile(char *filename) {
     size_t bytes_to_write = sdslen(server.aof_child_diff);
     const char *buf = server.aof_child_diff;
     long long cow_updated_time = mstime();
-
+    long long key_count = dbTotalServerKeyCount();
     while (bytes_to_write) {
         /* We write the AOF buffer in chunk of 8MB so that we can check the time in between them */
         size_t chunk_size = bytes_to_write < (8<<20) ? bytes_to_write : (8<<20);
@@ -1627,7 +1625,7 @@ int rewriteAppendOnlyFile(char *filename) {
         /* Update COW info */
         long long now = mstime();
         if (now - cow_updated_time >= 1000) {
-            sendChildCOWInfo(CHILD_TYPE_AOF, 0, "AOF rewrite");
+            sendChildInfo(CHILD_INFO_TYPE_CURRENT_INFO, key_count, "AOF rewrite");
             cow_updated_time = now;
         }
     }
@@ -1761,7 +1759,7 @@ int rewriteAppendOnlyFileBackground(void) {
         redisSetCpuAffinity(server.aof_rewrite_cpulist);
         snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
         if (rewriteAppendOnlyFile(tmpfile) == C_OK) {
-            sendChildCOWInfo(CHILD_TYPE_AOF, 1, "AOF rewrite");
+            sendChildCowInfo(CHILD_INFO_TYPE_AOF_COW_SIZE, "AOF rewrite");
             exitFromChild(0);
         } else {
             exitFromChild(1);
diff --git a/src/childinfo.c b/src/childinfo.c
index cae73fe46..d9dd17025 100644
--- a/src/childinfo.c
+++ b/src/childinfo.c
@@ -31,9 +31,10 @@
 #include <unistd.h>
 
 typedef struct {
-    int process_type;           /* AOF or RDB child? */
-    int on_exit;                /* COW size of active or exited child */
-    size_t cow_size;            /* Copy on write size. */
+    size_t keys;
+    size_t cow;
+    double progress;
+    childInfoType information_type; /* Type of information */
 } child_info_data;
 
 /* Open a child-parent channel used in order to move information about the
@@ -64,39 +65,48 @@ void closeChildInfoPipe(void) {
     }
 }
 
-/* Send COW data to parent. */
-void sendChildInfo(int process_type, int on_exit, size_t cow_size) {
+/* Send save data to parent. */
+void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, char *pname) {
     if (server.child_info_pipe[1] == -1) return;
 
-    child_info_data buffer = {.process_type = process_type, .on_exit = on_exit, .cow_size = cow_size};
-    ssize_t wlen = sizeof(buffer);
+    child_info_data data = {.information_type=info_type,
+            .keys=keys,
+            .cow=zmalloc_get_private_dirty(-1),
+            .progress=progress};
 
-    if (write(server.child_info_pipe[1],&buffer,wlen) != wlen) {
+    if (data.cow) {
+        serverLog((info_type == CHILD_INFO_TYPE_CURRENT_INFO) ? LL_VERBOSE : LL_NOTICE,
+                  "%s: %zu MB of memory used by copy-on-write",
+                  pname, data.cow/(1024*1024));
+    }
+
+    ssize_t wlen = sizeof(data);
+
+    if (write(server.child_info_pipe[1], &data, wlen) != wlen) {
         /* Nothing to do on error, this will be detected by the other side. */
     }
 }
 
-/* Update COW data. */
-void updateChildInfo(int process_type, int on_exit, size_t cow_size) {
-    if (!on_exit) {
-        server.stat_current_cow_bytes = cow_size;
-        return;
-    }
-
-    if (process_type == CHILD_TYPE_RDB) {
-        server.stat_rdb_cow_bytes = cow_size;
-    } else if (process_type == CHILD_TYPE_AOF) {
-        server.stat_aof_cow_bytes = cow_size;
-    } else if (process_type == CHILD_TYPE_MODULE) {
-        server.stat_module_cow_bytes = cow_size;
+/* Update Child info. */
+void updateChildInfo(childInfoType information_type, size_t cow, size_t keys, double progress) {
+    if (information_type == CHILD_INFO_TYPE_CURRENT_INFO) {
+        server.stat_current_cow_bytes = cow;
+        server.stat_current_save_keys_processed = keys;
+        if (progress != -1) server.stat_module_progress = progress;
+    } else if (information_type == CHILD_INFO_TYPE_AOF_COW_SIZE) {
+        server.stat_aof_cow_bytes = cow;
+    } else if (information_type == CHILD_INFO_TYPE_RDB_COW_SIZE) {
+        server.stat_rdb_cow_bytes = cow;
+    } else if (information_type == CHILD_INFO_TYPE_MODULE_COW_SIZE) {
+        server.stat_module_cow_bytes = cow;
     }
 }
 
-/* Read COW info data from the pipe.
- * if complete data read into the buffer, process type, copy-on-write type and copy-on-write size
- * are stored into *process_type, *on_exit and *cow_size respectively and returns 1.
+/* Read child info data from the pipe.
+ * if complete data read into the buffer, 
+ * data is stored into *buffer, and returns 1.
  * otherwise, the partial data is left in the buffer, waiting for the next read, and returns 0. */
-int readChildInfo(int *process_type, int *on_exit, size_t *cow_size) {
+int readChildInfo(childInfoType *information_type, size_t *cow, size_t *keys, double* progress) {
     /* We are using here a static buffer in combination with the server.child_info_nread to handle short reads */
     static child_info_data buffer;
     ssize_t wlen = sizeof(buffer);
@@ -111,25 +121,27 @@ int readChildInfo(int *process_type, int *on_exit, size_t *cow_size) {
 
     /* We have complete child info */
     if (server.child_info_nread == wlen) {
-        *process_type = buffer.process_type;
-        *on_exit = buffer.on_exit;
-        *cow_size = buffer.cow_size;
+        *information_type = buffer.information_type;
+        *cow = buffer.cow;
+        *keys = buffer.keys;
+        *progress = buffer.progress;
         return 1;
     } else {
         return 0;
     }
 }
 
-/* Receive COW data from child. */
+/* Receive info data from child. */
 void receiveChildInfo(void) {
     if (server.child_info_pipe[0] == -1) return;
 
-    int process_type;
-    int on_exit;
-    size_t cow_size;
+    size_t cow;
+    size_t keys;
+    double progress;
+    childInfoType information_type;
 
     /* Drain the pipe and update child info so that we get the final message. */
-    while (readChildInfo(&process_type, &on_exit, &cow_size)) {
-        updateChildInfo(process_type, on_exit, cow_size);
+    while (readChildInfo(&information_type, &cow, &keys, &progress)) {
+        updateChildInfo(information_type, cow, keys, progress);
     }
 }
diff --git a/src/module.c b/src/module.c
index 5282bd742..fe0cd8345 100644
--- a/src/module.c
+++ b/src/module.c
@@ -7696,16 +7696,18 @@ int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) {
 }
 
 /* The module is advised to call this function from the fork child once in a while,
- * so that it can report COW memory to the parent which will be reported in INFO */
-void RM_SendChildCOWInfo(void) {
-    sendChildCOWInfo(CHILD_TYPE_MODULE, 0, "Module fork");
+ * so that it can report progress and COW memory to the parent which will be
+ * reported in INFO.
+ * The `progress` argument should between 0 and 1, or -1 when not available. */
+void RM_SendChildHeartbeat(double progress) {
+    sendChildInfoGeneric(CHILD_INFO_TYPE_CURRENT_INFO, 0, progress, "Module fork");
 }
 
 /* Call from the child process when you want to terminate it.
  * retcode will be provided to the done handler executed on the parent process.
  */
 int RM_ExitFromChild(int retcode) {
-    sendChildCOWInfo(CHILD_TYPE_MODULE, 1, "Module fork");
+    sendChildCowInfo(CHILD_INFO_TYPE_MODULE_COW_SIZE, "Module fork");
     exitFromChild(retcode);
     return REDISMODULE_OK;
 }
@@ -9239,7 +9241,7 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(CommandFilterArgReplace);
     REGISTER_API(CommandFilterArgDelete);
     REGISTER_API(Fork);
-    REGISTER_API(SendChildCOWInfo);
+    REGISTER_API(SendChildHeartbeat);
     REGISTER_API(ExitFromChild);
     REGISTER_API(KillForkChild);
     REGISTER_API(RegisterInfoFunc);
diff --git a/src/rdb.c b/src/rdb.c
index 7deed2a2d..630417302 100644
--- a/src/rdb.c
+++ b/src/rdb.c
@@ -1223,7 +1223,8 @@ int rdbSaveRio(rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi) {
     size_t processed = 0;
     int j;
     long key_count = 0;
-    long long cow_updated_time = 0;
+    long long info_updated_time = 0;
+    char *pname = (rdbflags & RDBFLAGS_AOF_PREAMBLE) ? "AOF rewrite" :  "RDB";
 
     if (server.rdb_checksum)
         rdb->update_cksum = rioGenericUpdateChecksum;
@@ -1270,22 +1271,16 @@ int rdbSaveRio(rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi) {
                 aofReadDiffFromParent();
             }
 
-            /* Update COW info every 1 second (approximately).
+            /* Update child info every 1 second (approximately).
              * in order to avoid calling mstime() on each iteration, we will
              * check the diff every 1024 keys */
-            if ((key_count & 1023) == 0) {
-                key_count = 0;
+            if ((key_count++ & 1023) == 0) {
                 long long now = mstime();
-                if (now - cow_updated_time >= 1000) {
-                    if (rdbflags & RDBFLAGS_AOF_PREAMBLE) {
-                        sendChildCOWInfo(CHILD_TYPE_AOF, 0, "AOF rewrite");
-                    } else {
-                        sendChildCOWInfo(CHILD_TYPE_RDB, 0, "RDB");
-                    }
-                    cow_updated_time = now;
+                if (now - info_updated_time >= 1000) {
+                    sendChildInfo(CHILD_INFO_TYPE_CURRENT_INFO, key_count, pname);
+                    info_updated_time = now;
                 }
             }
-            key_count++;
         }
         dictReleaseIterator(di);
         di = NULL; /* So that we don't release it again on error. */
@@ -1438,7 +1433,7 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
         redisSetCpuAffinity(server.bgsave_cpulist);
         retval = rdbSave(filename,rsi);
         if (retval == C_OK) {
-            sendChildCOWInfo(CHILD_TYPE_RDB, 1, "RDB");
+            sendChildCowInfo(CHILD_INFO_TYPE_RDB_COW_SIZE, "RDB");
         }
         exitFromChild((retval == C_OK) ? 0 : 1);
     } else {
@@ -2805,7 +2800,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
             retval = C_ERR;
 
         if (retval == C_OK) {
-            sendChildCOWInfo(CHILD_TYPE_RDB, 1, "RDB");
+            sendChildCowInfo(CHILD_INFO_TYPE_RDB_COW_SIZE, "RDB");
         }
 
         rioFreeFd(&rdb);
diff --git a/src/redismodule.h b/src/redismodule.h
index 60a152452..0c2801bea 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -813,7 +813,7 @@ REDISMODULE_API int (*RedisModule_CommandFilterArgInsert)(RedisModuleCommandFilt
 REDISMODULE_API int (*RedisModule_CommandFilterArgReplace)(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_CommandFilterArgDelete)(RedisModuleCommandFilterCtx *fctx, int pos) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_Fork)(RedisModuleForkDoneHandler cb, void *user_data) REDISMODULE_ATTR;
-REDISMODULE_API void (*RedisModule_SendChildCOWInfo)(void) REDISMODULE_ATTR;
+REDISMODULE_API void (*RedisModule_SendChildHeartbeat)(double progress) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_ExitFromChild)(int retcode) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_KillForkChild)(int child_pid) REDISMODULE_ATTR;
 REDISMODULE_API float (*RedisModule_GetUsedMemoryRatio)() REDISMODULE_ATTR;
@@ -1082,7 +1082,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(CommandFilterArgReplace);
     REDISMODULE_GET_API(CommandFilterArgDelete);
     REDISMODULE_GET_API(Fork);
-    REDISMODULE_GET_API(SendChildCOWInfo);
+    REDISMODULE_GET_API(SendChildHeartbeat);
     REDISMODULE_GET_API(ExitFromChild);
     REDISMODULE_GET_API(KillForkChild);
     REDISMODULE_GET_API(GetUsedMemoryRatio);
diff --git a/src/server.c b/src/server.c
index 0d8a55000..993f3cb8b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1621,6 +1621,9 @@ void resetChildState() {
     server.child_type = CHILD_TYPE_NONE;
     server.child_pid = -1;
     server.stat_current_cow_bytes = 0;
+    server.stat_current_save_keys_processed = 0;
+    server.stat_module_progress = 0;
+    server.stat_current_save_keys_total = 0;
     updateDictResizePolicy();
     closeChildInfoPipe();
     moduleFireServerEvent(REDISMODULE_EVENT_FORK_CHILD,
@@ -3236,9 +3239,12 @@ void initServer(void) {
     server.stat_starttime = time(NULL);
     server.stat_peak_memory = 0;
     server.stat_current_cow_bytes = 0;
+    server.stat_current_save_keys_processed = 0;
+    server.stat_current_save_keys_total = 0;
     server.stat_rdb_cow_bytes = 0;
     server.stat_aof_cow_bytes = 0;
     server.stat_module_cow_bytes = 0;
+    server.stat_module_progress = 0;
     for (int j = 0; j < CLIENT_TYPE_COUNT; j++)
         server.stat_clients_type_memory[j] = 0;
     server.cron_malloc_stats.zmalloc_used = 0;
@@ -4769,10 +4775,20 @@ sds genRedisInfoString(const char *section) {
     /* Persistence */
     if (allsections || defsections || !strcasecmp(section,"persistence")) {
         if (sections++) info = sdscat(info,"\r\n");
+        double fork_perc = 0;
+        if (server.stat_module_progress) {
+            fork_perc = server.stat_module_progress * 100;
+        } else if (server.stat_current_save_keys_total) {
+            fork_perc = ((double)server.stat_current_save_keys_processed / server.stat_current_save_keys_total) * 100;
+        }
+  
         info = sdscatprintf(info,
             "# Persistence\r\n"
             "loading:%d\r\n"
             "current_cow_size:%zu\r\n"
+            "current_fork_perc:%.2f%%\r\n"
+            "current_save_keys_processed:%zu\r\n"
+            "current_save_keys_total:%zu\r\n"
             "rdb_changes_since_last_save:%lld\r\n"
             "rdb_bgsave_in_progress:%d\r\n"
             "rdb_last_save_time:%jd\r\n"
@@ -4792,6 +4808,9 @@ sds genRedisInfoString(const char *section) {
             "module_fork_last_cow_size:%zu\r\n",
             (int)server.loading,
             server.stat_current_cow_bytes,
+            fork_perc,
+            server.stat_current_save_keys_processed,
+            server.stat_current_save_keys_total,
             server.dirty,
             server.child_type == CHILD_TYPE_RDB,
             (intmax_t)server.lastsave,
@@ -5662,6 +5681,9 @@ int redisFork(int purpose) {
             server.child_pid = childpid;
             server.child_type = purpose;
             server.stat_current_cow_bytes = 0;
+            server.stat_current_save_keys_processed = 0;
+            server.stat_module_progress = 0;
+            server.stat_current_save_keys_total = dbTotalServerKeyCount();
         }
 
         updateDictResizePolicy();
@@ -5672,16 +5694,12 @@ int redisFork(int purpose) {
     return childpid;
 }
 
-void sendChildCOWInfo(int ptype, int on_exit, char *pname) {
-    size_t private_dirty = zmalloc_get_private_dirty(-1);
+void sendChildCowInfo(childInfoType info_type, char *pname) {
+    sendChildInfoGeneric(info_type, 0, -1, pname);
+}
 
-    if (private_dirty) {
-        serverLog(on_exit ? LL_NOTICE : LL_VERBOSE,
-            "%s: %zu MB of memory used by copy-on-write",
-            pname, private_dirty/(1024*1024));
-    }
-
-    sendChildInfo(ptype, on_exit, private_dirty);
+void sendChildInfo(childInfoType info_type, size_t keys, char *pname) {
+    sendChildInfoGeneric(info_type, keys, -1, pname);
 }
 
 void memtest(size_t megabytes, int passes);
diff --git a/src/server.h b/src/server.h
index a9841886d..2bfcc015d 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1142,6 +1142,13 @@ struct clusterState;
 #define CHILD_TYPE_LDB 3
 #define CHILD_TYPE_MODULE 4
 
+typedef enum childInfoType {
+    CHILD_INFO_TYPE_CURRENT_INFO,
+    CHILD_INFO_TYPE_AOF_COW_SIZE,
+    CHILD_INFO_TYPE_RDB_COW_SIZE,
+    CHILD_INFO_TYPE_MODULE_COW_SIZE
+} childInfoType;
+
 struct redisServer {
     /* General */
     pid_t pid;                  /* Main process pid. */
@@ -1270,9 +1277,12 @@ struct redisServer {
     redisAtomic long long stat_net_input_bytes; /* Bytes read from network. */
     redisAtomic long long stat_net_output_bytes; /* Bytes written to network. */
     size_t stat_current_cow_bytes;  /* Copy on write bytes while child is active. */
+    size_t stat_current_save_keys_processed;  /* Processed keys while child is active. */
+    size_t stat_current_save_keys_total;  /* Number of keys when child started. */
     size_t stat_rdb_cow_bytes;      /* Copy on write bytes during RDB saving. */
     size_t stat_aof_cow_bytes;      /* Copy on write bytes during AOF rewrite. */
     size_t stat_module_cow_bytes;   /* Copy on write bytes during module fork. */
+    double stat_module_progress;   /* Module save progress. */
     uint64_t stat_clients_type_memory[CLIENT_TYPE_COUNT];/* Mem usage by type */
     long long stat_unexpected_error_replies; /* Number of unexpected (aof-loading, replica to master, etc.) error replies */
     long long stat_total_error_replies; /* Total number of issued error replies ( command + rejected errors ) */
@@ -2060,7 +2070,9 @@ void restartAOFAfterSYNC();
 /* Child info */
 void openChildInfoPipe(void);
 void closeChildInfoPipe(void);
-void sendChildInfo(int process_type, int on_exit, size_t cow_size);
+void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, char *pname);
+void sendChildCowInfo(childInfoType info_type, char *pname);
+void sendChildInfo(childInfoType info_type, size_t keys, char *pname);
 void receiveChildInfo(void);
 
 /* Fork helpers */
@@ -2068,7 +2080,6 @@ int redisFork(int type);
 int hasActiveChildProcess();
 void resetChildState();
 int isMutuallyExclusiveChildType(int type);
-void sendChildCOWInfo(int ptype, int on_exit, char *pname);
 
 /* acl.c -- Authentication related prototypes. */
 extern rax *Users;
diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl
index a89221197..7df1e2f74 100644
--- a/tests/integration/rdb.tcl
+++ b/tests/integration/rdb.tcl
@@ -206,10 +206,13 @@ set system_name [string tolower [exec uname -s]]
 if {$system_name eq {linux}} {
 
 start_server {overrides {save ""}} {
-    test {Test child sending COW info} {
+    test {Test child sending info} {
         # make sure that rdb_last_cow_size and current_cow_size are zero (the test using new server),
         # so that the comparisons during the test will be valid
         assert {[s current_cow_size] == 0}
+        assert {[s current_save_keys_processed] == 0}
+        assert {[s current_save_keys_total] == 0}
+
         assert {[s rdb_last_cow_size] == 0}
 
         # using a 200us delay, the bgsave is empirically taking about 10 seconds.
@@ -234,23 +237,35 @@ start_server {overrides {save ""}} {
         # start background rdb save
         r bgsave
 
+        set current_save_keys_total [s current_save_keys_total]
+        if {$::verbose} {
+            puts "Keys before bgsave start: current_save_keys_total"
+        }
+
         # on each iteration, we will write some key to the server to trigger copy-on-write, and
         # wait to see that it reflected in INFO.
         set iteration 1
         while 1 {
-            # take a sample before writing new data to the server
+            # take samples before writing new data to the server
             set cow_size [s current_cow_size]
             if {$::verbose} {
                 puts "COW info before copy-on-write: $cow_size"
             }
 
+            set keys_processed [s current_save_keys_processed]
+            if {$::verbose} {
+                puts "current_save_keys_processed info : $keys_processed"
+            }
+
             # trigger copy-on-write
             r setrange key$iteration 0 [string repeat B $size]
 
             # wait to see that current_cow_size value updated (as long as the child is in progress)
             wait_for_condition 80 100 {
                 [s rdb_bgsave_in_progress] == 0 ||
-                [s current_cow_size] >= $cow_size + $size
+                [s current_cow_size] >= $cow_size + $size && 
+                [s current_save_keys_processed] > $keys_processed &&
+                [s current_fork_perc] > 0
             } else {
                 if {$::verbose} {
                     puts "COW info on fail: [s current_cow_size]"
@@ -259,6 +274,9 @@ start_server {overrides {save ""}} {
                 fail "COW info wasn't reported"
             }
 
+            # assert that $keys_processed is not greater than total keys.
+            assert_morethan_equal $current_save_keys_total $keys_processed
+
             # for no accurate, stop after 2 iterations
             if {!$::accurate && $iteration == 2} {
                 break

From aab479f8cfaa4493f5618ba05cfec0e2b406e77c Mon Sep 17 00:00:00 2001
From: yihuang <huang@crypto.com>
Date: Tue, 16 Feb 2021 22:17:38 +0800
Subject: [PATCH 1060/1098] Optimize listpack for stream usage to avoid
 repeated reallocs (#6281)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Avoid repeated reallocs growing the listpack while entries are being added.
This is done by pre-allocating the listpack to near maximum size, and using
malloc_size to check if it needs realloc or not.
When the listpack reaches the maximum number of entries, we shrink it to fit it's used size.

Co-authored-by: Viktor Söderqvist <viktor@zuiderkwast.se>
Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/listpack.c        | 22 ++++++++++++++++++----
 src/listpack.h        |  4 +++-
 src/listpack_malloc.h |  1 +
 src/t_stream.c        | 25 ++++++++++++++++++++++---
 4 files changed, 44 insertions(+), 8 deletions(-)

diff --git a/src/listpack.c b/src/listpack.c
index b403a1200..a2255f0d7 100644
--- a/src/listpack.c
+++ b/src/listpack.c
@@ -219,9 +219,12 @@ int lpStringToInt64(const char *s, unsigned long slen, int64_t *value) {
 }
 
 /* Create a new, empty listpack.
- * On success the new listpack is returned, otherwise an error is returned. */
-unsigned char *lpNew(void) {
-    unsigned char *lp = lp_malloc(LP_HDR_SIZE+1);
+ * On success the new listpack is returned, otherwise an error is returned.
+ * Pre-allocate at least `capacity` bytes of memory,
+ * over-allocated memory can be shrinked by `lpShrinkToFit`.
+ * */
+unsigned char *lpNew(size_t capacity) {
+    unsigned char *lp = lp_malloc(capacity > LP_HDR_SIZE+1 ? capacity : LP_HDR_SIZE+1);
     if (lp == NULL) return NULL;
     lpSetTotalBytes(lp,LP_HDR_SIZE+1);
     lpSetNumElements(lp,0);
@@ -234,6 +237,16 @@ void lpFree(unsigned char *lp) {
     lp_free(lp);
 }
 
+/* Shrink the memory to fit. */
+unsigned char* lpShrinkToFit(unsigned char *lp) {
+    size_t size = lpGetTotalBytes(lp);
+    if (size < lp_malloc_size(lp)) {
+        return lp_realloc(lp, size);
+    } else {
+        return lp;
+    }
+}
+
 /* Given an element 'ele' of size 'size', determine if the element can be
  * represented inside the listpack encoded as integer, and returns
  * LP_ENCODING_INT if so. Otherwise returns LP_ENCODING_STR if no integer
@@ -702,7 +715,8 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un
     unsigned char *dst = lp + poff; /* May be updated after reallocation. */
 
     /* Realloc before: we need more room. */
-    if (new_listpack_bytes > old_listpack_bytes) {
+    if (new_listpack_bytes > old_listpack_bytes &&
+        new_listpack_bytes > lp_malloc_size(lp)) {
         if ((lp = lp_realloc(lp,new_listpack_bytes)) == NULL) return NULL;
         dst = lp + poff;
     }
diff --git a/src/listpack.h b/src/listpack.h
index e8375628b..f87622c18 100644
--- a/src/listpack.h
+++ b/src/listpack.h
@@ -35,6 +35,7 @@
 #ifndef __LISTPACK_H
 #define __LISTPACK_H
 
+#include <stdlib.h>
 #include <stdint.h>
 
 #define LP_INTBUF_SIZE 21 /* 20 digits of -2^63 + 1 null term = 21. */
@@ -44,8 +45,9 @@
 #define LP_AFTER 1
 #define LP_REPLACE 2
 
-unsigned char *lpNew(void);
+unsigned char *lpNew(size_t capacity);
 void lpFree(unsigned char *lp);
+unsigned char* lpShrinkToFit(unsigned char *lp);
 unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, unsigned char *p, int where, unsigned char **newp);
 unsigned char *lpAppend(unsigned char *lp, unsigned char *ele, uint32_t size);
 unsigned char *lpDelete(unsigned char *lp, unsigned char *p, unsigned char **newp);
diff --git a/src/listpack_malloc.h b/src/listpack_malloc.h
index 401ab6f74..3a9050052 100644
--- a/src/listpack_malloc.h
+++ b/src/listpack_malloc.h
@@ -42,4 +42,5 @@
 #define lp_malloc zmalloc
 #define lp_realloc zrealloc
 #define lp_free zfree
+#define lp_malloc_size zmalloc_usable_size
 #endif
diff --git a/src/t_stream.c b/src/t_stream.c
index 7b4ffe3c4..da43fce18 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -43,6 +43,10 @@
  * avoid malloc allocation.*/
 #define STREAMID_STATIC_VECTOR_LEN 8
 
+/* Max pre-allocation for listpack. This is done to avoid abuse of a user
+ * setting stream_node_max_bytes to a huge number. */
+#define STREAM_LISTPACK_MAX_PRE_ALLOCATE 4096
+
 void streamFreeCG(streamCG *cg);
 void streamFreeNACK(streamNACK *na);
 size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start, streamID *end, size_t count, streamConsumer *consumer);
@@ -509,7 +513,13 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_
             lp = NULL;
         } else if (server.stream_node_max_entries) {
             int64_t count = lpGetInteger(lpFirst(lp));
-            if (count >= server.stream_node_max_entries) lp = NULL;
+            if (count >= server.stream_node_max_entries) {
+                /* Shrink extra pre-allocated memory */
+                lp = lpShrinkToFit(lp);
+                if (ri.data != lp)
+                    raxInsert(s->rax,ri.key,ri.key_len,lp,NULL);
+                lp = NULL;
+            }
         }
     }
 
@@ -517,8 +527,17 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_
     if (lp == NULL) {
         master_id = id;
         streamEncodeID(rax_key,&id);
-        /* Create the listpack having the master entry ID and fields. */
-        lp = lpNew();
+        /* Create the listpack having the master entry ID and fields.
+         * Pre-allocate some bytes when creating listpack to avoid realloc on
+         * every XADD. Since listpack.c uses malloc_size, it'll grow in steps,
+         * and won't realloc on every XADD.
+         * When listpack reaches max number of entries, we'll shrink the
+         * allocation to fit the data. */
+        size_t prealloc = STREAM_LISTPACK_MAX_PRE_ALLOCATE;
+        if (server.stream_node_max_bytes > 0 && server.stream_node_max_bytes < prealloc) {
+            prealloc = server.stream_node_max_bytes;
+        }
+        lp = lpNew(prealloc);
         lp = lpAppendInteger(lp,1); /* One item, the one we are adding. */
         lp = lpAppendInteger(lp,0); /* Zero deleted so far. */
         lp = lpAppendInteger(lp,numfields);

From 71ab81ec69f04c218820b4c9151cb9f7c1bd833b Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Wed, 17 Feb 2021 12:30:29 +0200
Subject: [PATCH 1061/1098] solve valgrind warning in child_info (#8505)

Valgrind warns about `write` accessing uninitialized memory, which was the struct padding.
---
 src/childinfo.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/childinfo.c b/src/childinfo.c
index d9dd17025..e3f33a96c 100644
--- a/src/childinfo.c
+++ b/src/childinfo.c
@@ -69,10 +69,11 @@ void closeChildInfoPipe(void) {
 void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, char *pname) {
     if (server.child_info_pipe[1] == -1) return;
 
-    child_info_data data = {.information_type=info_type,
-            .keys=keys,
-            .cow=zmalloc_get_private_dirty(-1),
-            .progress=progress};
+    child_info_data data = {0}; /* zero everything, including padding to sattisfy valgrind */
+    data.information_type = info_type;
+    data.keys = keys;
+    data.cow = zmalloc_get_private_dirty(-1);
+    data.progress = progress;
 
     if (data.cow) {
         serverLog((info_type == CHILD_INFO_TYPE_CURRENT_INFO) ? LL_VERBOSE : LL_NOTICE,

From 303465af35c13691f989b3400b028a94df1235d4 Mon Sep 17 00:00:00 2001
From: Harkrishn Patro <30795839+hpatro@users.noreply.github.com>
Date: Wed, 17 Feb 2021 23:13:50 +0100
Subject: [PATCH 1062/1098] Remove redundant pubsub list to store the patterns.
 (#8472)

Remove redundant pubsub list to store the patterns.
---
 src/cluster.c |  2 +-
 src/dict.c    |  2 +-
 src/pubsub.c  | 36 ++++++------------------------------
 src/server.c  |  7 ++-----
 src/server.h  | 10 +---------
 5 files changed, 11 insertions(+), 46 deletions(-)

diff --git a/src/cluster.c b/src/cluster.c
index e3fab02e9..efe2f652d 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -2126,7 +2126,7 @@ int clusterProcessPacket(clusterLink *link) {
         /* Don't bother creating useless objects if there are no
          * Pub/Sub subscribers. */
         if (dictSize(server.pubsub_channels) ||
-           listLength(server.pubsub_patterns))
+           dictSize(server.pubsub_patterns))
         {
             channel_len = ntohl(hdr->data.publish.msg.channel_len);
             message_len = ntohl(hdr->data.publish.msg.message_len);
diff --git a/src/dict.c b/src/dict.c
index 6c203b850..9ae066f33 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -301,7 +301,7 @@ int dictAdd(dict *d, void *key, void *val)
 /* Low level add or find:
  * This function adds the entry but instead of setting a value returns the
  * dictEntry structure to the user, that will make sure to fill the value
- * field as he wishes.
+ * field as they wish.
  *
  * This function is also directly exposed to the user API to be called
  * mainly in order to store non-pointers inside the hash value, example:
diff --git a/src/pubsub.c b/src/pubsub.c
index 7355e10b9..a7b370d5d 100644
--- a/src/pubsub.c
+++ b/src/pubsub.c
@@ -124,20 +124,6 @@ void addReplyPubsubPatUnsubscribed(client *c, robj *pattern) {
  * Pubsub low level API
  *----------------------------------------------------------------------------*/
 
-void freePubsubPattern(void *p) {
-    pubsubPattern *pat = p;
-
-    decrRefCount(pat->pattern);
-    zfree(pat);
-}
-
-int listMatchPubsubPattern(void *a, void *b) {
-    pubsubPattern *pa = a, *pb = b;
-
-    return (pa->client == pb->client) &&
-           (equalStringObjects(pa->pattern,pb->pattern));
-}
-
 /* Return the number of channels + patterns a client is subscribed to. */
 int clientSubscriptionsCount(client *c) {
     return dictSize(c->pubsub_channels)+
@@ -212,18 +198,13 @@ int pubsubSubscribePattern(client *c, robj *pattern) {
 
     if (listSearchKey(c->pubsub_patterns,pattern) == NULL) {
         retval = 1;
-        pubsubPattern *pat;
         listAddNodeTail(c->pubsub_patterns,pattern);
         incrRefCount(pattern);
-        pat = zmalloc(sizeof(*pat));
-        pat->pattern = getDecodedObject(pattern);
-        pat->client = c;
-        listAddNodeTail(server.pubsub_patterns,pat);
         /* Add the client to the pattern -> list of clients hash table */
-        de = dictFind(server.pubsub_patterns_dict,pattern);
+        de = dictFind(server.pubsub_patterns,pattern);
         if (de == NULL) {
             clients = listCreate();
-            dictAdd(server.pubsub_patterns_dict,pattern,clients);
+            dictAdd(server.pubsub_patterns,pattern,clients);
             incrRefCount(pattern);
         } else {
             clients = dictGetVal(de);
@@ -241,19 +222,14 @@ int pubsubUnsubscribePattern(client *c, robj *pattern, int notify) {
     dictEntry *de;
     list *clients;
     listNode *ln;
-    pubsubPattern pat;
     int retval = 0;
 
     incrRefCount(pattern); /* Protect the object. May be the same we remove */
     if ((ln = listSearchKey(c->pubsub_patterns,pattern)) != NULL) {
         retval = 1;
         listDelNode(c->pubsub_patterns,ln);
-        pat.client = c;
-        pat.pattern = pattern;
-        ln = listSearchKey(server.pubsub_patterns,&pat);
-        listDelNode(server.pubsub_patterns,ln);
         /* Remove the client from the pattern -> clients list hash table */
-        de = dictFind(server.pubsub_patterns_dict,pattern);
+        de = dictFind(server.pubsub_patterns,pattern);
         serverAssertWithInfo(c,NULL,de != NULL);
         clients = dictGetVal(de);
         ln = listSearchKey(clients,c);
@@ -262,7 +238,7 @@ int pubsubUnsubscribePattern(client *c, robj *pattern, int notify) {
         if (listLength(clients) == 0) {
             /* Free the list and associated hash entry at all if this was
              * the latest client. */
-            dictDelete(server.pubsub_patterns_dict,pattern);
+            dictDelete(server.pubsub_patterns,pattern);
         }
     }
     /* Notify the client */
@@ -329,7 +305,7 @@ int pubsubPublishMessage(robj *channel, robj *message) {
         }
     }
     /* Send to clients listening to matching channels */
-    di = dictGetIterator(server.pubsub_patterns_dict);
+    di = dictGetIterator(server.pubsub_patterns);
     if (di) {
         channel = getDecodedObject(channel);
         while((de = dictNext(di)) != NULL) {
@@ -502,7 +478,7 @@ NULL
         }
     } else if (!strcasecmp(c->argv[1]->ptr,"numpat") && c->argc == 2) {
         /* PUBSUB NUMPAT */
-        addReplyLongLong(c,listLength(server.pubsub_patterns));
+        addReplyLongLong(c,dictSize(server.pubsub_patterns));
     } else {
         addReplySubcommandSyntaxError(c);
     }
diff --git a/src/server.c b/src/server.c
index 993f3cb8b..17a25168f 100644
--- a/src/server.c
+++ b/src/server.c
@@ -3206,10 +3206,7 @@ void initServer(void) {
     }
     evictionPoolAlloc(); /* Initialize the LRU keys pool. */
     server.pubsub_channels = dictCreate(&keylistDictType,NULL);
-    server.pubsub_patterns = listCreate();
-    server.pubsub_patterns_dict = dictCreate(&keylistDictType,NULL);
-    listSetFreeMethod(server.pubsub_patterns,freePubsubPattern);
-    listSetMatchMethod(server.pubsub_patterns,listMatchPubsubPattern);
+    server.pubsub_patterns = dictCreate(&keylistDictType,NULL);
     server.cronloops = 0;
     server.in_eval = 0;
     server.in_exec = 0;
@@ -4959,7 +4956,7 @@ sds genRedisInfoString(const char *section) {
             server.stat_keyspace_hits,
             server.stat_keyspace_misses,
             dictSize(server.pubsub_channels),
-            listLength(server.pubsub_patterns),
+            dictSize(server.pubsub_patterns),
             server.stat_fork_time,
             server.stat_total_forks,
             dictSize(server.migrate_cached_sockets),
diff --git a/src/server.h b/src/server.h
index 2bfcc015d..8f7e70261 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1521,8 +1521,7 @@ struct redisServer {
     long long blocked_last_cron; /* Indicate the mstime of the last time we did cron jobs from a blocking operation */
     /* Pubsub */
     dict *pubsub_channels;  /* Map channels to list of subscribed clients */
-    list *pubsub_patterns;  /* A list of pubsub_patterns */
-    dict *pubsub_patterns_dict;  /* A dict of pubsub_patterns */
+    dict *pubsub_patterns;  /* A dict of pubsub_patterns */
     int notify_keyspace_events; /* Events to propagate via Pub/Sub. This is an
                                    xor of NOTIFY_... flags. */
     /* Cluster */
@@ -1609,11 +1608,6 @@ struct redisServer {
     int failover_state; /* Failover state */
 };
 
-typedef struct pubsubPattern {
-    client *client;
-    robj *pattern;
-} pubsubPattern;
-
 #define MAX_KEYS_BUFFER 256
 
 /* A result structure for the various getkeys function calls. It lists the
@@ -2280,8 +2274,6 @@ int hashZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep);
 /* Pub / Sub */
 int pubsubUnsubscribeAllChannels(client *c, int notify);
 int pubsubUnsubscribeAllPatterns(client *c, int notify);
-void freePubsubPattern(void *p);
-int listMatchPubsubPattern(void *a, void *b);
 int pubsubPublishMessage(robj *channel, robj *message);
 void addReplyPubsubMessage(client *c, robj *channel, robj *msg);
 

From 46346e9e3a30d0d3c52dd21a2ce93898005c7ad9 Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Fri, 19 Feb 2021 19:01:25 +0800
Subject: [PATCH 1063/1098] Fix timing error oom-score-adj test (#8513)

fixes timing issue, fork didn't always get to set the oom score before the test verified it.
---
 tests/unit/oom-score-adj.tcl | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/unit/oom-score-adj.tcl b/tests/unit/oom-score-adj.tcl
index efa61b759..892b1a7ad 100644
--- a/tests/unit/oom-score-adj.tcl
+++ b/tests/unit/oom-score-adj.tcl
@@ -39,7 +39,12 @@ if {$system_name eq {linux}} {
             r bgsave
 
             set child_pid [get_child_pid 0]
-            assert_equal [get_oom_score_adj $child_pid] [expr $base + 30]
+            # Wait until background child process to setOOMScoreAdj success.
+            wait_for_condition 100 10 {
+                [get_oom_score_adj $child_pid] == [expr $base + 30]
+            } else {
+                fail "Set oom-score-adj of background child process is not ok"
+            }
         }
 
         # Failed oom-score-adj tests can only run unprivileged

From 06966d2a0e6a2087e4f70265e0f03fa8e1d1b94b Mon Sep 17 00:00:00 2001
From: Jim Brunner <brunnerj@amazon.com>
Date: Sat, 20 Feb 2021 02:56:30 -0800
Subject: [PATCH 1064/1098] dict: pause rehash, minor readability refactor
 (#8515)

The `dict` field `iterators` is misleading and incorrect.
This variable is used for 1 purpose - to pause rehashing.

The current `iterators` field doesn't actually count "iterators".
It counts "safe iterators".  But - it doesn't actually count safe iterators
either.  For one, it's only incremented once the safe iterator begins to
iterate, not when it's created.  It's also incremented in `dictScan` to
prevent rehashing (and commented to make it clear why `iterators` is
being incremented during a scan).

This update renames the field as `pauserehash` and creates 2 helper
macros `dictPauseRehashing(d)` and `dictResumeRehashing(d)`
---
 src/dict.c | 24 +++++++++++-------------
 src/dict.h |  4 +++-
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/dict.c b/src/dict.c
index 9ae066f33..4a9f3fb0a 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -126,7 +126,7 @@ int _dictInit(dict *d, dictType *type,
     d->type = type;
     d->privdata = privDataPtr;
     d->rehashidx = -1;
-    d->iterators = 0;
+    d->pauserehash = 0;
     return DICT_OK;
 }
 
@@ -264,7 +264,7 @@ long long timeInMilliseconds(void) {
  * than 0, and is smaller than 1 in most cases. The exact upper bound 
  * depends on the running time of dictRehash(d,100).*/
 int dictRehashMilliseconds(dict *d, int ms) {
-    if (d->iterators > 0) return 0;
+    if (d->pauserehash > 0) return 0;
 
     long long start = timeInMilliseconds();
     int rehashes = 0;
@@ -276,8 +276,8 @@ int dictRehashMilliseconds(dict *d, int ms) {
     return rehashes;
 }
 
-/* This function performs just a step of rehashing, and only if there are
- * no safe iterators bound to our hash table. When we have iterators in the
+/* This function performs just a step of rehashing, and only if hashing has
+ * not been paused for our hash table. When we have iterators in the
  * middle of a rehashing we can't mess with the two hash tables otherwise
  * some element can be missed or duplicated.
  *
@@ -285,7 +285,7 @@ int dictRehashMilliseconds(dict *d, int ms) {
  * dictionary so that the hash table automatically migrates from H1 to H2
  * while it is actively used. */
 static void _dictRehashStep(dict *d) {
-    if (d->iterators == 0) dictRehash(d,1);
+    if (d->pauserehash == 0) dictRehash(d,1);
 }
 
 /* Add an element to the target hash table */
@@ -593,7 +593,7 @@ dictEntry *dictNext(dictIterator *iter)
             dictht *ht = &iter->d->ht[iter->table];
             if (iter->index == -1 && iter->table == 0) {
                 if (iter->safe)
-                    iter->d->iterators++;
+                    dictPauseRehashing(iter->d);
                 else
                     iter->fingerprint = dictFingerprint(iter->d);
             }
@@ -625,7 +625,7 @@ void dictReleaseIterator(dictIterator *iter)
 {
     if (!(iter->index == -1 && iter->table == 0)) {
         if (iter->safe)
-            iter->d->iterators--;
+            dictResumeRehashing(iter->d);
         else
             assert(iter->fingerprint == dictFingerprint(iter->d));
     }
@@ -896,9 +896,8 @@ unsigned long dictScan(dict *d,
 
     if (dictSize(d) == 0) return 0;
 
-    /* Having a safe iterator means no rehashing can happen, see _dictRehashStep.
-     * This is needed in case the scan callback tries to do dictFind or alike. */
-    d->iterators++;
+    /* This is needed in case the scan callback tries to do dictFind or alike. */
+    dictPauseRehashing(d);
 
     if (!dictIsRehashing(d)) {
         t0 = &(d->ht[0]);
@@ -966,8 +965,7 @@ unsigned long dictScan(dict *d,
         } while (v & (m0 ^ m1));
     }
 
-    /* undo the ++ at the top */
-    d->iterators--;
+    dictResumeRehashing(d);
 
     return v;
 }
@@ -1056,7 +1054,7 @@ void dictEmpty(dict *d, void(callback)(void*)) {
     _dictClear(d,&d->ht[0],callback);
     _dictClear(d,&d->ht[1],callback);
     d->rehashidx = -1;
-    d->iterators = 0;
+    d->pauserehash = 0;
 }
 
 void dictEnableResize(void) {
diff --git a/src/dict.h b/src/dict.h
index d96c3148f..bd57f859e 100644
--- a/src/dict.h
+++ b/src/dict.h
@@ -82,7 +82,7 @@ typedef struct dict {
     void *privdata;
     dictht ht[2];
     long rehashidx; /* rehashing not in progress if rehashidx == -1 */
-    unsigned long iterators; /* number of iterators currently running */
+    int16_t pauserehash; /* If >0 rehashing is paused (<0 indicates coding error) */
 } dict;
 
 /* If safe is set to 1 this is a safe iterator, that means, you can call
@@ -150,6 +150,8 @@ typedef void (dictScanBucketFunction)(void *privdata, dictEntry **bucketref);
 #define dictSlots(d) ((d)->ht[0].size+(d)->ht[1].size)
 #define dictSize(d) ((d)->ht[0].used+(d)->ht[1].used)
 #define dictIsRehashing(d) ((d)->rehashidx != -1)
+#define dictPauseRehashing(d) (d)->pauserehash++
+#define dictResumeRehashing(d) (d)->pauserehash--
 
 /* If our unsigned long type can store a 64 bit number, use a 64 bit PRNG. */
 #if ULONG_MAX >= 0xffffffffffffffff

From 0772098b1b8587266e3a459d9b1225e6e5d4ce58 Mon Sep 17 00:00:00 2001
From: Gnanesh <gnaneshkunal@outlook.com>
Date: Sun, 21 Feb 2021 12:39:54 +0530
Subject: [PATCH 1065/1098] EXPIRE, EXPIREAT, SETEX, GETEX: Return error when
 expire time overflows (#8287)

Respond with error if expire time overflows from positive to negative of vice versa.

* `SETEX`, `SET EX`, `GETEX` etc would have already error on negative value,
but now they would also error on overflows (i.e. when the input was positive but
after the manipulation it becomes negative, which would have passed before)
* `EXPIRE` and `EXPIREAT` was ok taking negative values (would implicitly delete
the key), we keep that, but we do error if the user provided a value that changes
sign when manipulated (except the case of changing sign when `basetime` is added)

Signed-off-by: Gnanesh <gnaneshkunal@outlook.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/expire.c          |  9 ++++++--
 src/t_string.c        | 51 ++++++++++++++++++++++++------------------
 tests/unit/expire.tcl | 52 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 88 insertions(+), 24 deletions(-)

diff --git a/src/expire.c b/src/expire.c
index 0ce34aea8..982301542 100644
--- a/src/expire.c
+++ b/src/expire.c
@@ -506,10 +506,15 @@ void expireGenericCommand(client *c, long long basetime, int unit) {
 
     if (getLongLongFromObjectOrReply(c, param, &when, NULL) != C_OK)
         return;
-
+    int negative_when = when < 0;
     if (unit == UNIT_SECONDS) when *= 1000;
     when += basetime;
-
+    if (((when < 0) && !negative_when) || ((when-basetime > 0) && negative_when)) {
+        /* EXPIRE allows negative numbers, but we can at least detect an
+         * overflow by either unit conversion or basetime addition. */
+        addReplyErrorFormat(c, "invalid expire time in %s", c->cmd->name);
+        return;
+    }
     /* No key, return zero. */
     if (lookupKeyWrite(c->db,key) == NULL) {
         addReply(c,shared.czero);
diff --git a/src/t_string.c b/src/t_string.c
index 6a3b256b8..cdf69110f 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -73,16 +73,25 @@ static int checkStringLength(client *c, long long size) {
 #define OBJ_PERSIST (1<<8)         /* Set if we need to remove the ttl */
 
 void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire, int unit, robj *ok_reply, robj *abort_reply) {
-    long long milliseconds = 0; /* initialized to avoid any harmness warning */
+    long long milliseconds = 0, when = 0; /* initialized to avoid any harmness warning */
 
     if (expire) {
         if (getLongLongFromObjectOrReply(c, expire, &milliseconds, NULL) != C_OK)
             return;
-        if (milliseconds <= 0) {
-            addReplyErrorFormat(c,"invalid expire time in %s",c->cmd->name);
+        if (milliseconds <= 0 || (unit == UNIT_SECONDS && milliseconds >= LLONG_MAX / 1000)) {
+            /* Negative value provided or multiplication is gonna overflow. */
+            addReplyErrorFormat(c, "invalid expire time in %s", c->cmd->name);
             return;
         }
         if (unit == UNIT_SECONDS) milliseconds *= 1000;
+        when = milliseconds;
+        if ((flags & OBJ_PX) || (flags & OBJ_EX))
+            when += mstime();
+        if (when <= 0) {
+            /* Overflow detected. */
+            addReplyErrorFormat(c, "invalid expire time in %s", c->cmd->name);
+            return;
+        }
     }
 
     if ((flags & OBJ_SET_NX && lookupKeyWrite(c->db,key) != NULL) ||
@@ -100,14 +109,7 @@ void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire,
     server.dirty++;
     notifyKeyspaceEvent(NOTIFY_STRING,"set",key,c->db->id);
     if (expire) {
-        robj *exp = shared.pxat;
-
-        if ((flags & OBJ_PX) || (flags & OBJ_EX)) {
-            setExpire(c,c->db,key,milliseconds + mstime());
-            exp = shared.px;
-        } else {
-            setExpire(c,c->db,key,milliseconds);
-        }
+        setExpire(c,c->db,key,when);
         notifyKeyspaceEvent(NOTIFY_GENERIC,"expire",key,c->db->id);
 
         /* Propagate as SET Key Value PXAT millisecond-timestamp if there is EXAT/PXAT or
@@ -119,6 +121,7 @@ void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire,
          * Additional care is required while modifying the argument order. AOF relies on the
          * exp argument being at index 3. (see feedAppendOnlyFile)
          * */
+        robj *exp = (flags & OBJ_PXAT) || (flags & OBJ_EXAT) ? shared.pxat : shared.px;
         robj *millisecondObj = createStringObjectFromLongLong(milliseconds);
         rewriteClientCommandVector(c,5,shared.set,key,val,exp,millisecondObj);
         decrRefCount(millisecondObj);
@@ -335,17 +338,26 @@ void getexCommand(client *c) {
         return;
     }
 
-    long long milliseconds = 0;
+    long long milliseconds = 0, when = 0;
 
     /* Validate the expiration time value first */
     if (expire) {
         if (getLongLongFromObjectOrReply(c, expire, &milliseconds, NULL) != C_OK)
             return;
-        if (milliseconds <= 0) {
-            addReplyErrorFormat(c,"invalid expire time in %s",c->cmd->name);
+        if (milliseconds <= 0 || (unit == UNIT_SECONDS && milliseconds >= LLONG_MAX / 1000)) {
+            /* Negative value provided or multiplication is gonna overflow. */
+            addReplyErrorFormat(c, "invalid expire time in %s", c->cmd->name);
             return;
         }
         if (unit == UNIT_SECONDS) milliseconds *= 1000;
+        when = milliseconds;
+        if ((flags & OBJ_PX) || (flags & OBJ_EX))
+            when += mstime();
+        if (when <= 0) {
+            /* Overflow detected. */
+            addReplyErrorFormat(c, "invalid expire time in %s", c->cmd->name);
+            return;
+        }
     }
 
     /* We need to do this before we expire the key or delete it */
@@ -365,14 +377,9 @@ void getexCommand(client *c) {
         notifyKeyspaceEvent(NOTIFY_GENERIC, "del", c->argv[1], c->db->id);
         server.dirty++;
     } else if (expire) {
-        robj *exp = shared.pexpireat;
-        if ((flags & OBJ_PX) || (flags & OBJ_EX)) {
-            setExpire(c,c->db,c->argv[1],milliseconds + mstime());
-            exp = shared.pexpire;
-        } else {
-            setExpire(c,c->db,c->argv[1],milliseconds);
-        }
-
+        setExpire(c,c->db,c->argv[1],when);
+        /* Propagate */
+        robj *exp = (flags & OBJ_PXAT) || (flags & OBJ_EXAT) ? shared.pexpireat : shared.pexpire;
         robj* millisecondObj = createStringObjectFromLongLong(milliseconds);
         rewriteClientCommandVector(c,3,exp,c->argv[1],millisecondObj);
         decrRefCount(millisecondObj);
diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl
index 9bde4809f..900856848 100644
--- a/tests/unit/expire.tcl
+++ b/tests/unit/expire.tcl
@@ -209,6 +209,58 @@ start_server {tags {"expire"}} {
         set e
     } {*not an integer*}
 
+    test {SET with EX with big integer should report an error} {
+        catch {r set foo bar EX 10000000000000000} e
+        set e
+    } {ERR invalid expire time in set}
+
+    test {SET with EX with smallest integer should report an error} {
+        catch {r SET foo bar EX -9999999999999999} e
+        set e
+    } {ERR invalid expire time in set}
+
+    test {GETEX with big integer should report an error} {
+        r set foo bar
+        catch {r GETEX foo EX 10000000000000000} e
+        set e
+    } {ERR invalid expire time in getex}
+
+    test {GETEX with smallest integer should report an error} {
+        r set foo bar
+        catch {r GETEX foo EX -9999999999999999} e
+        set e
+    } {ERR invalid expire time in getex}
+
+    test {EXPIRE with big integer overflows when converted to milliseconds} {
+        r set foo bar
+        catch {r EXPIRE foo 10000000000000000} e
+        set e
+    } {ERR invalid expire time in expire}
+
+    test {PEXPIRE with big integer overflow when basetime is added} {
+        r set foo bar
+        catch {r PEXPIRE foo 9223372036854770000} e
+        set e
+    } {ERR invalid expire time in pexpire}
+
+    test {EXPIRE with big negative integer} {
+        r set foo bar
+        catch {r EXPIRE foo -9999999999999999} e
+        assert_match {ERR invalid expire time in expire} $e
+        r ttl foo
+    } {-1}
+
+    test {PEXPIREAT with big integer works} {
+        r set foo bar
+        r PEXPIREAT foo 9223372036854770000
+    } {1}
+
+    test {PEXPIREAT with big negative integer works} {
+        r set foo bar
+        r PEXPIREAT foo -9223372036854770000
+        r ttl foo
+    } {-2}
+
     test {EXPIRE and SET/GETEX EX/PX/EXAT/PXAT option, TTL should not be reset after loadaof} {
         # This test makes sure that expire times are propagated as absolute
         # times to the AOF file and not as relative time, so that when the AOF

From f687ac0c32e3857aff56ddc4711be18dd87c336c Mon Sep 17 00:00:00 2001
From: Huang Zw <huang_zhw@126.com>
Date: Sun, 21 Feb 2021 15:34:46 +0800
Subject: [PATCH 1066/1098] Client tracking tracking-redir-broken push len is 2
 not 3 (#8456)

When redis responds with tracking-redir-broken push message (RESP3),
it was responding with a broken protocol: an array of 3 elements, but only
pushes 2 elements.

Some bugs in the test make this pass. Read the push reply
will consume an extra reply, because the reply length is 3, but there
are only two elements, so the next reply will be treated as third
element. So the test is corrected too.

Other changes:
* checkPrefixCollisionsOrReply success should return 1 instead of -1,
  this bug didn't have any implications.
* improve client tracking tests to validate more of the response it reads.
---
 src/tracking.c          |   4 +-
 tests/support/redis.tcl |   1 +
 tests/unit/tracking.tcl | 121 ++++++++++++++++++++++++++++------------
 3 files changed, 87 insertions(+), 39 deletions(-)

diff --git a/src/tracking.c b/src/tracking.c
index 1cf226e52..a11e4b7d7 100644
--- a/src/tracking.c
+++ b/src/tracking.c
@@ -147,7 +147,7 @@ int checkPrefixCollisionsOrReply(client *c, robj **prefixes, size_t numprefix) {
             }
         }
     }
-    return -1;
+    return 1;
 }
 
 /* Set the client 'c' to track the prefix 'prefix'. If the client 'c' is
@@ -269,7 +269,7 @@ void sendTrackingMessage(client *c, char *keyname, size_t keylen, int proto) {
              * are unable to send invalidation messages to the redirected
              * connection, because the client no longer exist. */
             if (c->resp > 2) {
-                addReplyPushLen(c,3);
+                addReplyPushLen(c,2);
                 addReplyBulkCBuffer(c,"tracking-redir-broken",21);
                 addReplyLongLong(c,c->client_tracking_redirection);
             }
diff --git a/tests/support/redis.tcl b/tests/support/redis.tcl
index 54b49920d..373058daf 100644
--- a/tests/support/redis.tcl
+++ b/tests/support/redis.tcl
@@ -248,6 +248,7 @@ proc ::redis::redis_read_reply {id fd} {
         - {return -code error [redis_read_line $fd]}
         $ {redis_bulk_read $fd}
         > -
+        ~ -
         * {redis_multi_bulk_read $id $fd}
         % {redis_read_map $id $fd}
         default {
diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl
index 7aaca47ca..40f1a2a66 100644
--- a/tests/unit/tracking.tcl
+++ b/tests/unit/tracking.tcl
@@ -17,6 +17,10 @@ start_server {tags {"tracking network"}} {
 
     proc clean_all {} {
         uplevel {
+            # We should make r TRACKING off first. If r is in RESP3,
+            # r FLUSH ALL will send us tracking-redir-broken or other
+            # info which will not be consumed.
+            r CLIENT TRACKING off
             $rd QUIT
             $rd_redirection QUIT
             set rd [redis_deferring_client]
@@ -26,7 +30,6 @@ start_server {tags {"tracking network"}} {
             $rd_redirection subscribe __redis__:invalidate
             $rd_redirection read ; # Consume the SUBSCRIBE reply.
             r FLUSHALL
-            r CLIENT TRACKING off
             r HELLO 2
             r config set tracking-table-max-keys 1000000
         }
@@ -52,7 +55,7 @@ start_server {tags {"tracking network"}} {
         # so that we can check for leaks, as a side effect.
         r MGET a b c d e f g
         r CLIENT TRACKING off
-    }
+    } {*OK}
 
     test {Clients can enable the BCAST mode with the empty prefix} {
         r CLIENT TRACKING on BCAST REDIRECT $redir_id
@@ -72,6 +75,8 @@ start_server {tags {"tracking network"}} {
         r INCR a:2
         r INCR b:1
         r INCR b:2
+        # we should not get this key
+        r INCR c:1
         r EXEC
         # Because of the internals, we know we are going to receive
         # two separated notifications for the two different prefixes.
@@ -96,7 +101,7 @@ start_server {tags {"tracking network"}} {
         r SET loopkey 1 ; # We should not get this
         $rd_sg SET otherkey2 1; # We should get this
         # Because of the internals, we know we are going to receive
-        # two separated notifications for the two different prefixes.
+        # two separated notifications for the two different keys.
         set keys1 [lsort [lindex [$rd_redirection read] 2]]
         set keys2 [lsort [lindex [$rd_redirection read] 2]]
         set keys [lsort [list {*}$keys1 {*}$keys2]]
@@ -109,8 +114,8 @@ start_server {tags {"tracking network"}} {
         $rd_sg SET otherkey1 1; # We should get this
         r SET loopkey 1 ; # We should not get this
         $rd_sg SET otherkey2 1; # We should get this
-        # Because of the internals, we know we are going to receive
-        # two separated notifications for the two different prefixes.
+        # Because $rd_sg send command synchronously, we know we are
+        # going to receive two separated notifications.
         set keys1 [lsort [lindex [$rd_redirection read] 2]]
         set keys2 [lsort [lindex [$rd_redirection read] 2]]
         set keys [lsort [list {*}$keys1 {*}$keys2]]
@@ -123,10 +128,7 @@ start_server {tags {"tracking network"}} {
         r SET mykey myval px 1
         r SET mykeyotherkey myval ; # We should not get it
         after 1000
-        # Because of the internals, we know we are going to receive
-        # two separated notifications for the two different prefixes.
-        set keys1 [lsort [lindex [$rd_redirection read] 2]]
-        set keys [lsort [list {*}$keys1]]
+        set keys [lsort [lindex [$rd_redirection read] 2]]
         assert {$keys eq {mykey}}
     }
 
@@ -172,6 +174,7 @@ start_server {tags {"tracking network"}} {
     }
 
     test {Invalidations of previous keys can be redirected after switching to RESP3} {
+        r HELLO 2
         $rd_sg SET key1 1
         r GET key1
         r HELLO 3
@@ -194,28 +197,33 @@ start_server {tags {"tracking network"}} {
         $rd_sg SET key1 1
         r GET key1
         $rd_redirection QUIT
+        assert_equal OK [$rd_redirection read]
         $rd_sg SET key1 2
         set MAX_TRIES 100
-        for {set i 0} {$i <= $MAX_TRIES && $res <= 0} {incr i} {
+        set res -1
+        for {set i 0} {$i <= $MAX_TRIES && $res < 0} {incr i} {
             set res [lsearch -exact [r PING] "tracking-redir-broken"]
         }
         assert {$res >= 0}
-    }
+        # Consume PING reply
+        assert_equal PONG [r read]
 
-    test {Different clients can redirect to the same connection} {
         # Reinstantiating after QUIT
         set rd_redirection [redis_deferring_client]
         $rd_redirection CLIENT ID
         set redir_id [$rd_redirection read]
         $rd_redirection SUBSCRIBE __redis__:invalidate
         $rd_redirection read ; # Consume the SUBSCRIBE reply
+    }
+
+    test {Different clients can redirect to the same connection} {
         r CLIENT TRACKING on REDIRECT $redir_id
         $rd CLIENT TRACKING on REDIRECT $redir_id 
-        $rd read ; # Consume the TRACKING reply
+        assert_equal OK [$rd read] ; # Consume the TRACKING reply
         $rd_sg MSET key1 1 key2 1
         r GET key1
         $rd GET key2 
-        $rd read ; # Consume the GET reply
+        assert_equal 1 [$rd read] ; # Consume the GET reply
         $rd_sg INCR key1
         $rd_sg INCR key2
         set res1 [lindex [$rd_redirection read] 2]
@@ -226,18 +234,19 @@ start_server {tags {"tracking network"}} {
 
     test {Different clients using different protocols can track the same key} {
         $rd HELLO 3 
-        $rd read ; # Consume the HELLO reply
+        set reply [$rd read] ; # Consume the HELLO reply
+        assert_equal 3 [dict get $reply proto]
         $rd CLIENT TRACKING on 
-        $rd read ; # Consume the TRACKING reply
+        assert_equal OK [$rd read] ; # Consume the TRACKING reply
         $rd_sg set key1 1
         r GET key1
         $rd GET key1 
-        $rd read ; # Consume the GET reply
+        assert_equal 1 [$rd read] ; # Consume the GET reply
         $rd_sg INCR key1
         set res1 [lindex [$rd_redirection read] 2]
         $rd PING ; # Non redirecting client has to talk to the server in order to get invalidation message
         set res2 [lindex [split [$rd read] " "] 1] 
-        $rd read ; # Consume the PING reply, which comes together with the invalidation message
+        assert_equal PONG [$rd read] ; # Consume the PING reply, which comes together with the invalidation message
         assert {$res1 eq {key1}}
         assert {$res2 eq {key1}}
     }
@@ -292,35 +301,36 @@ start_server {tags {"tracking network"}} {
 
     test {Able to redirect to a RESP3 client} {
         $rd_redirection UNSUBSCRIBE __redis__:invalidate ; # Need to unsub first before we can do HELLO 3
-        $rd_redirection read ; # Consume the UNSUBSCRIBE reply
+        set res [$rd_redirection read] ; # Consume the UNSUBSCRIBE reply
+        assert_equal {__redis__:invalidate} [lindex $res 1]
         $rd_redirection HELLO 3
-        $rd_redirection read ; # Consume the HELLO reply
+        set res [$rd_redirection read] ; # Consume the HELLO reply
+        assert_equal [dict get $reply proto] 3
         $rd_redirection SUBSCRIBE __redis__:invalidate
-        $rd_redirection read ; # Consume the SUBSCRIBE reply
+        set res [$rd_redirection read] ; # Consume the SUBSCRIBE reply
+        assert_equal {__redis__:invalidate} [lindex $res 1]
         r CLIENT TRACKING on REDIRECT $redir_id
         $rd_sg SET key1 1
         r GET key1
         $rd_sg INCR key1
         set res [lindex [$rd_redirection read] 1]
         assert {$res eq {key1}}
+        $rd_redirection HELLO 2
+        set res [$rd_redirection read] ; # Consume the HELLO reply
+        assert_equal [dict get $res proto] 2
     }
 
     test {After switching from normal tracking to BCAST mode, no invalidation message is produced for pre-BCAST keys} {
-        $rd HELLO 3
-        $rd read ; # Consume the HELLO reply
-        $rd CLIENT TRACKING on
-        $rd read ; # Consume the TRACKING reply
+        r CLIENT TRACKING off
+        r HELLO 3
+        r CLIENT TRACKING on
         $rd_sg SET key1 1
-        $rd GET key1
-        $rd read ; # Consume the GET reply
-        $rd CLIENT TRACKING off 
-        $rd read ; # Consume the TRACKING reply
-        $rd CLIENT TRACKING on BCAST
-        $rd read ; # Consume the TRACKING reply
+        r GET key1
+        r CLIENT TRACKING off 
+        r CLIENT TRACKING on BCAST
         $rd_sg INCR key1
-        $rd PING
-        set inv_msg [$rd read]
-        set ping_reply [$rd read]
+        set inv_msg [r PING]
+        set ping_reply [r read]
         assert {$inv_msg eq {invalidate key1}}
         assert {$ping_reply eq {PONG}}
     }
@@ -344,7 +354,6 @@ start_server {tags {"tracking network"}} {
     }
 
     test {Tracking gets notification on tracking table key eviction} {
-        $rd_redirection HELLO 2
         r CLIENT TRACKING off
         r CLIENT TRACKING on REDIRECT $redir_id NOLOOP
         r MSET key1 1 key2 2
@@ -368,7 +377,6 @@ start_server {tags {"tracking network"}} {
 
     test {Invalidation message received for flushall} {
         clean_all
-        r CLIENT TRACKING off
         r CLIENT TRACKING on REDIRECT $redir_id
         $rd_sg SET key1 1
         r GET key1
@@ -379,7 +387,6 @@ start_server {tags {"tracking network"}} {
 
     test {Invalidation message received for flushdb} {
         clean_all
-        r CLIENT TRACKING off
         r CLIENT TRACKING on REDIRECT $redir_id
         $rd_sg SET key1 1
         r GET key1
@@ -422,15 +429,29 @@ start_server {tags {"tracking network"}} {
         r GET key1 
         r GET key2
         $rd CLIENT TRACKING on BCAST PREFIX prefix:
+        assert [string match *OK* [$rd read]]
         $rd_sg SET prefix:key1 1 
         $rd_sg SET prefix:key2 2
         set info [r info]
         regexp "\r\ntracking_total_items:(.*?)\r\n" $info _ total_items
         regexp "\r\ntracking_total_keys:(.*?)\r\n" $info _ total_keys
         regexp "\r\ntracking_total_prefixes:(.*?)\r\n" $info _ total_prefixes
+        regexp "\r\ntracking_clients:(.*?)\r\n" $info _ tracking_clients
         assert {$total_items == 2}
         assert {$total_keys == 2}
         assert {$total_prefixes == 1}
+        assert {$tracking_clients == 2}
+    }
+
+    test {CLIENT GETREDIR provides correct client id} {
+        set res [r CLIENT GETREDIR]
+        assert_equal $redir_id $res
+        r CLIENT TRACKING off
+        set res [r CLIENT GETREDIR]
+        assert_equal -1 $res
+        r CLIENT TRACKING on
+        set res [r CLIENT GETREDIR]
+        assert_equal 0 $res
     }
 
     test {CLIENT TRACKINGINFO provides reasonable results when tracking off} {
@@ -510,6 +531,32 @@ start_server {tags {"tracking network"}} {
         assert_equal {0} $redirect
         set prefixes [lsort [dict get $res prefixes]]
         assert_equal {bar foo} $prefixes
+
+        r CLIENT TRACKING off
+        r CLIENT TRACKING on BCAST
+        set res [r client trackinginfo]
+        set prefixes [dict get $res prefixes]
+        assert_equal {{}} $prefixes
+    }
+
+    test {CLIENT TRACKINGINFO provides reasonable results when tracking redir broken} {
+        clean_all
+        r HELLO 3
+        r CLIENT TRACKING on REDIRECT $redir_id
+        $rd_sg SET key1 1
+        r GET key1
+        $rd_redirection QUIT
+        assert_equal OK [$rd_redirection read]
+        $rd_sg SET key1 2
+        set res [lsearch -exact [r read] "tracking-redir-broken"]
+        assert {$res >= 0}
+        set res [r client trackinginfo]
+        set flags [dict get $res flags]
+        assert_equal {on broken_redirect} $flags
+        set redirect [dict get $res redirect]
+        assert_equal $redir_id $redirect
+        set prefixes [dict get $res prefixes]
+        assert_equal {} $prefixes
     }
 
     $rd_redirection close

From d828f90c26bf796cb54d6622389e5c14fcc9cbf0 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Sun, 21 Feb 2021 11:22:36 +0200
Subject: [PATCH 1067/1098] Fix allowed length for REPLCONF ip-address. (#8517)

Originally this was limited to IPv6 address length, but effectively it
has been used for host names and now that Sentinel accepts that as well
we need to be able to store full hostnames.

Fixes #8507
---
 src/networking.c  |  3 ++-
 src/replication.c | 41 ++++++++++++++++++++---------------------
 src/server.c      |  4 ++--
 src/server.h      |  4 +++-
 4 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/src/networking.c b/src/networking.c
index 5a465c460..50e4b71bc 100644
--- a/src/networking.c
+++ b/src/networking.c
@@ -155,7 +155,7 @@ client *createClient(connection *conn) {
     c->repl_ack_off = 0;
     c->repl_ack_time = 0;
     c->slave_listening_port = 0;
-    c->slave_ip[0] = '\0';
+    c->slave_addr = NULL;
     c->slave_capa = SLAVE_CAPA_NONE;
     c->reply = listCreate();
     c->reply_bytes = 0;
@@ -1400,6 +1400,7 @@ void freeClient(client *c) {
     freeClientMultiState(c);
     sdsfree(c->peerid);
     sdsfree(c->sockname);
+    sdsfree(c->slave_addr);
     zfree(c);
 }
 
diff --git a/src/replication.c b/src/replication.c
index 90fa6c782..eb5fa54c0 100644
--- a/src/replication.c
+++ b/src/replication.c
@@ -57,21 +57,19 @@ int RDBGeneratedByReplication = 0;
  * IP address and its listening port which is more clear for the user, for
  * example: "Closing connection with replica 10.1.2.3:6380". */
 char *replicationGetSlaveName(client *c) {
-    static char buf[NET_ADDR_STR_LEN];
+    static char buf[NET_HOST_PORT_STR_LEN];
     char ip[NET_IP_STR_LEN];
 
     ip[0] = '\0';
     buf[0] = '\0';
-    if (c->slave_ip[0] != '\0' ||
+    if (c->slave_addr ||
         connPeerToString(c->conn,ip,sizeof(ip),NULL) != -1)
     {
-        /* Note that the 'ip' buffer is always larger than 'c->slave_ip' */
-        if (c->slave_ip[0] != '\0') memcpy(ip,c->slave_ip,sizeof(c->slave_ip));
-
+        char *addr = c->slave_addr ? c->slave_addr : ip;
         if (c->slave_listening_port)
-            anetFormatAddr(buf,sizeof(buf),ip,c->slave_listening_port);
+            anetFormatAddr(buf,sizeof(buf),addr,c->slave_listening_port);
         else
-            snprintf(buf,sizeof(buf),"%s:<unknown-replica-port>",ip);
+            snprintf(buf,sizeof(buf),"%s:<unknown-replica-port>",addr);
     } else {
         snprintf(buf,sizeof(buf),"client id #%llu",
             (unsigned long long) c->id);
@@ -925,12 +923,13 @@ void replconfCommand(client *c) {
                 return;
             c->slave_listening_port = port;
         } else if (!strcasecmp(c->argv[j]->ptr,"ip-address")) {
-            sds ip = c->argv[j+1]->ptr;
-            if (sdslen(ip) < sizeof(c->slave_ip)) {
-                memcpy(c->slave_ip,ip,sdslen(ip)+1);
+            sds addr = c->argv[j+1]->ptr;
+            if (sdslen(addr) < NET_HOST_STR_LEN) {
+                if (c->slave_addr) sdsfree(c->slave_addr);
+                c->slave_addr = sdsdup(addr);
             } else {
                 addReplyErrorFormat(c,"REPLCONF ip-address provided by "
-                    "replica instance is too long: %zd bytes", sdslen(ip));
+                    "replica instance is too long: %zd bytes", sdslen(addr));
                 return;
             }
         } else if (!strcasecmp(c->argv[j]->ptr,"capa")) {
@@ -2797,16 +2796,16 @@ void roleCommand(client *c) {
         listRewind(server.slaves,&li);
         while((ln = listNext(&li))) {
             client *slave = ln->value;
-            char ip[NET_IP_STR_LEN], *slaveip = slave->slave_ip;
+            char ip[NET_IP_STR_LEN], *slaveaddr = slave->slave_addr;
 
-            if (slaveip[0] == '\0') {
+            if (!slaveaddr) {
                 if (connPeerToString(slave->conn,ip,sizeof(ip),NULL) == -1)
                     continue;
-                slaveip = ip;
+                slaveaddr = ip;
             }
             if (slave->replstate != SLAVE_STATE_ONLINE) continue;
             addReplyArrayLen(c,3);
-            addReplyBulkCString(c,slaveip);
+            addReplyBulkCString(c,slaveaddr);
             addReplyBulkLongLong(c,slave->slave_listening_port);
             addReplyBulkLongLong(c,slave->repl_ack_off);
             slaves++;
@@ -3492,9 +3491,9 @@ static client *findReplica(char *host, int port) {
     listRewind(server.slaves,&li);
     while((ln = listNext(&li))) {
         replica = ln->value;
-        char ip[NET_IP_STR_LEN], *replicaip = replica->slave_ip;
+        char ip[NET_IP_STR_LEN], *replicaip = replica->slave_addr;
 
-        if (replicaip[0] == '\0') {
+        if (!replicaip) {
             if (connPeerToString(replica->conn, ip, sizeof(ip), NULL) == -1)
                 continue;
             replicaip = ip;
@@ -3723,16 +3722,16 @@ void updateFailoverStatus(void) {
         while((ln = listNext(&li))) {
             replica = ln->value;
             if (replica->repl_ack_off == server.master_repl_offset) {
-                char ip[NET_IP_STR_LEN], *replicaip = replica->slave_ip;
+                char ip[NET_IP_STR_LEN], *replicaaddr = replica->slave_addr;
 
-                if (replicaip[0] == '\0') {
+                if (!replicaaddr) {
                     if (connPeerToString(replica->conn,ip,sizeof(ip),NULL) == -1)
                         continue;
-                    replicaip = ip;
+                    replicaaddr = ip;
                 }
 
                 /* We are now failing over to this specific node */
-                server.target_replica_host = zstrdup(replicaip);
+                server.target_replica_host = zstrdup(replicaaddr);
                 server.target_replica_port = replica->slave_listening_port;
                 break;
             }
diff --git a/src/server.c b/src/server.c
index 17a25168f..b22af980b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -5062,11 +5062,11 @@ sds genRedisInfoString(const char *section) {
             while((ln = listNext(&li))) {
                 client *slave = listNodeValue(ln);
                 char *state = NULL;
-                char ip[NET_IP_STR_LEN], *slaveip = slave->slave_ip;
+                char ip[NET_IP_STR_LEN], *slaveip = slave->slave_addr;
                 int port;
                 long lag = 0;
 
-                if (slaveip[0] == '\0') {
+                if (!slaveip) {
                     if (connPeerToString(slave->conn,ip,sizeof(ip),&port) == -1)
                         continue;
                     slaveip = ip;
diff --git a/src/server.h b/src/server.h
index 8f7e70261..55f718111 100644
--- a/src/server.h
+++ b/src/server.h
@@ -111,8 +111,10 @@ typedef long long ustime_t; /* microsecond time type. */
 #define CONFIG_DEFAULT_CLUSTER_CONFIG_FILE "nodes.conf"
 #define CONFIG_DEFAULT_UNIX_SOCKET_PERM 0
 #define CONFIG_DEFAULT_LOGFILE ""
+#define NET_HOST_STR_LEN 256 /* Longest valid hostname */
 #define NET_IP_STR_LEN 46 /* INET6_ADDRSTRLEN is 46, but we need to be sure */
 #define NET_ADDR_STR_LEN (NET_IP_STR_LEN+32) /* Must be enough for ip:port */
+#define NET_HOST_PORT_STR_LEN (NET_HOST_STR_LEN+32) /* Must be enough for hostname:port */
 #define CONFIG_BINDADDR_MAX 16
 #define CONFIG_MIN_RESERVED_FDS 32
 #define CONFIG_DEFAULT_PROC_TITLE_TEMPLATE "{title} {listen-addr} {server-mode}"
@@ -901,7 +903,7 @@ typedef struct client {
                                        should use. */
     char replid[CONFIG_RUN_ID_SIZE+1]; /* Master replication ID (if master). */
     int slave_listening_port; /* As configured with: REPLCONF listening-port */
-    char slave_ip[NET_IP_STR_LEN]; /* Optionally given by REPLCONF ip-address */
+    char *slave_addr;       /* Optionally given by REPLCONF ip-address */
     int slave_capa;         /* Slave capabilities: SLAVE_CAPA_* bitwise OR. */
     multiState mstate;      /* MULTI/EXEC state */
     int btype;              /* Type of blocking op if CLIENT_BLOCKED. */

From 362f2a84bd67a1b65d810b064163e8432c79138e Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Mon, 22 Feb 2021 14:45:26 +0800
Subject: [PATCH 1068/1098] Improve overflow check of expire time (#8519)

When milliseconds == LLONG_MAX / 1000, *1000 will not overflow.
---
 src/t_string.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/t_string.c b/src/t_string.c
index cdf69110f..3f73363e0 100644
--- a/src/t_string.c
+++ b/src/t_string.c
@@ -78,7 +78,7 @@ void setGenericCommand(client *c, int flags, robj *key, robj *val, robj *expire,
     if (expire) {
         if (getLongLongFromObjectOrReply(c, expire, &milliseconds, NULL) != C_OK)
             return;
-        if (milliseconds <= 0 || (unit == UNIT_SECONDS && milliseconds >= LLONG_MAX / 1000)) {
+        if (milliseconds <= 0 || (unit == UNIT_SECONDS && milliseconds > LLONG_MAX / 1000)) {
             /* Negative value provided or multiplication is gonna overflow. */
             addReplyErrorFormat(c, "invalid expire time in %s", c->cmd->name);
             return;
@@ -344,7 +344,7 @@ void getexCommand(client *c) {
     if (expire) {
         if (getLongLongFromObjectOrReply(c, expire, &milliseconds, NULL) != C_OK)
             return;
-        if (milliseconds <= 0 || (unit == UNIT_SECONDS && milliseconds >= LLONG_MAX / 1000)) {
+        if (milliseconds <= 0 || (unit == UNIT_SECONDS && milliseconds > LLONG_MAX / 1000)) {
             /* Negative value provided or multiplication is gonna overflow. */
             addReplyErrorFormat(c, "invalid expire time in %s", c->cmd->name);
             return;

From f5235b2d7660b163c739325ec9284f97d5b963be Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Mon, 22 Feb 2021 08:00:59 -0500
Subject: [PATCH 1069/1098] SRANDMEMBER RESP3 return should be Array, not Set
 (#8504)

SRANDMEMBER with negative count (non unique) can return the same member
multiple times, and the order of elements in the returned collection matters.
For these reasons returning a RESP3 Set type is not valid for the negative
count, but also not really valid for the positive (unique) variant either (the
command returns an array of random picks, not a set)

This PR also contains a minor optimization for SRANDMEMBER, HRANDFIELD,
and ZRANDMEMBER, to avoid the temporary dict from being rehashed while it grows.

Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/t_hash.c |  2 ++
 src/t_set.c  | 24 +++++++++++++++++++-----
 src/t_zset.c |  2 ++
 3 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/src/t_hash.c b/src/t_hash.c
index 15b57c1d7..eb980578c 100644
--- a/src/t_hash.c
+++ b/src/t_hash.c
@@ -1078,6 +1078,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
      * used into CASE 4 is highly inefficient. */
     if (count*HRANDFIELD_SUB_STRATEGY_MUL > size) {
         dict *d = dictCreate(&sdsReplyDictType, NULL);
+        dictExpand(d, size);
         hashTypeIterator *hi = hashTypeInitIterator(hash);
 
         /* Add all the elements into the temporary dictionary. */
@@ -1147,6 +1148,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
         unsigned long added = 0;
         ziplistEntry key, value;
         dict *d = dictCreate(&hashDictType, NULL);
+        dictExpand(d, count);
         while(added < count) {
             hashTypeRandomElement(hash, size, &key, withvalues? &value : NULL);
 
diff --git a/src/t_set.c b/src/t_set.c
index f7a05206d..b655b716d 100644
--- a/src/t_set.c
+++ b/src/t_set.c
@@ -674,13 +674,13 @@ void srandmemberWithCountCommand(client *c) {
         uniq = 0;
     }
 
-    if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.emptyset[c->resp]))
+    if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.emptyarray))
         == NULL || checkType(c,set,OBJ_SET)) return;
     size = setTypeSize(set);
 
     /* If count is zero, serve it ASAP to avoid special cases later. */
     if (count == 0) {
-        addReply(c,shared.emptyset[c->resp]);
+        addReply(c,shared.emptyarray);
         return;
     }
 
@@ -690,7 +690,7 @@ void srandmemberWithCountCommand(client *c) {
      * structures. This case is the only one that also needs to return the
      * elements in random order. */
     if (!uniq || count == 1) {
-        addReplySetLen(c,count);
+        addReplyArrayLen(c,count);
         while(count--) {
             encoding = setTypeRandomElement(set,&ele,&llele);
             if (encoding == OBJ_ENCODING_INTSET) {
@@ -706,7 +706,19 @@ void srandmemberWithCountCommand(client *c) {
      * The number of requested elements is greater than the number of
      * elements inside the set: simply return the whole set. */
     if (count >= size) {
-        sunionDiffGenericCommand(c,c->argv+1,1,NULL,SET_OP_UNION);
+        setTypeIterator *si;
+        addReplyArrayLen(c,size);
+        si = setTypeInitIterator(set);
+        while ((encoding = setTypeNext(si,&ele,&llele)) != -1) {
+            if (encoding == OBJ_ENCODING_INTSET) {
+                addReplyBulkLongLong(c,llele);
+            } else {
+                addReplyBulkCBuffer(c,ele,sdslen(ele));
+            }
+            size--;
+        }
+        setTypeReleaseIterator(si);
+        serverAssert(size==0);
         return;
     }
 
@@ -727,6 +739,7 @@ void srandmemberWithCountCommand(client *c) {
 
         /* Add all the elements into the temporary dictionary. */
         si = setTypeInitIterator(set);
+        dictExpand(d, size);
         while ((encoding = setTypeNext(si,&ele,&llele)) != -1) {
             int retval = DICT_ERR;
 
@@ -759,6 +772,7 @@ void srandmemberWithCountCommand(client *c) {
         unsigned long added = 0;
         sds sdsele;
 
+        dictExpand(d, count);
         while (added < count) {
             encoding = setTypeRandomElement(set,&ele,&llele);
             if (encoding == OBJ_ENCODING_INTSET) {
@@ -781,7 +795,7 @@ void srandmemberWithCountCommand(client *c) {
         dictIterator *di;
         dictEntry *de;
 
-        addReplySetLen(c,count);
+        addReplyArrayLen(c,count);
         di = dictGetIterator(d);
         while((de = dictNext(di)) != NULL)
             addReplyBulkSds(c,dictGetKey(de));
diff --git a/src/t_zset.c b/src/t_zset.c
index 869cf6c9a..6df21e300 100644
--- a/src/t_zset.c
+++ b/src/t_zset.c
@@ -4085,6 +4085,7 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) {
      * used into CASE 4 is highly inefficient. */
     if (count*ZRANDMEMBER_SUB_STRATEGY_MUL > size) {
         dict *d = dictCreate(&sdsReplyDictType, NULL);
+        dictExpand(d, size);
         /* Add all the elements into the temporary dictionary. */
         while (zuiNext(&src, &zval)) {
             sds key = zuiNewSdsFromValue(&zval);
@@ -4143,6 +4144,7 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) {
         /* Hashtable encoding (generic implementation) */
         unsigned long added = 0;
         dict *d = dictCreate(&hashDictType, NULL);
+        dictExpand(d, count);
 
         while (added < count) {
             ziplistEntry key;

From b164c4dec32a71e74c7ca510a7ab8c6423d30eb4 Mon Sep 17 00:00:00 2001
From: Huang Zw <huang_zhw@126.com>
Date: Mon, 22 Feb 2021 21:08:16 +0800
Subject: [PATCH 1070/1098] In XADD take deleted items into consideration when
 switch to new node (#8390)

If we set stream-node-max-bytes = 0, then we insert entry then delete,
do this many times, the last stream node will be very big.
---
 src/t_stream.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index da43fce18..d9db97c3c 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -512,7 +512,9 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_
         {
             lp = NULL;
         } else if (server.stream_node_max_entries) {
-            int64_t count = lpGetInteger(lpFirst(lp));
+            unsigned char *lp_ele = lpFirst(lp);
+            /* Count both live entries and deleted ones. */
+            int64_t count = lpGetInteger(lp_ele) + lpGetInteger(lpNext(lp,lp_ele));
             if (count >= server.stream_node_max_entries) {
                 /* Shrink extra pre-allocated memory */
                 lp = lpShrinkToFit(lp);

From 4739131ca6c966592dce739b9330a989eb6b6b2e Mon Sep 17 00:00:00 2001
From: Harkrishn Patro <30795839+hpatro@users.noreply.github.com>
Date: Mon, 22 Feb 2021 14:22:25 +0100
Subject: [PATCH 1071/1098] Remove acl subcommand validation if fully added
 command exists. (#8483)

This validation was only done for sub-commands and not for commands.

These would have been valid (not produce any error)
ACL SETUSER bob +@all +client
ACL SETUSER bob +client +client

so no reason for this one to fail:
ACL SETUSER bob +client +client|id

One example why this is needed is that pfdebug wasn't part of the @hyperloglog
group and now it is. so something like:
acl setuser user1 +@hyperloglog +pfdebug|test
would have succeeded in early 6.0.x, and fail in 6.2 RC3

Co-authored-by: Harkrishn Patro <harkrisp@amazon.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
---
 src/acl.c          | 22 +++-------------------
 tests/unit/acl.tcl | 19 +++++++++++++++++++
 2 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index 74210ebce..e120b36fc 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -814,8 +814,6 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) {
  *         invalid (contains non allowed characters).
  * ENOENT: The command name or command category provided with + or - is not
  *         known.
- * EBUSY:  The subcommand you want to add is about a command that is currently
- *         fully added.
  * EEXIST: You are adding a key pattern after "*" was already added. This is
  *         almost surely an error on the user side.
  * EISDIR: You are adding a channel pattern after "*" was already added. This is
@@ -976,22 +974,12 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
                 return C_ERR;
             }
 
-            /* The command should not be set right now in the command
-             * bitmap, because adding a subcommand of a fully added
-             * command is probably an error on the user side. */
             unsigned long id = ACLGetCommandID(copy);
-            if (ACLGetUserCommandBit(u,id) == 1) {
-                zfree(copy);
-                errno = EBUSY;
-                return C_ERR;
+            /* Add the subcommand to the list of valid ones, if the command is not set. */
+            if (ACLGetUserCommandBit(u,id) == 0) {
+                ACLAddAllowedSubcommand(u,id,sub);
             }
 
-            /* Add the subcommand to the list of valid ones. */
-            ACLAddAllowedSubcommand(u,id,sub);
-
-            /* We have to clear the command bit so that we force the
-             * subcommand check. */
-            ACLSetUserCommandBit(u,id,0);
             zfree(copy);
         }
     } else if (op[0] == '-' && op[1] != '@') {
@@ -1030,10 +1018,6 @@ const char *ACLSetUserStringError(void) {
         errmsg = "Unknown command or category name in ACL";
     else if (errno == EINVAL)
         errmsg = "Syntax error";
-    else if (errno == EBUSY)
-        errmsg = "Adding a subcommand of a command already fully "
-                 "added is not allowed. Remove the command to start. "
-                 "Example: -DEBUG +DEBUG|DIGEST";
     else if (errno == EEXIST)
         errmsg = "Adding a pattern after the * pattern (or the "
                  "'allkeys' flag) is not valid and does not have any "
diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl
index f5e3259fe..7c09195a1 100644
--- a/tests/unit/acl.tcl
+++ b/tests/unit/acl.tcl
@@ -217,6 +217,25 @@ start_server {tags {"acl"}} {
         set e
     } {*NOPERM*}
 
+    test {ACLs set can include subcommands, if already full command exists} {
+        r ACL setuser bob +memory|doctor
+        set cmdstr [dict get [r ACL getuser bob] commands]
+        assert_equal {-@all +memory|doctor} $cmdstr
+
+        # Validate the commands have got engulfed to +memory.
+        r ACL setuser bob +memory
+        set cmdstr [dict get [r ACL getuser bob] commands]
+        assert_equal {-@all +memory} $cmdstr
+
+        # Appending to the existing access string of bob.
+        r ACL setuser bob +@all +client|id
+        # Validate the new commands has got engulfed to +@all.
+        set cmdstr [dict get [r ACL getuser bob] commands]
+        assert_equal {+@all} $cmdstr
+        r CLIENT ID; # Should not fail
+        r MEMORY DOCTOR; # Should not fail
+    }
+
     # Note that the order of the generated ACL rules is not stable in Redis
     # so we need to match the different parts and not as a whole string.
     test {ACL GETUSER is able to translate back command permissions} {

From d32f2e9999ce003bad0bd2c3bca29f64dcce4433 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Mon, 22 Feb 2021 15:41:32 +0200
Subject: [PATCH 1072/1098] Fix integer overflow (CVE-2021-21309). (#8522)

On 32-bit systems, setting the proto-max-bulk-len config parameter to a high value may result with integer overflow and a subsequent heap overflow when parsing an input bulk (CVE-2021-21309).

This fix has two parts:

Set a reasonable limit to the config parameter.
Add additional checks to prevent the problem in other potential but unknown code paths.
---
 src/config.c  |  2 +-
 src/sds.c     |  3 +++
 src/zmalloc.c | 10 ++++++++++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/config.c b/src/config.c
index 0bd89c2b9..9081d0312 100644
--- a/src/config.c
+++ b/src/config.c
@@ -2517,7 +2517,7 @@ standardConfig configs[] = {
     createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, 15000, INTEGER_CONFIG, NULL, NULL),
     createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, 10000, INTEGER_CONFIG, NULL, NULL),
     createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, 0, INTEGER_CONFIG, NULL, NULL),
-    createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 1024*1024, LLONG_MAX, server.proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */
+    createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 1024*1024, LONG_MAX, server.proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */
     createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL),
     createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */
 
diff --git a/src/sds.c b/src/sds.c
index ad30e2ad4..6385ab14b 100644
--- a/src/sds.c
+++ b/src/sds.c
@@ -111,6 +111,7 @@ sds _sdsnewlen(const void *init, size_t initlen, int trymalloc) {
     unsigned char *fp; /* flags pointer. */
     size_t usable;
 
+    assert(initlen + hdrlen + 1 > initlen); /* Catch size_t overflow */
     sh = trymalloc?
         s_trymalloc_usable(hdrlen+initlen+1, &usable) :
         s_malloc_usable(hdrlen+initlen+1, &usable);
@@ -243,6 +244,7 @@ sds sdsMakeRoomFor(sds s, size_t addlen) {
     len = sdslen(s);
     sh = (char*)s-sdsHdrSize(oldtype);
     newlen = (len+addlen);
+    assert(newlen > len);   /* Catch size_t overflow */
     if (newlen < SDS_MAX_PREALLOC)
         newlen *= 2;
     else
@@ -256,6 +258,7 @@ sds sdsMakeRoomFor(sds s, size_t addlen) {
     if (type == SDS_TYPE_5) type = SDS_TYPE_8;
 
     hdrlen = sdsHdrSize(type);
+    assert(hdrlen + newlen + 1 > len);  /* Catch size_t overflow */
     if (oldtype==type) {
         newsh = s_realloc_usable(sh, hdrlen+newlen+1, &usable);
         if (newsh == NULL) return NULL;
diff --git a/src/zmalloc.c b/src/zmalloc.c
index eacce67bd..c8d6c825f 100644
--- a/src/zmalloc.c
+++ b/src/zmalloc.c
@@ -57,6 +57,12 @@ void zlibc_free(void *ptr) {
 #endif
 #endif
 
+#if PREFIX_SIZE > 0
+#define ASSERT_NO_SIZE_OVERFLOW(sz) assert((sz) + PREFIX_SIZE > (sz))
+#else
+#define ASSERT_NO_SIZE_OVERFLOW(sz)
+#endif
+
 /* Explicitly override malloc/free etc when using tcmalloc. */
 #if defined(USE_TCMALLOC)
 #define malloc(size) tc_malloc(size)
@@ -89,6 +95,7 @@ static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom;
 /* Try allocating memory, and return NULL if failed.
  * '*usable' is set to the usable size if non NULL. */
 void *ztrymalloc_usable(size_t size, size_t *usable) {
+    ASSERT_NO_SIZE_OVERFLOW(size);
     void *ptr = malloc(size+PREFIX_SIZE);
 
     if (!ptr) return NULL;
@@ -131,6 +138,7 @@ void *zmalloc_usable(size_t size, size_t *usable) {
  * Currently implemented only for jemalloc. Used for online defragmentation. */
 #ifdef HAVE_DEFRAG
 void *zmalloc_no_tcache(size_t size) {
+    ASSERT_NO_SIZE_OVERFLOW(size);
     void *ptr = mallocx(size+PREFIX_SIZE, MALLOCX_TCACHE_NONE);
     if (!ptr) zmalloc_oom_handler(size);
     update_zmalloc_stat_alloc(zmalloc_size(ptr));
@@ -147,6 +155,7 @@ void zfree_no_tcache(void *ptr) {
 /* Try allocating memory and zero it, and return NULL if failed.
  * '*usable' is set to the usable size if non NULL. */
 void *ztrycalloc_usable(size_t size, size_t *usable) {
+    ASSERT_NO_SIZE_OVERFLOW(size);
     void *ptr = calloc(1, size+PREFIX_SIZE);
     if (ptr == NULL) return NULL;
 
@@ -187,6 +196,7 @@ void *zcalloc_usable(size_t size, size_t *usable) {
 /* Try reallocating memory, and return NULL if failed.
  * '*usable' is set to the usable size if non NULL. */
 void *ztryrealloc_usable(void *ptr, size_t size, size_t *usable) {
+    ASSERT_NO_SIZE_OVERFLOW(size);
 #ifndef HAVE_MALLOC_SIZE
     void *realptr;
 #endif

From 8e83bcd2acb18370e2d6cea3718339792322e80f Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 22 Feb 2021 15:48:17 +0200
Subject: [PATCH 1073/1098] update help.h from redis.io commands.json (#8524)

---
 src/help.h | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/help.h b/src/help.h
index b8b1efb95..97de00231 100644
--- a/src/help.h
+++ b/src/help.h
@@ -458,6 +458,11 @@ struct commandHelp {
     "Set the expiration for a key as a UNIX timestamp",
     0,
     "1.2.0" },
+    { "FAILOVER",
+    "[TO host port [FORCE]] [ABORT] [TIMEOUT milliseconds]",
+    "Start a coordinated failover between this server and one of its replicas.",
+    9,
+    "6.2.0" },
     { "FLUSHALL",
     "[ASYNC|SYNC]",
     "Remove all keys from all databases",

From f098fe319b483314c0db0937cf4fb1ab9488bfe7 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 22 Feb 2021 15:56:55 +0200
Subject: [PATCH 1074/1098] Redis 6.2.0 GA

---
 00-RELEASENOTES | 76 +++++++++++++++++++++++++++++++++++++++++++++++--
 src/version.h   |  4 +--
 2 files changed, 75 insertions(+), 5 deletions(-)

diff --git a/00-RELEASENOTES b/00-RELEASENOTES
index f52a22ed9..36ef7156f 100644
--- a/00-RELEASENOTES
+++ b/00-RELEASENOTES
@@ -1,3 +1,71 @@
+Redis 6.2 release notes
+=======================
+
+--------------------------------------------------------------------------------
+Upgrade urgency levels:
+
+LOW:      No need to upgrade unless there are new features you want to use.
+MODERATE: Program an upgrade of the server, but it's not urgent.
+HIGH:     There is a critical bug that may affect a subset of users. Upgrade!
+CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
+SECURITY: There are security fixes in the release.
+--------------------------------------------------------------------------------
+
+================================================================================
+Redis 6.2.0 GA  Released Tue Feb 22 14:00:00 IST 2021
+================================================================================
+
+Upgrade urgency: SECURITY if you use 32bit build of redis (see bellow), MODERATE
+if you used earlier versions of Redis 6.2, LOW otherwise.
+
+Integer overflow on 32-bit systems (CVE-2021-21309):
+Redis 4.0 or newer uses a configurable limit for the maximum supported bulk
+input size. By default, it is 512MB which is a safe value for all platforms.
+If the limit is significantly increased, receiving a large request from a client
+may trigger several integer overflow scenarios, which would result with buffer
+overflow and heap corruption.
+
+Here is a comprehensive list of changes in this release compared to 6.2 RC3,
+each one includes the PR number that added it, so you can get more details
+at https://github.com/redis/redis/pull/<number>
+
+Bug fixes:
+* Avoid 32-bit overflows when proto-max-bulk-len is set high (#8522)
+* Fix broken protocol in client tracking tracking-redir-broken message (#8456)
+* Avoid unsafe field name characters in INFO commandstats, errorstats, modules (#8492)
+* XINFO able to access expired keys during CLIENT PAUSE WRITE (#8436)
+* Fix allowed length for REPLCONF ip-address, needed due to Sentinel's support for hostnames (#8517)
+* Fix broken protocol in redis-benchmark when used with -a or --dbnum (#8486)
+* XADD counts deleted records too when considering switching to a new listpack (#8390)
+
+Bug fixes that are only applicable to previous releases of Redis 6.2:
+* Fixes in GEOSEARCH bybox (accuracy and mismatch between width and height) (#8445)
+* Fix risk of OOM panic in HRANDFIELD, ZRANDMEMBER commands with huge negative count (#8429)
+* Fix duplicate replicas issue in Sentinel, needed due to hostname support (#8481)
+* Fix Sentinel configuration rewrite, an improvement of #8271 (#8480)
+
+Command behavior changes:
+* SRANDMEMBER uses RESP3 array type instead of set type (#8504)
+* EXPIRE, EXPIREAT, SETEX, GETEX: Return error when provided expire time overflows (#8287)
+
+Other behavior changes:
+* Remove ACL subcommand validation if fully added command exists. (#8483)
+
+Improvements:
+* Optimize sorting in GEORADIUS / GEOSEARCH with COUNT (#8326)
+* Optimize HRANDFIELD and ZRANDMEMBER case 4 when ziplist encoded (#8444)
+* Optimize in-place replacement of elements in HSET, HINCRBY, LSET (#8493)
+* Remove redundant list to store pubsub patterns (#8472)
+* Add --insecure option to command line tools (#8416)
+
+Info fields and introspection changes:
+* Add INFO fields to track progress of BGSAVE, AOFRW, replication (#8414)
+
+Modules:
+* RM_ZsetRem: Delete key if empty, the bug could leave empty zset keys (#8453)
+* RM_HashSet: Add COUNT_ALL flag and set errno (#8446)
+
+================================================================================
 Redis 6.2 RC3   Released Tue Feb 1 14:00:00 IST 2021
 ================================================================================
 
@@ -305,11 +373,11 @@ and we don't get reports of serious issues for a while.
 A special thank you for the amount of work put into this release by:
 - Oran Agra
 - Yossi Gottlieb
-- Filipe Oliveira
 - Viktor Söderqvist
+- Yang Bodong
+- Filipe Oliveira
 - Guy Benoish
 - Itamar Haber
-- Yang Bodong
 - Madelyn Olson
 - Wang Yuan
 - Felipe Machado
@@ -322,15 +390,17 @@ A special thank you for the amount of work put into this release by:
 - Allen Farris
 - Chen Yang
 - Nitai Caro
+- sundb
 - Meir Shpilraien
 - maohuazhu
 - Valentino Geron
 - Zhao Zhao
-- sundb
 - Qu Chen
 - George Prekas
 - Tyson Andre
+- Uri Yagelnik
 - Michael Grunder
+- Huang Zw
 - alexronke-channeladvisor
 - Andy Pan
 - Wu Yunlong
diff --git a/src/version.h b/src/version.h
index 1c8c1f2a6..62ddcfd59 100644
--- a/src/version.h
+++ b/src/version.h
@@ -1,2 +1,2 @@
-#define REDIS_VERSION "6.1.242"
-#define REDIS_VERSION_NUM 0x000601f2
+#define REDIS_VERSION "6.2.0"
+#define REDIS_VERSION_NUM 0x00060200

From 95ea74549cc454d6d6a7462b366462589cd96712 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 23 Feb 2021 12:57:45 +0200
Subject: [PATCH 1075/1098] Fix failed tests on Linux Alpine and add a CI job.
 (#8532)

* Remove linux/version.h dependency.

This introduces unnecessary dependencies, and generally not a good idea
as the platform we build on may be different than the platform we run
on.

To determine if sync_file_range exists we can simply rely on header file
hints.

* Fix setproctitle() on libmusl.

The previous ifdef checks were a bit too strict for no apparent
reason.

* Fix tests failure on Linux with no backtrace.

* Add alpine daily CI job.
---
 .github/workflows/daily.yml   | 20 ++++++++++++++++++++
 src/config.h                  | 17 ++---------------
 src/fmacros.h                 |  1 -
 src/setproctitle.c            |  2 +-
 tests/integration/logging.tcl | 14 +++++++++++++-
 5 files changed, 36 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 9d7eb65c9..6aa535c5e 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -221,3 +221,23 @@ jobs:
           MAKE=gmake ./runtest-moduleapi --verbose &&
           ./runtest-sentinel &&
           ./runtest-cluster
+
+  test-alpine:
+    runs-on: ubuntu-latest
+    container: alpine:latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: make
+      run: |
+          apk add build-base
+          make REDIS_CFLAGS='-Werror'
+    - name: test
+      run: |
+        apk add tcl procps
+        ./runtest --accurate --verbose --dump-logs
+    - name: module api test
+      run: ./runtest-moduleapi --verbose
+    - name: sentinel tests
+      run: ./runtest-sentinel
+    - name: cluster tests
+      run: ./runtest-cluster
diff --git a/src/config.h b/src/config.h
index 1d4cb5835..56c1ab6ae 100644
--- a/src/config.h
+++ b/src/config.h
@@ -35,7 +35,6 @@
 #endif
 
 #ifdef __linux__
-#include <linux/version.h>
 #include <features.h>
 #endif
 
@@ -114,19 +113,7 @@
 
 /* Define rdb_fsync_range to sync_file_range() on Linux, otherwise we use
  * the plain fsync() call. */
-#ifdef __linux__
-#if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
-#if (LINUX_VERSION_CODE >= 0x020611 && __GLIBC_PREREQ(2, 6))
-#define HAVE_SYNC_FILE_RANGE 1
-#endif
-#else
-#if (LINUX_VERSION_CODE >= 0x020611)
-#define HAVE_SYNC_FILE_RANGE 1
-#endif
-#endif
-#endif
-
-#ifdef HAVE_SYNC_FILE_RANGE
+#if (defined(__linux__) && defined(SYNC_FILE_RANGE_WAIT_BEFORE))
 #define rdb_fsync_range(fd,off,size) sync_file_range(fd,off,size,SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE)
 #else
 #define rdb_fsync_range(fd,off,size) fsync(fd)
@@ -143,7 +130,7 @@
 #define ESOCKTNOSUPPORT 0
 #endif
 
-#if ((defined __linux && defined(__GLIBC__)) || defined __APPLE__)
+#if (defined __linux || defined __APPLE__)
 #define USE_SETPROCTITLE
 #define INIT_SETPROCTITLE_REPLACEMENT
 void spt_init(int argc, char *argv[]);
diff --git a/src/fmacros.h b/src/fmacros.h
index 089dc8de7..a97d21a47 100644
--- a/src/fmacros.h
+++ b/src/fmacros.h
@@ -60,7 +60,6 @@
 
 #ifdef __linux__
 /* features.h uses the defines above to set feature specific defines.  */
-#include <linux/version.h>
 #include <features.h>
 #endif
 
diff --git a/src/setproctitle.c b/src/setproctitle.c
index 1c91570eb..019402348 100644
--- a/src/setproctitle.c
+++ b/src/setproctitle.c
@@ -232,7 +232,7 @@ void spt_init(int argc, char *argv[]) {
 	if (!(SPT.arg0 = strdup(argv[0])))
 		goto syerr;
 
-#if __GLIBC__
+#if __linux__
 	if (!(tmp = strdup(program_invocation_name)))
 		goto syerr;
 
diff --git a/tests/integration/logging.tcl b/tests/integration/logging.tcl
index fd9034644..fec14ff2b 100644
--- a/tests/integration/logging.tcl
+++ b/tests/integration/logging.tcl
@@ -1,6 +1,18 @@
 set system_name [string tolower [exec uname -s]]
+set system_supported 0
 
-if {$system_name eq {linux} || $system_name eq {darwin}} {
+# We only support darwin or Linux with glibc
+if {$system_name eq {darwin}} {
+    set system_supported 1
+} elseif {$system_name eq {linux}} {
+    # Avoid the test on libmusl, which does not support backtrace
+    set ldd [exec ldd src/redis-server]
+    if {![string match {*libc.musl*} $ldd]} {
+        set system_supported 1
+    }
+}
+
+if {$system_supported} {
     set server_path [tmpdir server.log]
     start_server [list overrides [list dir $server_path]] {
         test "Server is able to generate a stack trace on selected systems" {

From dd885780d67f18f356a5652ab6d4f947ee035305 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Tue, 23 Feb 2021 17:08:49 +0200
Subject: [PATCH 1076/1098] Fix compile errors with no HAVE_MALLOC_SIZE.
 (#8533)

Also adds a new daily CI test, relying on the fact that we don't use malloc_size() on alpine libmusl.

Fixes #8531
---
 .github/workflows/daily.yml | 22 +++++++++++++++++++++-
 src/zmalloc.c               |  7 ++-----
 2 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 6aa535c5e..9b9d15988 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -222,7 +222,7 @@ jobs:
           ./runtest-sentinel &&
           ./runtest-cluster
 
-  test-alpine:
+  test-alpine-jemalloc:
     runs-on: ubuntu-latest
     container: alpine:latest
     steps:
@@ -241,3 +241,23 @@ jobs:
       run: ./runtest-sentinel
     - name: cluster tests
       run: ./runtest-cluster
+
+  test-alpine-libc-malloc:
+    runs-on: ubuntu-latest
+    container: alpine:latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: make
+      run: |
+          apk add build-base
+          make REDIS_CFLAGS='-Werror' USE_JEMALLOC=no
+    - name: test
+      run: |
+        apk add tcl procps
+        ./runtest --accurate --verbose --dump-logs
+    - name: module api test
+      run: ./runtest-moduleapi --verbose
+    - name: sentinel tests
+      run: ./runtest-sentinel
+    - name: cluster tests
+      run: ./runtest-cluster
diff --git a/src/zmalloc.c b/src/zmalloc.c
index c8d6c825f..fbac09616 100644
--- a/src/zmalloc.c
+++ b/src/zmalloc.c
@@ -32,6 +32,7 @@
 #include <stdlib.h>
 #include <stdint.h>
 #include <unistd.h>
+#include <assert.h>
 
 /* This function provide us access to the original libc free(). This is useful
  * for instance to free results obtained by backtrace_symbols(). We need
@@ -49,18 +50,14 @@ void zlibc_free(void *ptr) {
 
 #ifdef HAVE_MALLOC_SIZE
 #define PREFIX_SIZE (0)
+#define ASSERT_NO_SIZE_OVERFLOW(sz)
 #else
 #if defined(__sun) || defined(__sparc) || defined(__sparc__)
 #define PREFIX_SIZE (sizeof(long long))
 #else
 #define PREFIX_SIZE (sizeof(size_t))
 #endif
-#endif
-
-#if PREFIX_SIZE > 0
 #define ASSERT_NO_SIZE_OVERFLOW(sz) assert((sz) + PREFIX_SIZE > (sz))
-#else
-#define ASSERT_NO_SIZE_OVERFLOW(sz)
 #endif
 
 /* Explicitly override malloc/free etc when using tcmalloc. */

From f745c0181a827927b220caafe6ccd9a5291907f3 Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Tue, 23 Feb 2021 19:28:03 +0100
Subject: [PATCH 1077/1098] Fix race in CONFIG REWRITE sanity (#8536)

server may still be LOADING the RDB when receiving the ping
---
 tests/unit/introspection.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl
index ba28341ff..c1cf72f0c 100644
--- a/tests/unit/introspection.tcl
+++ b/tests/unit/introspection.tcl
@@ -173,7 +173,7 @@ start_server {tags {"introspection"}} {
         # Rewrite entire configuration, restart and confirm the
         # server is able to parse it and start.
         assert_equal [r debug config-rewrite-force-all] "OK"
-        restart_server 0 false false
+        restart_server 0 true false
         assert_equal [r ping] "PONG"
 
         # Verify no changes were introduced

From e3667cb908f70d3ffb04a3d5b095f91f9cc7ee97 Mon Sep 17 00:00:00 2001
From: Huang Zw <huang_zhw@126.com>
Date: Wed, 24 Feb 2021 08:55:10 +0800
Subject: [PATCH 1078/1098] In luaRedisGenericCommand check channel return is
 ACL_DENIED_CHANNEL (#8535)

not ACL_DENIED_AUTH
---
 src/scripting.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/scripting.c b/src/scripting.c
index e23d49e8b..7bf9f1c04 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -619,7 +619,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) {
                               "at least one of the keys mentioned in the "
                               "command arguments");
             break;
-        case ACL_DENIED_AUTH:
+        case ACL_DENIED_CHANNEL:
             luaPushError(lua, "The user executing the script can't publish "
                               "to the channel mentioned in the command");
             break;

From ae7d5bf6172e4597a859605268dd960c198ed935 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 24 Feb 2021 09:48:04 +0200
Subject: [PATCH 1079/1098] Use malloc_usable_size() on FreeBSD. (#8545)

---
 src/zmalloc.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/zmalloc.h b/src/zmalloc.h
index 64bc9fc76..6f3f6354a 100644
--- a/src/zmalloc.h
+++ b/src/zmalloc.h
@@ -61,9 +61,12 @@
 #define zmalloc_size(p) malloc_size(p)
 #endif
 
+/* On native libc implementations, we should still do our best to provide a
+ * HAVE_MALLOC_SIZE capability.
+ */
 #ifndef ZMALLOC_LIB
 #define ZMALLOC_LIB "libc"
-#ifdef __GLIBC__
+#if defined(__GLIBC__) || defined(__FreeBSD__)
 #include <malloc.h>
 #define HAVE_MALLOC_SIZE 1
 #define zmalloc_size(p) malloc_usable_size(p)

From d1b5767a82889cfcd9c691131512262827059793 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Wed, 24 Feb 2021 10:10:02 +0200
Subject: [PATCH 1080/1098] Cleanup clang warnings. (#8546)

---
 src/Makefile  | 5 +++++
 src/lolwut5.c | 6 +++---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/Makefile b/src/Makefile
index 3bc9f11c0..ecd69295f 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -21,7 +21,12 @@ NODEPS:=clean distclean
 
 # Default settings
 STD=-pedantic -DREDIS_STATIC=''
+
+# Use -Wno-c11-extensions on clang, either where explicitly used or on
+# platforms we can assume it's being used.
 ifneq (,$(findstring clang,$(CC)))
+  STD+=-Wno-c11-extensions
+else
 ifneq (,$(findstring FreeBSD,$(uname_S)))
   STD+=-Wno-c11-extensions
 endif
diff --git a/src/lolwut5.c b/src/lolwut5.c
index d64e0bb27..d864888ba 100644
--- a/src/lolwut5.c
+++ b/src/lolwut5.c
@@ -84,9 +84,9 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_
              * rows. */
             float angle = 0;
             if (y > 1) {
-                float r1 = (float)rand() / RAND_MAX / squares_per_col * y;
-                float r2 = (float)rand() / RAND_MAX / squares_per_col * y;
-                float r3 = (float)rand() / RAND_MAX / squares_per_col * y;
+                float r1 = (float)rand() / (float) RAND_MAX / squares_per_col * y;
+                float r2 = (float)rand() / (float) RAND_MAX / squares_per_col * y;
+                float r3 = (float)rand() / (float) RAND_MAX / squares_per_col * y;
                 if (rand() % 2) r1 = -r1;
                 if (rand() % 2) r2 = -r2;
                 if (rand() % 2) r3 = -r3;

From 965356ae63487dafeec378d1821201800b91307e Mon Sep 17 00:00:00 2001
From: Huang Zw <huang_zhw@126.com>
Date: Wed, 24 Feb 2021 19:18:54 +0800
Subject: [PATCH 1081/1098] Fix quicklistDelRange dead code, delete range wrong
 (#8257)

This commit fixes a bug in what's currently dead code in redis.

In quicklistDelRange when delete entry from entry.offset to node tail,
extent only need gte node->count - entry.offset, not node->count

Co-authored-by: Yoav Steinberg <yoav@redislabs.com>
---
 src/quicklist.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/quicklist.c b/src/quicklist.c
index c8517414c..7b7aa7839 100644
--- a/src/quicklist.c
+++ b/src/quicklist.c
@@ -999,7 +999,7 @@ int quicklistDelRange(quicklist *quicklist, const long start,
              * can just delete the entire node without ziplist math. */
             delete_entire_node = 1;
             del = node->count;
-        } else if (entry.offset >= 0 && extent >= node->count) {
+        } else if (entry.offset >= 0 && extent + entry.offset >= node->count) {
             /* If deleting more nodes after this one, calculate delete based
              * on size of current node. */
             del = node->count - entry.offset;
@@ -2238,6 +2238,17 @@ int quicklistTest(int argc, char *argv[]) {
             quicklistRelease(ql);
         }
 
+        TEST("delete less than fill but across nodes") {
+            quicklist *ql = quicklistNew(-2, options[_i]);
+            quicklistSetFill(ql, 32);
+            for (int i = 0; i < 500; i++)
+                quicklistPushTail(ql, genstr("hello", i + 1), 32);
+            ql_verify(ql, 16, 500, 32, 20);
+            quicklistDelRange(ql, 60, 10);
+            ql_verify(ql, 16, 490, 32, 20);
+            quicklistRelease(ql);
+        }
+
         TEST("delete negative 1 from 500 list") {
             quicklist *ql = quicklistNew(-2, options[_i]);
             quicklistSetFill(ql, 32);

From aba61d9ebde0c2d7803c3f395cdb175422b8d6e8 Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Wed, 24 Feb 2021 15:41:00 +0100
Subject: [PATCH 1082/1098] XTRIM should take at least 4 args (#8549)

Fix command arity.
Other than key name it must at least take the trimming strategy and it's limit
---
 src/server.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server.c b/src/server.c
index b22af980b..f4dff6b0e 100644
--- a/src/server.c
+++ b/src/server.c
@@ -1070,7 +1070,7 @@ struct redisCommand redisCommandTable[] = {
      "write fast @stream",
      0,NULL,1,1,1,0,0,0},
 
-    {"xtrim",xtrimCommand,-2,
+    {"xtrim",xtrimCommand,-4,
      "write random @stream",
      0,NULL,1,1,1,0,0,0},
 

From 21316d57090768fe2d5ff3dd5edcb328137aaaca Mon Sep 17 00:00:00 2001
From: guybe7 <guy.benoish@redislabs.com>
Date: Wed, 24 Feb 2021 15:41:50 +0100
Subject: [PATCH 1083/1098] XTRIM: Parse args before lookupKey (#8550)

This aligns better with other commands, specifically XADD
---
 src/t_stream.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index d9db97c3c..4a425fd78 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -3229,17 +3229,17 @@ cleanup:
 void xtrimCommand(client *c) {
     robj *o;
 
+    /* Argument parsing. */
+    streamAddTrimArgs parsed_args;
+    if (streamParseAddOrTrimArgsOrReply(c, &parsed_args, 1) < 0)
+        return; /* streamParseAddOrTrimArgsOrReply already replied. */
+
     /* If the key does not exist, we are ok returning zero, that is, the
      * number of elements removed from the stream. */
     if ((o = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL
         || checkType(c,o,OBJ_STREAM)) return;
     stream *s = o->ptr;
 
-    /* Argument parsing. */
-    streamAddTrimArgs parsed_args;
-    if (streamParseAddOrTrimArgsOrReply(c, &parsed_args, 1) < 0)
-        return; /* streamParseAddOrTrimArgsOrReply already replied. */
-
     /* Perform the trimming. */
     int64_t deleted = streamTrim(s, &parsed_args);
     if (deleted) {

From 60d5ef4d828bc8732ac1d4c274471655a13d667d Mon Sep 17 00:00:00 2001
From: sundb <sundbcn@gmail.com>
Date: Thu, 25 Feb 2021 00:45:13 +0800
Subject: [PATCH 1084/1098] Use addReplyErrorObject with shared.noscripterr
 (#8544)

---
 src/scripting.c     |  4 ++--
 tests/unit/info.tcl | 13 +++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/scripting.c b/src/scripting.c
index 7bf9f1c04..6830e7a70 100644
--- a/src/scripting.c
+++ b/src/scripting.c
@@ -1544,7 +1544,7 @@ void evalGenericCommand(client *c, int evalsha) {
          * return an error. */
         if (evalsha) {
             lua_pop(lua,1); /* remove the error handler from the stack. */
-            addReply(c, shared.noscripterr);
+            addReplyErrorObject(c, shared.noscripterr);
             return;
         }
         if (luaCreateFunction(c,lua,c->argv[1]) == NULL) {
@@ -1695,7 +1695,7 @@ void evalShaCommand(client *c) {
          * not the right length. So we return an error ASAP, this way
          * evalGenericCommand() can be implemented without string length
          * sanity check */
-        addReply(c, shared.noscripterr);
+        addReplyErrorObject(c, shared.noscripterr);
         return;
     }
     if (!(c->flags & CLIENT_LUA_DEBUG))
diff --git a/tests/unit/info.tcl b/tests/unit/info.tcl
index 08171fff9..0602e7147 100644
--- a/tests/unit/info.tcl
+++ b/tests/unit/info.tcl
@@ -62,6 +62,19 @@ start_server {tags {"info"}} {
             assert_equal [s total_error_replies] 2
         }
 
+        test {errorstats: failed call NOSCRIPT error} {
+            r config resetstat
+            assert_equal [s total_error_replies] 0
+            assert_match {} [errorstat NOSCRIPT]
+            catch {r evalsha NotValidShaSUM 0} e
+            assert_match {NOSCRIPT*} $e
+            assert_match {*count=1*} [errorstat NOSCRIPT]
+            assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat evalsha]
+            assert_equal [s total_error_replies] 1
+            r config resetstat
+            assert_match {} [errorstat NOSCRIPT]
+        }
+
         test {errorstats: failed call NOGROUP error} {
             r config resetstat
             assert_match {} [errorstat NOGROUP]

From c6f0ea2c81ffd5452dc6373d79786fdec8874b20 Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Wed, 24 Feb 2021 14:26:16 -0800
Subject: [PATCH 1085/1098] Allow stopped redis processes to be killed in tests
 (#8552)

---
 tests/instances.tcl      | 2 ++
 tests/support/server.tcl | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/tests/instances.tcl b/tests/instances.tcl
index 793bce80d..255d9740f 100644
--- a/tests/instances.tcl
+++ b/tests/instances.tcl
@@ -186,6 +186,8 @@ proc is_alive pid {
 
 proc stop_instance pid {
     catch {exec kill $pid}
+    # Node might have been stopped in the test
+    catch {exec kill -SIGCONT $pid}
     if {$::valgrind} {
         set max_wait 60000
     } else {
diff --git a/tests/support/server.tcl b/tests/support/server.tcl
index 4fbb99920..3ff923d7e 100644
--- a/tests/support/server.tcl
+++ b/tests/support/server.tcl
@@ -72,6 +72,8 @@ proc kill_server config {
     # kill server and wait for the process to be totally exited
     send_data_packet $::test_server_fd server-killing $pid
     catch {exec kill $pid}
+    # Node might have been stopped in the test
+    catch {exec kill -SIGCONT $pid}
     if {$::valgrind} {
         set max_wait 60000
     } else {

From 3ea4c43add4f6038830ae0a6e42821481f7ce9a0 Mon Sep 17 00:00:00 2001
From: Yossi Gottlieb <yossigo@gmail.com>
Date: Thu, 25 Feb 2021 09:24:41 +0200
Subject: [PATCH 1086/1098] Cleanup usage of malloc_usable_size. (#8554)

* Add better control of malloc_usable_size() usage.
* Use malloc_usable_size on alpine libc daily job.
* Add no-malloc-usable-size daily jobs.
* Fix zmalloc(0) when HAVE_MALLOC_SIZE is undefined.

In order to align with the jemalloc behavior, this should never return
NULL or OOM panic.
---
 .github/workflows/daily.yml | 37 ++++++++++++++++++++++++++++++++++++-
 src/zmalloc.c               | 10 ++++++++--
 src/zmalloc.h               | 11 +++++++++--
 3 files changed, 53 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
index 9b9d15988..59d236d9a 100644
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@@ -48,6 +48,25 @@ jobs:
     - name: cluster tests
       run: ./runtest-cluster
 
+  test-ubuntu-no-malloc-usable-size:
+    runs-on: ubuntu-latest
+    if: github.repository == 'redis/redis'
+    timeout-minutes: 14400
+    steps:
+    - uses: actions/checkout@v2
+    - name: make
+      run: make MALLOC=libc CFLAGS=-DNO_MALLOC_USABLE_SIZE
+    - name: test
+      run: |
+        sudo apt-get install tcl8.6
+        ./runtest --accurate --verbose --dump-logs
+    - name: module api test
+      run: ./runtest-moduleapi --verbose
+    - name: sentinel tests
+      run: ./runtest-sentinel
+    - name: cluster tests
+      run: ./runtest-cluster
+
   test-ubuntu-32bit:
     runs-on: ubuntu-latest
     if: github.repository == 'redis/redis'
@@ -132,6 +151,22 @@ jobs:
     - name: module api test
       run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1
 
+  test-valgrind-no-malloc-usable-size:
+    runs-on: ubuntu-latest
+    if: github.repository == 'redis/redis'
+    timeout-minutes: 14400
+    steps:
+    - uses: actions/checkout@v2
+    - name: make
+      run: make valgrind CFLAGS="-DNO_MALLOC_USABLE_SIZE"
+    - name: test
+      run: |
+        sudo apt-get update
+        sudo apt-get install tcl8.6 valgrind -y
+        ./runtest --valgrind --verbose --clients 1 --dump-logs
+    - name: module api test
+      run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1
+
   test-centos7-jemalloc:
     runs-on: ubuntu-latest
     if: github.repository == 'redis/redis'
@@ -250,7 +285,7 @@ jobs:
     - name: make
       run: |
           apk add build-base
-          make REDIS_CFLAGS='-Werror' USE_JEMALLOC=no
+          make REDIS_CFLAGS='-Werror' USE_JEMALLOC=no CFLAGS=-DUSE_MALLOC_USABLE_SIZE
     - name: test
       run: |
         apk add tcl procps
diff --git a/src/zmalloc.c b/src/zmalloc.c
index fbac09616..1dc662e57 100644
--- a/src/zmalloc.c
+++ b/src/zmalloc.c
@@ -60,6 +60,11 @@ void zlibc_free(void *ptr) {
 #define ASSERT_NO_SIZE_OVERFLOW(sz) assert((sz) + PREFIX_SIZE > (sz))
 #endif
 
+/* When using the libc allocator, use a minimum allocation size to match the
+ * jemalloc behavior that doesn't return NULL in this case.
+ */
+#define MALLOC_MIN_SIZE(x) ((x) > 0 ? (x) : sizeof(long))
+
 /* Explicitly override malloc/free etc when using tcmalloc. */
 #if defined(USE_TCMALLOC)
 #define malloc(size) tc_malloc(size)
@@ -93,7 +98,7 @@ static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom;
  * '*usable' is set to the usable size if non NULL. */
 void *ztrymalloc_usable(size_t size, size_t *usable) {
     ASSERT_NO_SIZE_OVERFLOW(size);
-    void *ptr = malloc(size+PREFIX_SIZE);
+    void *ptr = malloc(MALLOC_MIN_SIZE(size)+PREFIX_SIZE);
 
     if (!ptr) return NULL;
 #ifdef HAVE_MALLOC_SIZE
@@ -153,7 +158,7 @@ void zfree_no_tcache(void *ptr) {
  * '*usable' is set to the usable size if non NULL. */
 void *ztrycalloc_usable(size_t size, size_t *usable) {
     ASSERT_NO_SIZE_OVERFLOW(size);
-    void *ptr = calloc(1, size+PREFIX_SIZE);
+    void *ptr = calloc(1, MALLOC_MIN_SIZE(size)+PREFIX_SIZE);
     if (ptr == NULL) return NULL;
 
 #ifdef HAVE_MALLOC_SIZE
@@ -675,6 +680,7 @@ int zmalloc_test(int argc, char **argv) {
 
     UNUSED(argc);
     UNUSED(argv);
+    printf("Malloc prefix size: %d\n", (int) PREFIX_SIZE);
     printf("Initial used memory: %zu\n", zmalloc_used_memory());
     ptr = zmalloc(123);
     printf("Allocated 123 bytes; used: %zu\n", zmalloc_used_memory());
diff --git a/src/zmalloc.h b/src/zmalloc.h
index 6f3f6354a..d44c7b389 100644
--- a/src/zmalloc.h
+++ b/src/zmalloc.h
@@ -62,11 +62,18 @@
 #endif
 
 /* On native libc implementations, we should still do our best to provide a
- * HAVE_MALLOC_SIZE capability.
+ * HAVE_MALLOC_SIZE capability. This can be set explicitly as well:
+ *
+ * NO_MALLOC_USABLE_SIZE disables it on all platforms, even if they are
+ *      known to support it.
+ * USE_MALLOC_USABLE_SIZE forces use of malloc_usable_size() regardless
+ *      of platform.
  */
 #ifndef ZMALLOC_LIB
 #define ZMALLOC_LIB "libc"
-#if defined(__GLIBC__) || defined(__FreeBSD__)
+#if !defined(NO_MALLOC_USABLE_SIZE) && \
+    (defined(__GLIBC__) || defined(__FreeBSD__) || \
+     defined(USE_MALLOC_USABLE_SIZE))
 #include <malloc.h>
 #define HAVE_MALLOC_SIZE 1
 #define zmalloc_size(p) malloc_usable_size(p)

From 7d43159a7dab7d8e85709dfdfc74cb589db3cc70 Mon Sep 17 00:00:00 2001
From: Wen Hui <wen.hui.ware@gmail.com>
Date: Thu, 25 Feb 2021 13:08:24 -0500
Subject: [PATCH 1087/1098] redis-cli prompt: fix db number and transaction
 state inconsistencies after reconnect (#8551)

After reconnect, the prompt was showing the db ID and multi state of the previous connection.
---
 src/redis-cli.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index ef1d49e3e..3930a2d8e 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -819,6 +819,9 @@ static int cliConnect(int flags) {
     if (context == NULL || flags & CC_FORCE) {
         if (context != NULL) {
             redisFree(context);
+            config.dbnum = 0;
+            config.in_multi = 0;
+            cliRefreshPrompt();
         }
 
         if (config.hostsocket == NULL) {

From 4a474843fbd018cd323971b57dec976d7ad0278d Mon Sep 17 00:00:00 2001
From: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Date: Thu, 25 Feb 2021 21:00:27 -0800
Subject: [PATCH 1088/1098] Moved requirepass and querybuf length to generic
 configs (#8557)

Moved additional configs to generic infrastructure.
---
 src/acl.c    | 13 +++++++-
 src/config.c | 92 +++++++++-------------------------------------------
 src/server.c |  4 ++-
 src/server.h |  2 +-
 4 files changed, 31 insertions(+), 80 deletions(-)

diff --git a/src/acl.c b/src/acl.c
index e120b36fc..445409ecd 100644
--- a/src/acl.c
+++ b/src/acl.c
@@ -1054,7 +1054,6 @@ void ACLInit(void) {
     UsersToLoad = listCreate();
     ACLLog = listCreate();
     ACLInitDefaultUser();
-    server.requirepass = NULL; /* Only used for backward compatibility. */
 }
 
 /* Check the username and password pair and return C_OK if they are valid,
@@ -2251,3 +2250,15 @@ void authCommand(client *c) {
     }
 }
 
+/* Set the password for the "default" ACL user. This implements supports for
+ * requirepass config, so passing in NULL will set the user to be nopass. */
+void ACLUpdateDefaultUserPassword(sds password) {
+    ACLSetUser(DefaultUser,"resetpass",-1);
+    if (password) {
+        sds aclop = sdscatlen(sdsnew(">"), password, sdslen(password));
+        ACLSetUser(DefaultUser,aclop,sdslen(aclop));
+        sdsfree(aclop);
+    } else {
+        ACLSetUser(DefaultUser,"nopass",-1);
+    }
+}
diff --git a/src/config.c b/src/config.c
index 9081d0312..13a6dee14 100644
--- a/src/config.c
+++ b/src/config.c
@@ -504,8 +504,6 @@ void loadServerConfigFromString(char *config) {
             }
         } else if (!strcasecmp(argv[0],"include") && argc == 2) {
             loadServerConfig(argv[1], 0, NULL);
-        } else if ((!strcasecmp(argv[0],"client-query-buffer-limit")) && argc == 2) {
-             server.client_max_querybuf_len = memtoll(argv[1],NULL);
         } else if ((!strcasecmp(argv[0],"slaveof") ||
                     !strcasecmp(argv[0],"replicaof")) && argc == 3) {
             slaveof_linenum = linenum;
@@ -521,26 +519,6 @@ void loadServerConfigFromString(char *config) {
                 err = "Invalid master port"; goto loaderr;
             }
             server.repl_state = REPL_STATE_CONNECT;
-        } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) {
-            if (sdslen(argv[1]) > CONFIG_AUTHPASS_MAX_LEN) {
-                err = "Password is longer than CONFIG_AUTHPASS_MAX_LEN";
-                goto loaderr;
-            }
-            /* The old "requirepass" directive just translates to setting
-             * a password to the default user. The only thing we do
-             * additionally is to remember the cleartext password in this
-             * case, for backward compatibility with Redis <= 5. */
-            ACLSetUser(DefaultUser,"resetpass",-1);
-            sdsfree(server.requirepass);
-            server.requirepass = NULL;
-            if (sdslen(argv[1])) {
-                sds aclop = sdscatlen(sdsnew(">"), argv[1], sdslen(argv[1]));
-                ACLSetUser(DefaultUser,aclop,sdslen(aclop));
-                sdsfree(aclop);
-                server.requirepass = sdsdup(argv[1]);
-            } else {
-                ACLSetUser(DefaultUser,"nopass",-1);
-            }
         } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){
             /* DEAD OPTION */
         } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) {
@@ -750,24 +728,7 @@ void configSetCommand(client *c) {
     if (0) { /* this starts the config_set macros else-if chain. */
 
     /* Special fields that can't be handled with general macros. */
-    config_set_special_field("requirepass") {
-        if (sdslen(o->ptr) > CONFIG_AUTHPASS_MAX_LEN) goto badfmt;
-        /* The old "requirepass" directive just translates to setting
-         * a password to the default user. The only thing we do
-         * additionally is to remember the cleartext password in this
-         * case, for backward compatibility with Redis <= 5. */
-        ACLSetUser(DefaultUser,"resetpass",-1);
-        sdsfree(server.requirepass);
-        server.requirepass = NULL;
-        if (sdslen(o->ptr)) {
-            sds aclop = sdscatlen(sdsnew(">"), o->ptr, sdslen(o->ptr));
-            ACLSetUser(DefaultUser,aclop,sdslen(aclop));
-            sdsfree(aclop);
-            server.requirepass = sdsdup(o->ptr);
-        } else {
-            ACLSetUser(DefaultUser,"nopass",-1);
-        }
-    } config_set_special_field("save") {
+    config_set_special_field("save") {
         int vlen, j;
         sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen);
 
@@ -876,10 +837,6 @@ void configSetCommand(client *c) {
             enableWatchdog(ll);
         else
             disableWatchdog();
-    /* Memory fields.
-     * config_set_memory_field(name,var) */
-    } config_set_memory_field(
-      "client-query-buffer-limit",server.client_max_querybuf_len) {
     /* Everything else is an error... */
     } config_set_else {
         addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s",
@@ -959,7 +916,6 @@ void configGetCommand(client *c) {
     config_get_string_field("logfile",server.logfile);
 
     /* Numerical values */
-    config_get_numerical_field("client-query-buffer-limit",server.client_max_querybuf_len);
     config_get_numerical_field("watchdog-period",server.watchdog_period);
 
     /* Everything we can't handle with macros follows. */
@@ -1046,16 +1002,6 @@ void configGetCommand(client *c) {
         sdsfree(aux);
         matches++;
     }
-    if (stringmatch(pattern,"requirepass",1)) {
-        addReplyBulkCString(c,"requirepass");
-        sds password = server.requirepass;
-        if (password) {
-            addReplyBulkCBuffer(c,password,sdslen(password));
-        } else {
-            addReplyBulkCString(c,"");
-        }
-        matches++;
-    }
 
     if (stringmatch(pattern,"oom-score-adj-values",0)) {
         sds buf = sdsempty();
@@ -1564,26 +1510,6 @@ void rewriteConfigBindOption(struct rewriteConfigState *state) {
     rewriteConfigRewriteLine(state,option,line,force);
 }
 
-/* Rewrite the requirepass option. */
-void rewriteConfigRequirepassOption(struct rewriteConfigState *state, char *option) {
-    int force = 1;
-    sds line;
-    sds password = server.requirepass;
-
-    /* If there is no password set, we don't want the requirepass option
-     * to be present in the configuration at all. */
-    if (password == NULL) {
-        rewriteConfigMarkAsProcessed(state,option);
-        return;
-    }
-
-    line = sdsnew(option);
-    line = sdscatlen(line, " ", 1);
-    line = sdscatsds(line, password);
-
-    rewriteConfigRewriteLine(state,option,line,force);
-}
-
 /* Glue together the configuration lines in the current configuration
  * rewrite state into a single string, stripping multiple empty lines. */
 sds rewriteConfigGetContentFromState(struct rewriteConfigState *state) {
@@ -1740,8 +1666,6 @@ int rewriteConfig(char *path, int force_all) {
     rewriteConfigUserOption(state);
     rewriteConfigDirOption(state);
     rewriteConfigSlaveofOption(state,"replicaof");
-    rewriteConfigRequirepassOption(state,"requirepass");
-    rewriteConfigBytesOption(state,"client-query-buffer-limit",server.client_max_querybuf_len,PROTO_MAX_QUERYBUF_LEN);
     rewriteConfigStringOption(state,"cluster-config-file",server.cluster_configfile,CONFIG_DEFAULT_CLUSTER_CONFIG_FILE);
     rewriteConfigNotifykeyspaceeventsOption(state);
     rewriteConfigClientoutputbufferlimitOption(state);
@@ -2368,6 +2292,18 @@ static int updateOOMScoreAdj(int val, int prev, const char **err) {
     return 1;
 }
 
+
+int updateRequirePass(sds val, sds prev, const char **err) {
+    UNUSED(prev);
+    UNUSED(err);
+    /* The old "requirepass" directive just translates to setting
+     * a password to the default user. The only thing we do
+     * additionally is to remember the cleartext password in this
+     * case, for backward compatibility with Redis <= 5. */
+    ACLUpdateDefaultUserPassword(val);
+    return 1;
+}
+
 #ifdef USE_OPENSSL
 static int updateTlsCfg(char *val, char *prev, const char **err) {
     UNUSED(val);
@@ -2458,6 +2394,7 @@ standardConfig configs[] = {
 
     /* SDS Configs */
     createSDSConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL, NULL, NULL),
+    createSDSConfig("requirepass", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.requirepass, NULL, NULL, updateRequirePass),
 
     /* Enum Configs */
     createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE, NULL, NULL),
@@ -2534,6 +2471,7 @@ standardConfig configs[] = {
     createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL),
     createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, 3000, MEMORY_CONFIG, NULL, NULL),
     createSizeTConfig("tracking-table-max-keys", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.tracking_table_max_keys, 1000000, INTEGER_CONFIG, NULL, NULL), /* Default: 1 million keys max. */
+    createSizeTConfig("client-query-buffer-limit", NULL, MODIFIABLE_CONFIG, 1024*1024, LONG_MAX, server.client_max_querybuf_len, 1024*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Default: 1GB max query buffer. */
 
     /* Other configs */
     createTimeTConfig("repl-backlog-ttl", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.repl_backlog_time_limit, 60*60, INTEGER_CONFIG, NULL, NULL), /* Default: 1 hour */
diff --git a/src/server.c b/src/server.c
index f4dff6b0e..6e9dca3bc 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2653,7 +2653,6 @@ void initServerConfig(void) {
     server.sofd = -1;
     server.active_expire_enabled = 1;
     server.skip_checksum_validation = 0;
-    server.client_max_querybuf_len = PROTO_MAX_QUERYBUF_LEN;
     server.saveparams = NULL;
     server.loading = 0;
     server.loading_rdb_used_mem = 0;
@@ -3324,6 +3323,9 @@ void initServer(void) {
     scriptingInit(1);
     slowlogInit();
     latencyMonitorInit();
+    
+    /* Initialize ACL default password if it exists */
+    ACLUpdateDefaultUserPassword(server.requirepass);
 }
 
 /* Some steps in server initialization need to be done last (after modules
diff --git a/src/server.h b/src/server.h
index 55f718111..5bdcaae64 100644
--- a/src/server.h
+++ b/src/server.h
@@ -138,7 +138,6 @@ typedef long long ustime_t; /* microsecond time type. */
 #define STATS_METRIC_COUNT 3
 
 /* Protocol and I/O related defines */
-#define PROTO_MAX_QUERYBUF_LEN  (1024*1024*1024) /* 1GB max query buffer. */
 #define PROTO_IOBUF_LEN         (1024*16)  /* Generic I/O buffer size */
 #define PROTO_REPLY_CHUNK_BYTES (16*1024) /* 16k output buffer */
 #define PROTO_INLINE_MAX_SIZE   (1024*64) /* Max size of inline reads */
@@ -2106,6 +2105,7 @@ void addReplyCommandCategories(client *c, struct redisCommand *cmd);
 user *ACLCreateUnlinkedUser();
 void ACLFreeUserAndKillClients(user *u);
 void addACLLogEntry(client *c, int reason, int keypos, sds username);
+void ACLUpdateDefaultUserPassword(sds password);
 
 /* Sorted sets data type */
 

From 6122f1c450846584eedf614551de0c97da305b45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= <viktor.soderqvist@est.tech>
Date: Sun, 28 Feb 2021 13:11:18 +0100
Subject: [PATCH 1089/1098] Shared reusable client for RM_Call() (#8516)

A single client pointer is added in the server struct. This is
initialized by the first RM_Call() and reused for every subsequent
RM_Call() except if it's already in use, which means that it's not
used for (recursive) module calls to modules. For these, a new
"fake" client is created each time.

Other changes:
* Avoid allocating a dict iterator in pubsubUnsubscribeAllChannels
  when not needed
---
 src/module.c                  | 34 +++++++++++++++++++++++++++++-----
 src/pubsub.c                  | 14 ++++++++------
 src/server.h                  |  1 +
 tests/unit/moduleapi/misc.tcl |  5 +++++
 4 files changed, 43 insertions(+), 11 deletions(-)

diff --git a/src/module.c b/src/module.c
index fe0cd8345..9a37826fb 100644
--- a/src/module.c
+++ b/src/module.c
@@ -3969,16 +3969,26 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
     RedisModuleCallReply *reply = NULL;
     int replicate = 0; /* Replicate this command? */
 
-    /* Create the client and dispatch the command. */
+    /* Handle arguments. */
     va_start(ap, fmt);
-    c = createClient(NULL);
-    c->user = NULL; /* Root user. */
     argv = moduleCreateArgvFromUserFormat(cmdname,fmt,&argc,&flags,ap);
     replicate = flags & REDISMODULE_ARGV_REPLICATE;
     va_end(ap);
 
     /* Setup our fake client for command execution. */
-    c->flags |= CLIENT_MODULE;
+    if (server.module_client == NULL) {
+        /* This is the first RM_Call() ever. Create reusable client. */
+        c = server.module_client = createClient(NULL);
+    } else if (server.module_client->argv == NULL) {
+        /* The reusable module client is not busy with a command. Use it. */
+        c = server.module_client;
+    } else {
+        /* The reusable module client is busy. (It is probably used in a
+         * recursive call to this module.) */
+        c = createClient(NULL);
+    }
+    c->user = NULL; /* Root user. */
+    c->flags = CLIENT_MODULE;
 
     /* We do not want to allow block, the module do not expect it */
     c->flags |= CLIENT_DENY_BLOCKING;
@@ -4066,7 +4076,18 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
 
 cleanup:
     if (ctx->module) ctx->module->in_call--;
-    freeClient(c);
+    if (c == server.module_client) {
+        /* reset shared client so it can be reused */
+        discardTransaction(c);
+        pubsubUnsubscribeAllChannels(c,0);
+        pubsubUnsubscribeAllPatterns(c,0);
+        resetClient(c); /* frees the contents of argv */
+        zfree(c->argv);
+        c->argv = NULL;
+        c->resp = 2;
+    } else {
+        freeClient(c); /* temporary client */
+    }
     return reply;
 }
 
@@ -8269,6 +8290,9 @@ void moduleInitModulesSystem(void) {
     /* Set up filter list */
     moduleCommandFilters = listCreate();
 
+    /* Reusable client for RM_Call() is created on first use */
+    server.module_client = NULL;
+
     moduleRegisterCoreAPI();
     if (pipe(server.module_blocked_pipe) == -1) {
         serverLog(LL_WARNING,
diff --git a/src/pubsub.c b/src/pubsub.c
index a7b370d5d..5f7335bbe 100644
--- a/src/pubsub.c
+++ b/src/pubsub.c
@@ -250,18 +250,20 @@ int pubsubUnsubscribePattern(client *c, robj *pattern, int notify) {
 /* Unsubscribe from all the channels. Return the number of channels the
  * client was subscribed to. */
 int pubsubUnsubscribeAllChannels(client *c, int notify) {
-    dictIterator *di = dictGetSafeIterator(c->pubsub_channels);
-    dictEntry *de;
     int count = 0;
+    if (dictSize(c->pubsub_channels) > 0) {
+        dictIterator *di = dictGetSafeIterator(c->pubsub_channels);
+        dictEntry *de;
 
-    while((de = dictNext(di)) != NULL) {
-        robj *channel = dictGetKey(de);
+        while((de = dictNext(di)) != NULL) {
+            robj *channel = dictGetKey(de);
 
-        count += pubsubUnsubscribeChannel(c,channel,notify);
+            count += pubsubUnsubscribeChannel(c,channel,notify);
+        }
+        dictReleaseIterator(di);
     }
     /* We were subscribed to nothing? Still reply to the client. */
     if (notify && count == 0) addReplyPubsubUnsubscribed(c,NULL);
-    dictReleaseIterator(di);
     return count;
 }
 
diff --git a/src/server.h b/src/server.h
index 5bdcaae64..f52c2ee06 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1195,6 +1195,7 @@ struct redisServer {
                                    to be processed. */
     pid_t child_pid;            /* PID of current child */
     int child_type;             /* Type of current child */
+    client *module_client;      /* "Fake" client to call Redis from modules */
     /* Networking */
     int port;                   /* TCP listening port */
     int tls_port;               /* TLS listening port */
diff --git a/tests/unit/moduleapi/misc.tcl b/tests/unit/moduleapi/misc.tcl
index d60ccca8f..b5cd6100c 100644
--- a/tests/unit/moduleapi/misc.tcl
+++ b/tests/unit/moduleapi/misc.tcl
@@ -16,6 +16,11 @@ start_server {tags {"modules"}} {
         assert { [string match "*cmdstat_module*" $info] }
     }
 
+    test {test RM_Call recursive} {
+        set info [r test.call_generic test.call_generic info commandstats]
+        assert { [string match "*cmdstat_module*" $info] }
+    }
+
     test {test redis version} {
         set version [s redis_version]
         assert_equal $version [r test.redisversion]

From 17c226b070ef4ecdbcc5b4b5b0afe571f0f1a948 Mon Sep 17 00:00:00 2001
From: Bonsai <LetBonsaiBe@gmail.com>
Date: Sun, 28 Feb 2021 06:36:37 -0600
Subject: [PATCH 1090/1098] Module API for getting user name of a client
 (#8508)

Adding RM_GetClientUserNameById to get the ACL user name of a client connection.
---
 src/module.c      | 23 +++++++++++++++++++++++
 src/redismodule.h |  2 ++
 2 files changed, 25 insertions(+)

diff --git a/src/module.c b/src/module.c
index 9a37826fb..274210590 100644
--- a/src/module.c
+++ b/src/module.c
@@ -1837,6 +1837,28 @@ unsigned long long RM_GetClientId(RedisModuleCtx *ctx) {
     return ctx->client->id;
 }
 
+/* Return the ACL user name used by the client with the specified client ID.
+ * Client ID can be obtained with RM_GetClientId() API. If the client does not
+ * exist, NULL is returned and errno is set to ENOENT. If the client isn't 
+ * using an ACL user, NULL is returned and errno is set to ENOTSUP */
+RedisModuleString *RM_GetClientUserNameById(RedisModuleCtx *ctx, uint64_t id) {
+    client *client = lookupClientByID(id);
+    if (client == NULL) {
+        errno = ENOENT;
+        return NULL;
+    }
+    
+    if (client->user == NULL) {
+        errno = ENOTSUP;
+        return NULL;
+    }
+
+    sds name = sdsnew(client->user->name);
+    robj *str = createObject(OBJ_STRING, name);
+    autoMemoryAdd(ctx, REDISMODULE_AM_STRING, str);
+    return str;
+}
+
 /* This is an helper for RM_GetClientInfoById() and other functions: given
  * a client, it populates the client info structure with the appropriate
  * fields depending on the version provided. If the version is not valid
@@ -9157,6 +9179,7 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(IsKeysPositionRequest);
     REGISTER_API(KeyAtPos);
     REGISTER_API(GetClientId);
+    REGISTER_API(GetClientUserNameById);
     REGISTER_API(GetContextFlags);
     REGISTER_API(AvoidReplicaTraffic);
     REGISTER_API(PoolAlloc);
diff --git a/src/redismodule.h b/src/redismodule.h
index 0c2801bea..ea271b82b 100644
--- a/src/redismodule.h
+++ b/src/redismodule.h
@@ -665,6 +665,7 @@ REDISMODULE_API long long (*RedisModule_StreamTrimByID)(RedisModuleKey *key, int
 REDISMODULE_API int (*RedisModule_IsKeysPositionRequest)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
 REDISMODULE_API void (*RedisModule_KeyAtPos)(RedisModuleCtx *ctx, int pos) REDISMODULE_ATTR;
 REDISMODULE_API unsigned long long (*RedisModule_GetClientId)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
+REDISMODULE_API RedisModuleString * (*RedisModule_GetClientUserNameById)(RedisModuleCtx *ctx, uint64_t id) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_GetClientInfoById)(void *ci, uint64_t id) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_PublishMessage)(RedisModuleCtx *ctx, RedisModuleString *channel, RedisModuleString *message) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_GetContextFlags)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
@@ -936,6 +937,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(IsKeysPositionRequest);
     REDISMODULE_GET_API(KeyAtPos);
     REDISMODULE_GET_API(GetClientId);
+    REDISMODULE_GET_API(GetClientUserNameById);
     REDISMODULE_GET_API(GetContextFlags);
     REDISMODULE_GET_API(AvoidReplicaTraffic);
     REDISMODULE_GET_API(PoolAlloc);

From 18ff8cd1fbe0d4c4bc6d090ef58a513ecb9767da Mon Sep 17 00:00:00 2001
From: Pavlo Yatsukhnenko <yatsukhnenko@users.noreply.github.com>
Date: Sun, 28 Feb 2021 15:28:09 +0200
Subject: [PATCH 1091/1098] Fix div by 0 in redis-cli cluster creation (#8553)

This could happen on an invalid use, when trying to create a cluster with
a single node and provide it's address 3 time to satisfy redis-cli requirements.
---
 src/redis-cli.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/redis-cli.c b/src/redis-cli.c
index 3930a2d8e..63d31f79a 100644
--- a/src/redis-cli.c
+++ b/src/redis-cli.c
@@ -2972,7 +2972,7 @@ static void clusterManagerOptimizeAntiAffinity(clusterManagerNodeArray *ipnodes,
                                                    ip_count,
                                                    &offenders,
                                                    &offending_len);
-        if (score == 0) break; // Optimal anti affinity reached
+        if (score == 0 || offending_len == 0) break; // Optimal anti affinity reached
         /* We'll try to randomly swap a slave's assigned master causing
          * an affinity problem with another random slave, to see if we
          * can improve the affinity. */

From a66814fd7ac2b37cefb43f2ec03dd8684ebcccf2 Mon Sep 17 00:00:00 2001
From: Qu Chen <QuChen88@users.noreply.github.com>
Date: Sun, 28 Feb 2021 21:54:52 -0800
Subject: [PATCH 1092/1098] Make dbid range check for SWAPDB command consistent
 with SELECT, MOVE, and COPY. (#8555)

DB ID used to be parsed as a long for SWAPDB command, now
make it into an int to be consistent with other commands that parses
the DB ID argument like SELECT, MOVE, COPY. See #8085

The implication is that the error message when the provided db index
is greater than 4M changes slightly.
---
 src/db.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/db.c b/src/db.c
index ed522401f..57705f003 100644
--- a/src/db.c
+++ b/src/db.c
@@ -1315,7 +1315,7 @@ void scanDatabaseForReadyLists(redisDb *db) {
  *
  * Returns C_ERR if at least one of the DB ids are out of range, otherwise
  * C_OK is returned. */
-int dbSwapDatabases(long id1, long id2) {
+int dbSwapDatabases(int id1, int id2) {
     if (id1 < 0 || id1 >= server.dbnum ||
         id2 < 0 || id2 >= server.dbnum) return C_ERR;
     if (id1 == id2) return C_OK;
@@ -1356,7 +1356,7 @@ int dbSwapDatabases(long id1, long id2) {
 
 /* SWAPDB db1 db2 */
 void swapdbCommand(client *c) {
-    long id1, id2;
+    int id1, id2;
 
     /* Not allowed in cluster mode: we have just DB 0 there. */
     if (server.cluster_enabled) {
@@ -1365,11 +1365,11 @@ void swapdbCommand(client *c) {
     }
 
     /* Get the two DBs indexes. */
-    if (getLongFromObjectOrReply(c, c->argv[1], &id1,
+    if (getIntFromObjectOrReply(c, c->argv[1], &id1,
         "invalid first DB index") != C_OK)
         return;
 
-    if (getLongFromObjectOrReply(c, c->argv[2], &id2,
+    if (getIntFromObjectOrReply(c, c->argv[2], &id2,
         "invalid second DB index") != C_OK)
         return;
 

From 1bf02e1ca9bdb95bb37d806800c356ef13afd001 Mon Sep 17 00:00:00 2001
From: David Gilman <davidgilman1@gmail.com>
Date: Mon, 1 Mar 2021 01:15:26 -0500
Subject: [PATCH 1093/1098] Fix compliation on arm64 Mac with jemalloc (#8458)

The arm_thread_state64_get_pc used later in the file is defined
in mach kernel headers. Apparently they get included if you use
the system malloc but not if you use jemalloc.
---
 src/debug.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/debug.c b/src/debug.c
index a725e5a30..e7fec293a 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -54,6 +54,10 @@ typedef ucontext_t sigcontext_t;
 #endif
 #endif
 
+#if defined(__APPLE__) && defined(__arm64__)
+#include <mach/mach.h>
+#endif
+
 /* Globals */
 static int bug_report_start = 0; /* True if bug report header was already logged. */
 static pthread_mutex_t bug_report_start_mutex = PTHREAD_MUTEX_INITIALIZER;

From 81a55d026ffe6614f78e4042921713112593457c Mon Sep 17 00:00:00 2001
From: Bonsai <LetBonsaiBe@gmail.com>
Date: Mon, 1 Mar 2021 00:18:14 -0600
Subject: [PATCH 1094/1098] fix: call CLIENT INFO from redis module will crash
 the server (#8560)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Because when the RM_Call is invoked. It will create a faker client.
The point is client connection is NULL, so server will crash in connGetInfo

Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
---
 src/connection.c              | 2 +-
 tests/unit/moduleapi/misc.tcl | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/connection.c b/src/connection.c
index 3e2021fb5..a59463220 100644
--- a/src/connection.c
+++ b/src/connection.c
@@ -427,7 +427,7 @@ int connGetState(connection *conn) {
  * For sockets, we always return "fd=<fdnum>" to maintain compatibility.
  */
 const char *connGetInfo(connection *conn, char *buf, size_t buf_len) {
-    snprintf(buf, buf_len-1, "fd=%i", conn->fd);
+    snprintf(buf, buf_len-1, "fd=%i", conn == NULL ? -1 : conn->fd);
     return buf;
 }
 
diff --git a/tests/unit/moduleapi/misc.tcl b/tests/unit/moduleapi/misc.tcl
index b5cd6100c..a6a7a78f9 100644
--- a/tests/unit/moduleapi/misc.tcl
+++ b/tests/unit/moduleapi/misc.tcl
@@ -111,4 +111,8 @@ start_server {tags {"modules"}} {
         r test.log_tsctx "info" "Test message"
         verify_log_message 0 "*<misc> Test message*" 0
     }
+
+    test {test RM_Call CLIENT INFO} {
+        assert_match "*fd=-1*" [r test.call_generic client info]
+    }
 }

From a031d268b127d67969e9a4ec6b728dbb321ea0ce Mon Sep 17 00:00:00 2001
From: YaacovHazan <31382944+YaacovHazan@users.noreply.github.com>
Date: Mon, 1 Mar 2021 16:04:44 +0200
Subject: [PATCH 1095/1098] Make port, tls-port and bind configurations
 modifiable (#8510)

Add ability to modify port, tls-port and bind configurations by CONFIG SET command.

To simplify the code and make it cleaner, a new structure
added, socketFds, which contains the file descriptors array and its counter,
and used for TCP, TLS and Cluster sockets file descriptors.
---
 src/cluster.c                |  18 +---
 src/config.c                 |  57 ++++++++++-
 src/server.c                 | 180 +++++++++++++++++++++++++++++------
 src/server.h                 |  19 ++--
 tests/test_helper.tcl        |   1 +
 tests/unit/introspection.tcl |   4 +-
 tests/unit/networking.tcl    |  34 +++++++
 tests/unit/tls.tcl           |  25 +++++
 8 files changed, 282 insertions(+), 56 deletions(-)
 create mode 100644 tests/unit/networking.tcl

diff --git a/src/cluster.c b/src/cluster.c
index efe2f652d..4605e3ea9 100644
--- a/src/cluster.c
+++ b/src/cluster.c
@@ -499,7 +499,7 @@ void clusterInit(void) {
     if (saveconf) clusterSaveConfigOrDie(1);
 
     /* We need a listening TCP port for our cluster messaging needs. */
-    server.cfd_count = 0;
+    server.cfd.count = 0;
 
     /* Port sanity check II
      * The other handshake port check is triggered too late to stop
@@ -512,19 +512,11 @@ void clusterInit(void) {
                    "Your Redis port number must be 55535 or less.");
         exit(1);
     }
-    if (listenToPort(port+CLUSTER_PORT_INCR,
-        server.cfd,&server.cfd_count) == C_ERR)
-    {
+    if (listenToPort(port+CLUSTER_PORT_INCR, &server.cfd) == C_ERR) {
         exit(1);
-    } else {
-        int j;
-
-        for (j = 0; j < server.cfd_count; j++) {
-            if (aeCreateFileEvent(server.el, server.cfd[j], AE_READABLE,
-                clusterAcceptHandler, NULL) == AE_ERR)
-                    serverPanic("Unrecoverable error creating Redis Cluster "
-                                "file event.");
-        }
+    }
+    if (createSocketAcceptHandler(&server.cfd, clusterAcceptHandler) != C_OK) {
+        serverPanic("Unrecoverable error creating Redis Cluster socket accept handler.");
     }
 
     /* The slots -> keys map is a radix tree. Initialize it here. */
diff --git a/src/config.c b/src/config.c
index 13a6dee14..1d6ce6b8c 100644
--- a/src/config.c
+++ b/src/config.c
@@ -728,7 +728,23 @@ void configSetCommand(client *c) {
     if (0) { /* this starts the config_set macros else-if chain. */
 
     /* Special fields that can't be handled with general macros. */
-    config_set_special_field("save") {
+    config_set_special_field("bind") {
+        int vlen;
+        sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen);
+
+        if (vlen < 1 || vlen > CONFIG_BINDADDR_MAX) {
+            addReplyError(c, "Too many bind addresses specified.");
+            sdsfreesplitres(v, vlen);
+            return;
+        }
+
+        if (changeBindAddr(v, vlen) == C_ERR) {
+            addReplyError(c, "Failed to bind to specified addresses.");
+            sdsfreesplitres(v, vlen);
+            return;
+        }
+        sdsfreesplitres(v, vlen);
+    } config_set_special_field("save") {
         int vlen, j;
         sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen);
 
@@ -2191,6 +2207,20 @@ static int updateHZ(long long val, long long prev, const char **err) {
     return 1;
 }
 
+static int updatePort(long long val, long long prev, const char **err) {
+    /* Do nothing if port is unchanged */
+    if (val == prev) {
+        return 1;
+    }
+
+    if (changeListenPort(val, &server.ipfd, acceptTcpHandler) == C_ERR) {
+        *err = "Unable to listen on this port. Check server logs.";
+        return 0;
+    }
+
+    return 1;
+}
+
 static int updateJemallocBgThread(int val, int prev, const char **err) {
     UNUSED(prev);
     UNUSED(err);
@@ -2329,6 +2359,27 @@ static int updateTlsCfgInt(long long val, long long prev, const char **err) {
     UNUSED(prev);
     return updateTlsCfg(NULL, NULL, err);
 }
+
+static int updateTLSPort(long long val, long long prev, const char **err) {
+    /* Do nothing if port is unchanged */
+    if (val == prev) {
+        return 1;
+    }
+
+    /* Configure TLS if tls is enabled */
+    if (prev == 0 && tlsConfigure(&server.tls_ctx_config) == C_ERR) {
+        *err = "Unable to update TLS configuration. Check server logs.";
+        return 0;
+    }
+
+    if (changeListenPort(val, &server.tlsfd, acceptTLSHandler) == C_ERR) {
+        *err = "Unable to listen on this port. Check server logs.";
+        return 0;
+    }
+
+    return 1;
+}
+
 #endif  /* USE_OPENSSL */
 
 standardConfig configs[] = {
@@ -2409,7 +2460,7 @@ standardConfig configs[] = {
 
     /* Integer configs */
     createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, 16, INTEGER_CONFIG, NULL, NULL),
-    createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.port, 6379, INTEGER_CONFIG, NULL, NULL), /* TCP port. */
+    createIntConfig("port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.port, 6379, INTEGER_CONFIG, NULL, updatePort), /* TCP port. */
     createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 128, server.io_threads_num, 1, INTEGER_CONFIG, NULL, NULL), /* Single threaded by default */
     createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.aof_rewrite_perc, 100, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_slave_validity_factor, 10, INTEGER_CONFIG, NULL, NULL), /* Slave max data age factor. */
@@ -2478,7 +2529,7 @@ standardConfig configs[] = {
     createOffTConfig("auto-aof-rewrite-min-size", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.aof_rewrite_min_size, 64*1024*1024, MEMORY_CONFIG, NULL, NULL),
 
 #ifdef USE_OPENSSL
-    createIntConfig("tls-port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG, NULL, updateTlsCfgInt), /* TCP port. */
+    createIntConfig("tls-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG, NULL, updateTLSPort), /* TCP port. */
     createIntConfig("tls-session-cache-size", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tls_ctx_config.session_cache_size, 20*1024, INTEGER_CONFIG, NULL, updateTlsCfgInt),
     createIntConfig("tls-session-cache-timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tls_ctx_config.session_cache_timeout, 300, INTEGER_CONFIG, NULL, updateTlsCfgInt),
     createBoolConfig("tls-cluster", NULL, MODIFIABLE_CONFIG, server.tls_cluster, 0, NULL, updateTlsCfgBool),
diff --git a/src/server.c b/src/server.c
index 6e9dca3bc..4784431ba 100644
--- a/src/server.c
+++ b/src/server.c
@@ -2648,8 +2648,8 @@ void initServerConfig(void) {
     server.arch_bits = (sizeof(long) == 8) ? 64 : 32;
     server.bindaddr_count = 0;
     server.unixsocketperm = CONFIG_DEFAULT_UNIX_SOCKET_PERM;
-    server.ipfd_count = 0;
-    server.tlsfd_count = 0;
+    server.ipfd.count = 0;
+    server.tlsfd.count = 0;
     server.sofd = -1;
     server.active_expire_enabled = 1;
     server.skip_checksum_validation = 0;
@@ -2988,6 +2988,34 @@ void checkTcpBacklogSettings(void) {
 #endif
 }
 
+void closeSocketListeners(socketFds *sfd) {
+    int j;
+
+    for (j = 0; j < sfd->count; j++) {
+        if (sfd->fd[j] == -1) continue;
+
+        aeDeleteFileEvent(server.el, sfd->fd[j], AE_READABLE);
+        close(sfd->fd[j]);
+    }
+
+    sfd->count = 0;
+}
+
+/* Create an event handler for accepting new connections in TCP or TLS domain sockets.
+ * This works atomically for all socket fds */
+int createSocketAcceptHandler(socketFds *sfd, aeFileProc *accept_handler) {
+    int j;
+
+    for (j = 0; j < sfd->count; j++) {
+        if (aeCreateFileEvent(server.el, sfd->fd[j], AE_READABLE, accept_handler,NULL) == AE_ERR) {
+            /* Rollback */
+            for (j = j-1; j >= 0; j--) aeDeleteFileEvent(server.el, sfd->fd[j], AE_READABLE);
+            return C_ERR;
+        }
+    }
+    return C_OK;
+}
+
 /* Initialize a set of file descriptors to listen to the specified 'port'
  * binding the addresses specified in the Redis server configuration.
  *
@@ -3006,7 +3034,7 @@ void checkTcpBacklogSettings(void) {
  * impossible to bind, or no bind addresses were specified in the server
  * configuration but the function is not able to bind * for at least
  * one of the IPv4 or IPv6 protocols. */
-int listenToPort(int port, int *fds, int *count) {
+int listenToPort(int port, socketFds *sfd) {
     int j;
     char **bindaddr = server.bindaddr;
     int bindaddr_count = server.bindaddr_count;
@@ -3024,12 +3052,12 @@ int listenToPort(int port, int *fds, int *count) {
         if (optional) addr++;
         if (strchr(addr,':')) {
             /* Bind IPv6 address. */
-            fds[*count] = anetTcp6Server(server.neterr,port,addr,server.tcp_backlog);
+            sfd->fd[sfd->count] = anetTcp6Server(server.neterr,port,addr,server.tcp_backlog);
         } else {
             /* Bind IPv4 address. */
-            fds[*count] = anetTcpServer(server.neterr,port,addr,server.tcp_backlog);
+            sfd->fd[sfd->count] = anetTcpServer(server.neterr,port,addr,server.tcp_backlog);
         }
-        if (fds[*count] == ANET_ERR) {
+        if (sfd->fd[sfd->count] == ANET_ERR) {
             serverLog(LL_WARNING,
                 "Could not create server TCP listening socket %s:%d: %s",
                 addr, port, server.neterr);
@@ -3039,11 +3067,14 @@ int listenToPort(int port, int *fds, int *count) {
                 errno == ESOCKTNOSUPPORT || errno == EPFNOSUPPORT ||
                 errno == EAFNOSUPPORT)
                 continue;
+
+            /* Rollback successful listens before exiting */
+            closeSocketListeners(sfd);
             return C_ERR;
         }
-        anetNonBlock(NULL,fds[*count]);
-        anetCloexec(fds[*count]);
-        (*count)++;
+        anetNonBlock(NULL,sfd->fd[sfd->count]);
+        anetCloexec(sfd->fd[sfd->count]);
+        sfd->count++;
     }
     return C_OK;
 }
@@ -3165,10 +3196,10 @@ void initServer(void) {
 
     /* Open the TCP listening socket for the user commands. */
     if (server.port != 0 &&
-        listenToPort(server.port,server.ipfd,&server.ipfd_count) == C_ERR)
+        listenToPort(server.port,&server.ipfd) == C_ERR)
         exit(1);
     if (server.tls_port != 0 &&
-        listenToPort(server.tls_port,server.tlsfd,&server.tlsfd_count) == C_ERR)
+        listenToPort(server.tls_port,&server.tlsfd) == C_ERR)
         exit(1);
 
     /* Open the listening Unix domain socket. */
@@ -3185,7 +3216,7 @@ void initServer(void) {
     }
 
     /* Abort if there are no listening sockets at all. */
-    if (server.ipfd_count == 0 && server.tlsfd_count == 0 && server.sofd < 0) {
+    if (server.ipfd.count == 0 && server.tlsfd.count == 0 && server.sofd < 0) {
         serverLog(LL_WARNING, "Configured to not listen anywhere, exiting.");
         exit(1);
     }
@@ -3263,21 +3294,11 @@ void initServer(void) {
 
     /* Create an event handler for accepting new connections in TCP and Unix
      * domain sockets. */
-    for (j = 0; j < server.ipfd_count; j++) {
-        if (aeCreateFileEvent(server.el, server.ipfd[j], AE_READABLE,
-            acceptTcpHandler,NULL) == AE_ERR)
-            {
-                serverPanic(
-                    "Unrecoverable error creating server.ipfd file event.");
-            }
+    if (createSocketAcceptHandler(&server.ipfd, acceptTcpHandler) != C_OK) {
+        serverPanic("Unrecoverable error creating TCP socket accept handler.");
     }
-    for (j = 0; j < server.tlsfd_count; j++) {
-        if (aeCreateFileEvent(server.el, server.tlsfd[j], AE_READABLE,
-            acceptTLSHandler,NULL) == AE_ERR)
-            {
-                serverPanic(
-                    "Unrecoverable error creating server.tlsfd file event.");
-            }
+    if (createSocketAcceptHandler(&server.tlsfd, acceptTLSHandler) != C_OK) {
+        serverPanic("Unrecoverable error creating TLS socket accept handler.");
     }
     if (server.sofd > 0 && aeCreateFileEvent(server.el,server.sofd,AE_READABLE,
         acceptUnixHandler,NULL) == AE_ERR) serverPanic("Unrecoverable error creating server.sofd file event.");
@@ -4182,11 +4203,11 @@ void incrementErrorCount(const char *fullerr, size_t namelen) {
 void closeListeningSockets(int unlink_unix_socket) {
     int j;
 
-    for (j = 0; j < server.ipfd_count; j++) close(server.ipfd[j]);
-    for (j = 0; j < server.tlsfd_count; j++) close(server.tlsfd[j]);
+    for (j = 0; j < server.ipfd.count; j++) close(server.ipfd.fd[j]);
+    for (j = 0; j < server.tlsfd.count; j++) close(server.tlsfd.fd[j]);
     if (server.sofd != -1) close(server.sofd);
     if (server.cluster_enabled)
-        for (j = 0; j < server.cfd_count; j++) close(server.cfd[j]);
+        for (j = 0; j < server.cfd.count; j++) close(server.cfd.fd[j]);
     if (unlink_unix_socket && server.unixsocket) {
         serverLog(LL_NOTICE,"Removing the unix socket file.");
         unlink(server.unixsocket); /* don't care if this fails */
@@ -5536,6 +5557,105 @@ void redisAsciiArt(void) {
     zfree(buf);
 }
 
+int changeBindAddr(sds *addrlist, int addrlist_len) {
+    int i;
+    int result = C_OK;
+
+    char *prev_bindaddr[CONFIG_BINDADDR_MAX];
+    int prev_bindaddr_count;
+
+    /* Close old TCP and TLS servers */
+    closeSocketListeners(&server.ipfd);
+    closeSocketListeners(&server.tlsfd);
+
+    /* Keep previous settings */
+    prev_bindaddr_count = server.bindaddr_count;
+    memcpy(prev_bindaddr, server.bindaddr, sizeof(server.bindaddr));
+
+    /* Copy new settings */
+    memset(server.bindaddr, 0, sizeof(server.bindaddr));
+    for (i = 0; i < addrlist_len; i++) {
+        server.bindaddr[i] = zstrdup(addrlist[i]);
+    }
+    server.bindaddr_count = addrlist_len;
+
+    /* Bind to the new port */
+    if ((server.port != 0 && listenToPort(server.port, &server.ipfd) != C_OK) ||
+        (server.tls_port != 0 && listenToPort(server.tls_port, &server.tlsfd) != C_OK)) {
+        serverLog(LL_WARNING, "Failed to bind, trying to restore old listening sockets.");
+
+        /* Restore old bind addresses */
+        for (i = 0; i < addrlist_len; i++) {
+            zfree(server.bindaddr[i]);
+        }
+        memcpy(server.bindaddr, prev_bindaddr, sizeof(server.bindaddr));
+        server.bindaddr_count = prev_bindaddr_count;
+
+        /* Re-Listen TCP and TLS */
+        server.ipfd.count = 0;
+        if (server.port != 0 && listenToPort(server.port, &server.ipfd) != C_OK) {
+            serverPanic("Failed to restore old listening sockets.");
+        }
+
+        server.tlsfd.count = 0;
+        if (server.tls_port != 0 && listenToPort(server.tls_port, &server.tlsfd) != C_OK) {
+            serverPanic("Failed to restore old listening sockets.");
+        }
+
+        result = C_ERR;
+    } else {
+        /* Free old bind addresses */
+        for (i = 0; i < prev_bindaddr_count; i++) {
+            zfree(prev_bindaddr[i]);
+        }
+    }
+
+    /* Create TCP and TLS event handlers */
+    if (createSocketAcceptHandler(&server.ipfd, acceptTcpHandler) != C_OK) {
+        serverPanic("Unrecoverable error creating TCP socket accept handler.");
+    }
+    if (createSocketAcceptHandler(&server.tlsfd, acceptTLSHandler) != C_OK) {
+        serverPanic("Unrecoverable error creating TLS socket accept handler.");
+    }
+
+    if (server.set_proc_title) redisSetProcTitle(NULL);
+
+    return result;
+}
+
+int changeListenPort(int port, socketFds *sfd, aeFileProc *accept_handler) {
+    socketFds new_sfd = {{0}};
+
+    /* Just close the server if port disabled */
+    if (port == 0) {
+        closeSocketListeners(sfd);
+        if (server.set_proc_title) redisSetProcTitle(NULL);
+        return C_OK;
+    }
+
+    /* Bind to the new port */
+    if (listenToPort(port, &new_sfd) != C_OK) {
+        return C_ERR;
+    }
+
+    /* Create event handlers */
+    if (createSocketAcceptHandler(&new_sfd, accept_handler) != C_OK) {
+        closeSocketListeners(&new_sfd);
+        return C_ERR;
+    }
+
+    /* Close old servers */
+    closeSocketListeners(sfd);
+
+    /* Copy new descriptors */
+    sfd->count = new_sfd.count;
+    memcpy(sfd->fd, new_sfd.fd, sizeof(new_sfd.fd));
+
+    if (server.set_proc_title) redisSetProcTitle(NULL);
+
+    return C_OK;
+}
+
 static void sigShutdownHandler(int sig) {
     char *msg;
 
@@ -6124,7 +6244,7 @@ int main(int argc, char **argv) {
                 exit(1);
             }
         }
-        if (server.ipfd_count > 0 || server.tlsfd_count > 0)
+        if (server.ipfd.count > 0 || server.tlsfd.count > 0)
             serverLog(LL_NOTICE,"Ready to accept connections");
         if (server.sofd > 0)
             serverLog(LL_NOTICE,"The server is now ready to accept connections at %s", server.unixsocket);
diff --git a/src/server.h b/src/server.h
index f52c2ee06..e241bad70 100644
--- a/src/server.h
+++ b/src/server.h
@@ -1104,6 +1104,11 @@ struct malloc_stats {
     size_t allocator_resident;
 };
 
+typedef struct socketFds {
+    int fd[CONFIG_BINDADDR_MAX];
+    int count;
+} socketFds;
+
 /*-----------------------------------------------------------------------------
  * TLS Context Configuration
  *----------------------------------------------------------------------------*/
@@ -1204,13 +1209,10 @@ struct redisServer {
     int bindaddr_count;         /* Number of addresses in server.bindaddr[] */
     char *unixsocket;           /* UNIX socket path */
     mode_t unixsocketperm;      /* UNIX socket permission */
-    int ipfd[CONFIG_BINDADDR_MAX]; /* TCP socket file descriptors */
-    int ipfd_count;             /* Used slots in ipfd[] */
-    int tlsfd[CONFIG_BINDADDR_MAX]; /* TLS socket file descriptors */
-    int tlsfd_count;            /* Used slots in tlsfd[] */
+    socketFds ipfd;             /* TCP socket file descriptors */
+    socketFds tlsfd;            /* TLS socket file descriptors */
     int sofd;                   /* Unix socket file descriptor */
-    int cfd[CONFIG_BINDADDR_MAX];/* Cluster bus listening socket */
-    int cfd_count;              /* Used slots in cfd[] */
+    socketFds cfd;              /* Cluster bus listening socket */
     list *clients;              /* List of active clients */
     list *clients_to_close;     /* Clients to close asynchronously */
     list *clients_pending_write; /* There is to write or install handler. */
@@ -1855,7 +1857,7 @@ int getClientTypeByName(char *name);
 char *getClientTypeName(int class);
 void flushSlavesOutputBuffers(void);
 void disconnectSlaves(void);
-int listenToPort(int port, int *fds, int *count);
+int listenToPort(int port, socketFds *fds);
 void pauseClients(mstime_t duration, pause_type type);
 void unpauseClients(void);
 int areClientsPaused(void);
@@ -2184,6 +2186,9 @@ int processCommand(client *c);
 int processPendingCommandsAndResetClient(client *c);
 void setupSignalHandlers(void);
 void removeSignalHandlers(void);
+int createSocketAcceptHandler(socketFds *sfd, aeFileProc *accept_handler);
+int changeListenPort(int port, socketFds *sfd, aeFileProc *accept_handler);
+int changeBindAddr(sds *addrlist, int addrlist_len);
 struct redisCommand *lookupCommand(sds name);
 struct redisCommand *lookupCommandByCString(const char *s);
 struct redisCommand *lookupCommandOrOriginal(sds name);
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index eb14b5ee9..3c7d46d57 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -75,6 +75,7 @@ set ::all_tests {
     unit/tracking
     unit/oom-score-adj
     unit/shutdown
+    unit/networking
 }
 # Index to the next test to run in the ::all_tests list.
 set ::next_test 0
diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl
index c1cf72f0c..698ed7789 100644
--- a/tests/unit/introspection.tcl
+++ b/tests/unit/introspection.tcl
@@ -100,13 +100,10 @@ start_server {tags {"introspection"}} {
             supervised
             syslog-facility
             databases
-            port
-            tls-port
             io-threads
             logfile
             unixsocketperm
             slaveof
-            bind
             requirepass
             server_cpulist
             bio_cpulist
@@ -131,6 +128,7 @@ start_server {tags {"introspection"}} {
                 tls-protocols
                 tls-ciphers
                 tls-ciphersuites
+                tls-port
             }
         }
 
diff --git a/tests/unit/networking.tcl b/tests/unit/networking.tcl
new file mode 100644
index 000000000..1b5f19709
--- /dev/null
+++ b/tests/unit/networking.tcl
@@ -0,0 +1,34 @@
+test {CONFIG SET port number} {
+    start_server {} {
+        # available port
+        set avail_port [find_available_port $::baseport $::portcount]
+        set rd [redis [srv 0 host] [srv 0 port] 0 0]
+        $rd CONFIG SET port $avail_port
+        $rd close
+        set rd [redis [srv 0 host] $avail_port 0 0]
+        $rd PING
+
+        # already inuse port
+        catch {$rd CONFIG SET port $::test_server_port} e
+        assert_match {*Unable to listen on this port*} $e
+        $rd close
+
+        # make sure server still listening on the previous port
+        set rd [redis [srv 0 host] $avail_port 0 0]
+        $rd PING
+        $rd close
+    }
+}
+
+test {CONFIG SET bind address} {
+    start_server {} {
+        # non-valid address
+        catch {r CONFIG SET bind "some.wrong.bind.address"} e
+        assert_match {*Failed to bind to specified addresses*} $e
+
+        # make sure server still bound to the previous address
+        set rd [redis [srv 0 host] [srv 0 port] 0 0]
+        $rd PING
+        $rd close
+    }
+}
\ No newline at end of file
diff --git a/tests/unit/tls.tcl b/tests/unit/tls.tcl
index a6c2f3c2c..14e06fcdf 100644
--- a/tests/unit/tls.tcl
+++ b/tests/unit/tls.tcl
@@ -114,5 +114,30 @@ start_server {tags {"tls"}} {
                 }
             }
         }
+
+        test {TLS: switch between tcp and tls ports} {
+            set srv_port [srv 0 port]
+
+            # TLS
+            set rd [redis [srv 0 host] $srv_port 0 1]
+            $rd PING
+
+            # TCP
+            $rd CONFIG SET tls-port 0
+            $rd CONFIG SET port $srv_port
+            $rd close
+
+            set rd [redis [srv 0 host] $srv_port 0 0]
+            $rd PING
+
+            # TLS
+            $rd CONFIG SET port 0
+            $rd CONFIG SET tls-port $srv_port
+            $rd close
+
+            set rd [redis [srv 0 host] $srv_port 0 1]
+            $rd PING
+            $rd close
+        }
     }
 }

From 349ef3f6a0c6a1e351caabcc5ec57c8e49a0f4c8 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 1 Mar 2021 17:23:29 +0200
Subject: [PATCH 1096/1098] fix stream deep sanitization with deleted records
 (#8568)

When sanitizing the stream listpack, we need to count the deleted records too.
otherwise the last line that checks the next pointer fails.

Add test to cover that state in the stream tests.
---
 src/t_stream.c                            | 3 ++-
 tests/integration/corrupt-dump-fuzzer.tcl | 1 +
 tests/integration/rdb.tcl                 | 1 +
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/t_stream.c b/src/t_stream.c
index 4a425fd78..18138bec5 100644
--- a/src/t_stream.c
+++ b/src/t_stream.c
@@ -3570,7 +3570,7 @@ int streamValidateListpackIntegrity(unsigned char *lp, size_t size, int deep) {
     p = next; if (!lpValidateNext(lp, &next, size)) return 0;
 
     /* deleted */
-    lpGetIntegerIfValid(p, &valid_record);
+    int64_t deleted_count = lpGetIntegerIfValid(p, &valid_record);
     if (!valid_record) return 0;
     p = next; if (!lpValidateNext(lp, &next, size)) return 0;
 
@@ -3589,6 +3589,7 @@ int streamValidateListpackIntegrity(unsigned char *lp, size_t size, int deep) {
     if (!valid_record || zero != 0) return 0;
     p = next; if (!lpValidateNext(lp, &next, size)) return 0;
 
+    entry_count += deleted_count;
     while (entry_count--) {
         if (!p) return 0;
         int64_t fields = master_fields, extra_fields = 3;
diff --git a/tests/integration/corrupt-dump-fuzzer.tcl b/tests/integration/corrupt-dump-fuzzer.tcl
index 2506205a5..4fb503b8e 100644
--- a/tests/integration/corrupt-dump-fuzzer.tcl
+++ b/tests/integration/corrupt-dump-fuzzer.tcl
@@ -32,6 +32,7 @@ proc generate_types {} {
     # add some metadata to the stream
     r xgroup create stream mygroup 0
     set records [r xreadgroup GROUP mygroup Alice COUNT 2 STREAMS stream >]
+    r xdel stream [lindex [lindex [lindex [lindex $records 0] 1] 1] 0]
     r xack stream mygroup [lindex [lindex [lindex [lindex $records 0] 1] 0] 0]
 
     # create other non-collection types
diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl
index 7df1e2f74..9e1c2651a 100644
--- a/tests/integration/rdb.tcl
+++ b/tests/integration/rdb.tcl
@@ -52,6 +52,7 @@ start_server [list overrides [list "dir" $server_path] keep_persistence true] {
         }
         r xgroup create stream mygroup 0
         set records [r xreadgroup GROUP mygroup Alice COUNT 2 STREAMS stream >]
+        r xdel stream [lindex [lindex [lindex [lindex $records 0] 1] 1] 0]
         r xack stream mygroup [lindex [lindex [lindex [lindex $records 0] 1] 0] 0]
         set digest [r debug digest]
         r config set sanitize-dump-payload no

From c19530bc71760ce45ac1404272c13553a061fcb8 Mon Sep 17 00:00:00 2001
From: YaacovHazan <31382944+YaacovHazan@users.noreply.github.com>
Date: Mon, 1 Mar 2021 20:53:02 +0200
Subject: [PATCH 1097/1098] fix new networking tests to work when the test
 suite is used in tls mode (#8582)

the tests were unable to connect to the server since the attempted to use normal tcp
---
 tests/unit/networking.tcl | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/tests/unit/networking.tcl b/tests/unit/networking.tcl
index 1b5f19709..19feee8c3 100644
--- a/tests/unit/networking.tcl
+++ b/tests/unit/networking.tcl
@@ -1,20 +1,22 @@
 test {CONFIG SET port number} {
     start_server {} {
+        if {$::tls} { set port_cfg tls-port} else { set port_cfg port }
+
         # available port
         set avail_port [find_available_port $::baseport $::portcount]
-        set rd [redis [srv 0 host] [srv 0 port] 0 0]
-        $rd CONFIG SET port $avail_port
+        set rd [redis [srv 0 host] [srv 0 port] 0 $::tls]
+        $rd CONFIG SET $port_cfg $avail_port
         $rd close
-        set rd [redis [srv 0 host] $avail_port 0 0]
+        set rd [redis [srv 0 host] $avail_port 0 $::tls]
         $rd PING
 
         # already inuse port
-        catch {$rd CONFIG SET port $::test_server_port} e
+        catch {$rd CONFIG SET $port_cfg $::test_server_port} e
         assert_match {*Unable to listen on this port*} $e
         $rd close
 
         # make sure server still listening on the previous port
-        set rd [redis [srv 0 host] $avail_port 0 0]
+        set rd [redis [srv 0 host] $avail_port 0 $::tls]
         $rd PING
         $rd close
     }
@@ -27,7 +29,7 @@ test {CONFIG SET bind address} {
         assert_match {*Failed to bind to specified addresses*} $e
 
         # make sure server still bound to the previous address
-        set rd [redis [srv 0 host] [srv 0 port] 0 0]
+        set rd [redis [srv 0 host] [srv 0 port] 0 $::tls]
         $rd PING
         $rd close
     }

From 7018ad69a22c992765ae3f3497ac5684edac6a81 Mon Sep 17 00:00:00 2001
From: Oran Agra <oran@redislabs.com>
Date: Mon, 1 Mar 2021 21:02:05 +0200
Subject: [PATCH 1098/1098] Redis 6.2.1

---
 00-RELEASENOTES | 27 +++++++++++++++++++++++++++
 src/version.h   |  4 ++--
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/00-RELEASENOTES b/00-RELEASENOTES
index 36ef7156f..d94c60465 100644
--- a/00-RELEASENOTES
+++ b/00-RELEASENOTES
@@ -11,6 +11,33 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
 SECURITY: There are security fixes in the release.
 --------------------------------------------------------------------------------
 
+================================================================================
+Redis 6.2.1 Released Mon Mar  1 17:51:36 IST 2021
+================================================================================
+
+Upgrade urgency: LOW.
+
+Here is a comprehensive list of changes in this release compared to 6.2.0,
+each one includes the PR number that added it, so you can get more details
+at https://github.com/redis/redis/pull/<number>
+
+Bug fixes:
+* Fix sanitize-dump-payload for stream with deleted records (#8568)
+* Prevent client-query-buffer-limit config from being set to lower than 1mb (#8557)
+
+Improvements:
+* Make port, tls-port and bind config options modifiable at runtime (#8510)
+
+Platform and deployment-related changes:
+* Fix compilation error on non-glibc systems if jemalloc is not used (#8533)
+* Improved memory consumption and memory usage tracking on FreeBSD (#8545)
+* Fix compilation on ARM64 MacOS with jemalloc (#8458)
+
+Modules:
+* New Module API for getting user name of a client (#8508)
+* Optimize RM_Call by utilizing a shared reusable client (#8516)
+* Fix crash running CLIENT INFO via RM_Call (#8560)
+
 ================================================================================
 Redis 6.2.0 GA  Released Tue Feb 22 14:00:00 IST 2021
 ================================================================================
diff --git a/src/version.h b/src/version.h
index 62ddcfd59..64fdbe89b 100644
--- a/src/version.h
+++ b/src/version.h
@@ -1,2 +1,2 @@
-#define REDIS_VERSION "6.2.0"
-#define REDIS_VERSION_NUM 0x00060200
+#define REDIS_VERSION "6.2.1"
+#define REDIS_VERSION_NUM 0x00060201